diff --git a/apps/cv-worker/src/appearance.test.ts b/apps/cv-worker/src/appearance.test.ts index 6831d08..af00f19 100644 --- a/apps/cv-worker/src/appearance.test.ts +++ b/apps/cv-worker/src/appearance.test.ts @@ -9,6 +9,7 @@ import { toHsv, torsoRect, } from './appearance.js'; +import { decodePlanFor } from './signatures.js'; /** * The safety property under all of this: a signature that cannot tell two teams @@ -95,6 +96,39 @@ describe('the torso crop', () => { }); }); +describe('deciding what to decode, and at what scale', () => { + it('scales boxes from their own space, not from the file being read', () => { + /** + * The bug this exists for: tracks are in source-video pixels (1920x1080) + * while the file read is the 540p proxy. Deriving the space from the proxy + * scaled every crop by 1, put every torso off the right edge of the frame, + * and returned a confident zero matches on footage with eight to find. + */ + const plan = decodePlanFor(1920, 1080, 960); + expect(plan.decodeWidth).toBe(960); + expect(plan.decodeHeight).toBe(540); + expect(plan.scale).toBe(0.5); + }); + + it('never upscales past the footage it was given', () => { + const plan = decodePlanFor(640, 360, 960); + expect(plan.decodeWidth).toBe(640); + expect(plan.scale).toBe(1); + }); + + it('keeps the aspect ratio, so a box’s y scales like its x', () => { + const plan = decodePlanFor(1440, 1080, 720); + expect(plan.scale).toBe(0.5); + expect(plan.decodeHeight).toBe(540); + }); + + it('survives a video with no readable width instead of dividing by zero', () => { + const plan = decodePlanFor(0, 0, 960); + expect(Number.isFinite(plan.scale)).toBe(true); + expect(plan.decodeHeight).toBeGreaterThan(0); + }); +}); + describe('comparing two players', () => { it('matches a shirt against itself', () => { const red = signatureOf(solidFrame(40, 80, [30, 30, 200]), 40, 80); diff --git a/apps/cv-worker/src/index.ts b/apps/cv-worker/src/index.ts index 1064d94..1e4add6 100644 --- a/apps/cv-worker/src/index.ts +++ b/apps/cv-worker/src/index.ts @@ -215,9 +215,15 @@ const appearance = async (flags: Record): Promise => { return; } - let request: { boxes?: SignatureBox[] }; + interface Request { + boxes?: SignatureBox[]; + /** The pixel space the boxes are in — see below. */ + sourceWidth?: number; + sourceHeight?: number; + } + let request: Request; try { - request = JSON.parse(await readStdin()) as { boxes?: SignatureBox[] }; + request = JSON.parse(await readStdin()) as Request; } catch (cause) { emit({ error: `stdin was not the JSON box list this expects: ${String(cause)}` }); return; @@ -229,8 +235,19 @@ const appearance = async (flags: Record): Promise => { } const media = await probe(input); - const width = media.video?.width ?? 0; - const height = media.video?.height ?? 0; + /** + * The boxes' coordinate space travels with the boxes, and is not the same + * thing as the size of the file being decoded. + * + * Tracks are stored in source-video pixels, but this reads the 540p proxy + * because a shirt's colour survives that and decodes in a fraction of the + * time. Taking the space from the decoded file measured every torso against + * the wrong scale — crops landed off the edge of the frame, signatures came + * back empty or meaningless, and the result was a confident zero matches on + * footage where eight were there to be found. + */ + const width = request.sourceWidth ?? media.video?.width ?? 0; + const height = request.sourceHeight ?? media.video?.height ?? 0; if (width <= 0 || height <= 0) { emit({ error: `${input} has no readable video stream.` }); return; diff --git a/apps/cv-worker/src/signatures.integration.test.ts b/apps/cv-worker/src/signatures.integration.test.ts new file mode 100644 index 0000000..877cbf4 --- /dev/null +++ b/apps/cv-worker/src/signatures.integration.test.ts @@ -0,0 +1,133 @@ +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +import { similarity } from './appearance.js'; +import { computeSignatures } from './signatures.js'; + +/** + * The seam a unit test cannot reach: boxes in one coordinate space, frames + * decoded in another. + * + * Tracks are stored in source-video pixels while this reads the 540p proxy, and + * taking the scale from the decoded file put every torso crop off the edge of + * the frame. The unit tests all passed; the feature returned a confident zero + * matches on footage with eight to find. Only measuring real pixels catches it, + * so this builds a video whose colours are known and checks that a box given in + * a *larger* space still lands on the right one. + * + * Skipped where ffmpeg is absent; the container image has it. + */ + +const ffmpeg = spawnSync('ffmpeg', ['-version']); +const available = ffmpeg.status === 0; + +let dir: string; +let video: string; + +beforeAll(() => { + if (!available) return; + dir = mkdtempSync(path.join(tmpdir(), 'reeleel-signatures-')); + video = path.join(dir, 'bands.mp4'); + + /** + * Two seconds of a 640x360 clip: a red left half and a blue right half. The + * "source" space the boxes will be quoted in is twice that, so a correct + * implementation has to halve them before cropping. + */ + const result = spawnSync( + 'ffmpeg', + [ + '-hide_banner', + '-loglevel', + 'error', + '-f', + 'lavfi', + '-i', + 'color=c=red:s=320x360:d=2:r=10', + '-f', + 'lavfi', + '-i', + 'color=c=blue:s=320x360:d=2:r=10', + '-filter_complex', + '[0:v][1:v]hstack=inputs=2[v]', + '-map', + '[v]', + '-pix_fmt', + 'yuv420p', + video, + ], + { encoding: 'utf8' }, + ); + if (result.status !== 0) throw new Error(`could not build the fixture: ${result.stderr}`); +}); + +afterAll(() => { + if (dir !== undefined) rmSync(dir, { recursive: true, force: true }); +}); + +describe.skipIf(!available)('signatures from real pixels', () => { + /** A box in 1280x720 space, over a 640x360 video. */ + const box = (x: number) => ({ ts: 1, x, y: 100, w: 200, h: 400 }); + + it('scales boxes out of their own space and onto the decoded frame', async () => { + const result = await computeSignatures({ + input: video, + ffmpegPath: 'ffmpeg', + // Twice the video's real size: this is the bug's shape. + sourceWidth: 1280, + sourceHeight: 720, + fps: 10, + boxes: [ + { track: 'left', ...box(100) }, + { track: 'right', ...box(900) }, + ], + }); + + // Both crops must have landed on actual pixels. + expect(result.pixels['left']).toBeGreaterThan(0); + expect(result.pixels['right']).toBeGreaterThan(0); + + const left = result.signatures['left'] ?? []; + const right = result.signatures['right'] ?? []; + expect(left.length).toBeGreaterThan(0); + expect(right.length).toBeGreaterThan(0); + + // The whole point: one landed on red, the other on blue. Get the scaling + // wrong and both land on the same place, or on nothing. + expect(similarity(left, right)).toBeLessThan(0.2); + expect(similarity(left, left)).toBeCloseTo(1); + }); + + it('reads the same shirt as the same shirt from two moments', async () => { + const result = await computeSignatures({ + input: video, + ffmpegPath: 'ffmpeg', + sourceWidth: 1280, + sourceHeight: 720, + fps: 10, + boxes: [ + { track: 'early', ...box(100), ts: 0.5 }, + { track: 'late', ...box(100), ts: 1.5 }, + ], + }); + expect( + similarity(result.signatures['early'] ?? [], result.signatures['late'] ?? []), + ).toBeGreaterThan(0.9); + }); + + it('measures nothing for a box that is off the frame', async () => { + const result = await computeSignatures({ + input: video, + ffmpegPath: 'ffmpeg', + sourceWidth: 1280, + sourceHeight: 720, + fps: 10, + boxes: [{ track: 'gone', ts: 1, x: -4000, y: 100, w: 200, h: 400 }], + }); + expect(result.signatures['gone']).toBeUndefined(); + }); +}); diff --git a/apps/cv-worker/src/signatures.ts b/apps/cv-worker/src/signatures.ts index baa0f5e..0f46a7e 100644 --- a/apps/cv-worker/src/signatures.ts +++ b/apps/cv-worker/src/signatures.ts @@ -50,6 +50,30 @@ export interface SignatureResult { export const frameIndexFor = (ts: number, fps: number, stride: number): number => Math.max(0, Math.round((ts * fps) / stride) * stride); +/** + * How to decode, and what to multiply a box by once decoded. + * + * `sourceWidth`/`sourceHeight` are the space the *boxes* are in, which is not + * necessarily the size of the file being read: tracks are stored in + * source-video pixels while this usually reads the 540p proxy. Taking the space + * from the decoded file instead scaled every crop by 1 and put every torso rect + * somewhere off the right-hand edge, which produced empty signatures and a + * confident zero matches. + */ +export const decodePlanFor = ( + sourceWidth: number, + sourceHeight: number, + requestedWidth: number, +): { decodeWidth: number; decodeHeight: number; scale: number } => { + const decodeWidth = Math.min(requestedWidth, Math.max(1, sourceWidth)); + const scale = sourceWidth > 0 ? decodeWidth / sourceWidth : 1; + return { + decodeWidth, + decodeHeight: Math.max(2, Math.round(sourceHeight * scale)), + scale, + }; +}; + /** * Colour signatures for a set of tracks, from one pass over the video. * @@ -64,9 +88,11 @@ export const computeSignatures = async ( const perSecond = request.samplesPerSecond ?? 2; const stride = Math.max(1, Math.round(fps / perSecond)); - const decodeWidth = Math.min(request.decodeWidth ?? 960, request.sourceWidth); - const scale = request.sourceWidth > 0 ? decodeWidth / request.sourceWidth : 1; - const decodeHeight = Math.max(2, Math.round(request.sourceHeight * scale)); + const { decodeWidth, decodeHeight, scale } = decodePlanFor( + request.sourceWidth, + request.sourceHeight, + request.decodeWidth ?? 960, + ); // Boxes bucketed by the frame they will be measured on, so each decoded frame // is a single lookup rather than a scan of every box. diff --git a/packages/core/src/appearance.ts b/packages/core/src/appearance.ts index 62be5d1..55bba0d 100644 --- a/packages/core/src/appearance.ts +++ b/packages/core/src/appearance.ts @@ -4,221 +4,46 @@ import { getAthlete } from './athletes.js'; import { resolveCvWorker } from './analyze.js'; import { ReelEelError } from './errors.js'; import { run } from './ffmpeg.js'; +import { + candidatesFrom, + chooseAthleteTracks, + COLOUR_FLOOR, + mergeSignatures, + sampleBoxes, +} from './stitch.js'; +import type { AthleteProposal } from './stitch.js'; import { loadTrackSeries, tracksForAthlete } from './tracks.js'; -import type { TrackSeries } from './scoring.js'; import { listVideos } from './videos.js'; /** - * Finding the same child in the rest of the game. + * Finding the same child in the rest of the game — the plumbing half. * * Re-identification matched on box overlap, which can only ever confirm an - * athlete where they were already known — it recovers a binding across a - * re-detection and cannot do anything else. Measured on production, that left - * an athlete identified for 31.7s of a 300s game across six fragments, all of - * them inside the one 32-second window the user had originally pointed at. - * Every signal that follows the athlete was therefore dark for 90% of the - * footage, and the moments that survived were scene-wide ones that had nothing - * to do with them. + * athlete where they were already known. Measured on production, that left an + * athlete identified for 31.7s of a 300s game across six fragments, all inside + * the one 32-second window the user had originally pointed at. * - * The missing ingredient is appearance. Nothing here decides anything: it - * ranks, and a human confirms. A wrong answer puts another family's child in - * your highlight reel, so the design point throughout is that a weak match is - * dropped rather than guessed. + * Every decision lives in `stitch.js`, which touches nothing, so the shipped + * judgement can be run against real footage without a database or a worker + * around it. This file only fetches, calls and returns. */ -export interface AthleteProposal { - trackId: string; - /** Colour-signature agreement with the athlete's known tracks, 0..1. */ - score: number; - startTs: number; - endTs: number; - seconds: number; - samples: number; - /** The link that justified it, so a person can judge the claim. */ - gapSeconds: number; - distancePx: number; -} - -/** - * Colour is a veto, never an identifier. - * - * Measured on a real game: at a 0.55 colour match, 661 of 1152 candidate tracks - * qualified — 2,306 seconds of "athlete" in a 300-second video. That is not a - * tuning failure, it is what a shirt means. Teammates wear the same one, so a - * colour signature identifies a *team*, and the children it wrongly volunteers - * are precisely the ones standing next to yours. - * - * So the identity claim rests on continuity, and colour only ever rules a link - * out. The same measurement, three ways: continuity alone recovered 120.3s - * across 56 tracks (too permissive — it links whoever is nearby); colour alone - * 2,306s; both together 51.2s across 14 tracks, up from 31.7s across 6, with - * every link under a second of gap and a few hundred pixels of travel. - */ -export const COLOUR_FLOOR = 0.7; - -/** Longest silence a link may be drawn across. */ -export const MAX_LINK_SECONDS = 2; - -/** - * How far a child may travel between two fragments, as a fraction of frame - * width per second, plus a fixed allowance for the tracker's own jitter. - * - * At four seconds and this speed the accepted links reached 894 pixels of a - * 1920-wide frame — most of the way across a court — for eight more seconds of - * coverage. The gap limit above is where that trade stops being worth taking. - */ -export const LINK_SPEED_FRACTION = 0.31; -export const LINK_SLACK_FRACTION = 0.03; - -/** Time span a track occupies. */ -export const spanOf = (series: TrackSeries): { start: number; end: number } => { - const first = series.samples[0]; - const last = series.samples[series.samples.length - 1]; - if (first === undefined || last === undefined) return { start: 0, end: 0 }; - return { start: first.ts, end: last.ts }; -}; - -/** - * Whether two tracks are ever on screen at the same moment. - * - * The hardest constraint available and the cheapest: one child cannot be in two - * places at once, so a candidate that coexists with a track already known to be - * the athlete is definitively somebody else — however similar their shirt. - * Teammates wear the same colour, so without this the strongest false matches - * would be exactly the children standing next to them. - */ -export const overlapsInTime = (a: TrackSeries, b: TrackSeries): boolean => { - const first = spanOf(a); - const second = spanOf(b); - return first.start <= second.end && second.start <= first.end; -}; - -/** - * A handful of boxes spread across a track's life, rather than all of them. - * - * A signature wants variety — different moments, poses and lighting — not - * volume. Sampling every half-second and capping keeps a thirty-second track - * from drowning out a three-second one in the reference average. - */ -export const sampleBoxes = ( - series: TrackSeries, - everySeconds = 0.5, - cap = 12, -): { ts: number; x: number; y: number; w: number; h: number }[] => { - const picked: { ts: number; x: number; y: number; w: number; h: number }[] = []; - let nextTs = Number.NEGATIVE_INFINITY; - for (const sample of series.samples) { - if (sample.ts < nextTs) continue; - picked.push({ ts: sample.ts, x: sample.x, y: sample.y, w: sample.w, h: sample.h }); - nextTs = sample.ts + everySeconds; - } - if (picked.length <= cap) return picked; - - // Thin evenly rather than truncating, so the tail of a long track is still - // represented — a child who changes ends of the court is still that child. - const step = picked.length / cap; - return Array.from({ length: cap }, (_unused, i) => picked[Math.floor(i * step)]).filter( - (box): box is { ts: number; x: number; y: number; w: number; h: number } => box !== undefined, - ); -}; - -/** Weighted mean of several signatures, renormalized. */ -export const mergeSignatures = ( - parts: { signature: number[]; weight: number }[], -): number[] => { - const usable = parts.filter((part) => part.weight > 0 && part.signature.length > 0); - const first = usable[0]; - if (first === undefined) return []; - - const totals = new Array(first.signature.length).fill(0); - let weightSum = 0; - for (const part of usable) { - weightSum += part.weight; - for (let i = 0; i < totals.length; i += 1) { - totals[i] = (totals[i] ?? 0) + (part.signature[i] ?? 0) * part.weight; - } - } - if (weightSum <= 0) return []; - const scaled = totals.map((value) => value / weightSum); - const sum = scaled.reduce((a, b) => a + b, 0); - return sum > 0 ? scaled.map((value) => value / sum) : scaled; -}; - -const centreOf = (sample: { x: number; y: number; w: number; h: number }): { x: number; y: number } => ({ - x: sample.x + sample.w / 2, - y: sample.y + sample.h / 2, -}); - -export interface Link { - gapSeconds: number; - distancePx: number; -} - -/** - * Whether a candidate plausibly continues a known track — picking up where it - * left off, or leading into where it began. - * - * This is the part that actually claims identity, so it is deliberately mean: - * a short silence, and a distance a child could really have covered in it. Both - * directions, because a fragment can extend an athlete backwards just as - * usefully as forwards. - */ -export const linkBetween = ( - known: TrackSeries, - candidate: TrackSeries, - frameWidth: number, - maxSeconds = MAX_LINK_SECONDS, -): Link | null => { - const knownFirst = known.samples[0]; - const knownLast = known.samples[known.samples.length - 1]; - const otherFirst = candidate.samples[0]; - const otherLast = candidate.samples[candidate.samples.length - 1]; - if ( - knownFirst === undefined || - knownLast === undefined || - otherFirst === undefined || - otherLast === undefined - ) { - return null; - } - - const reach = (gap: number): number => - frameWidth * LINK_SPEED_FRACTION * gap + frameWidth * LINK_SLACK_FRACTION; - - const forward = otherFirst.ts - knownLast.ts; - if (forward > 0 && forward <= maxSeconds) { - const distance = Math.hypot( - centreOf(knownLast).x - centreOf(otherFirst).x, - centreOf(knownLast).y - centreOf(otherFirst).y, - ); - if (distance <= reach(forward)) return { gapSeconds: forward, distancePx: distance }; - } - - const backward = knownFirst.ts - otherLast.ts; - if (backward > 0 && backward <= maxSeconds) { - const distance = Math.hypot( - centreOf(knownFirst).x - centreOf(otherLast).x, - centreOf(knownFirst).y - centreOf(otherLast).y, - ); - if (distance <= reach(backward)) return { gapSeconds: backward, distancePx: distance }; - } - - return null; -}; - -/** Histogram intersection, mirroring the worker's own comparison. */ -export const similarity = (a: number[], b: number[]): number => { - const length = Math.min(a.length, b.length); - let shared = 0; - for (let i = 0; i < length; i += 1) shared += Math.min(a[i] ?? 0, b[i] ?? 0); - return shared; -}; +export * from './stitch.js'; export interface ProposalOptions { videoId?: string; - /** Ignore candidates shorter than this. Default 1.5s. */ + /** + * Ignore candidates shorter than this. Deliberately far lower than the + * picker's own floor. + * + * A human choosing by eye needs a crop long enough to recognise, so the grid + * hides anything under 1.5s. Stitching is the opposite case: the short + * fragments are the connective tissue, and five of the eight links that + * recovered a real athlete were under 1.5s. Continuity and colour justify + * them without anyone having to recognise a face in a third of a second. + */ minSeconds?: number; - /** Minimum agreement to propose at all. Default {@link PROPOSAL_THRESHOLD}. */ + /** Minimum agreement to accept a link at all. Default {@link COLOUR_FLOOR}. */ threshold?: number; /** Most proposals to return. Default 40. */ limit?: number; @@ -273,16 +98,7 @@ export const proposeAthleteTracks = async ( ); } - const minSeconds = options.minSeconds ?? 1.5; - const candidates = series.filter((track) => { - if (assigned.has(track.id)) return false; - if (track.className !== 'player') return false; - const { start, end } = spanOf(track); - if (end - start < minSeconds) return false; - // One child, one place at a time. - return !reference.some((known) => overlapsInTime(known, track)); - }); - + const candidates = candidatesFrom(series, reference, assigned, options.minSeconds ?? 0.25); if (candidates.length === 0) { return { proposals: [], referenceTrackIds: [...assigned], considered: 0 }; } @@ -294,6 +110,7 @@ export const proposeAthleteTracks = async ( }); } + const frameWidth = video.probe?.video?.width ?? 1920; const boxes = [...reference, ...candidates].flatMap((track) => sampleBoxes(track).map((box) => ({ track: track.id, ...box })), ); @@ -306,14 +123,20 @@ export const proposeAthleteTracks = async ( const input = video.proxyPath !== null && existsSync(video.proxyPath) ? video.proxyPath : video.path; - const result = await run( - worker.command, - [...worker.args, 'appearance', '--input', input], - { - stdin: JSON.stringify({ boxes }), - ...(options.signal === undefined ? {} : { signal: options.signal }), - }, - ); + const result = await run(worker.command, [...worker.args, 'appearance', '--input', input], { + /** + * The boxes' pixel space goes with the boxes. Tracks are in source-video + * coordinates while the file being read is the much smaller proxy, and + * letting the worker infer the space from the file it opened measured every + * crop against the wrong scale — which shipped as a confident zero matches. + */ + stdin: JSON.stringify({ + boxes, + sourceWidth: frameWidth, + sourceHeight: video.probe?.video?.height ?? 1080, + }), + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); if (result.code !== 0) { throw new ReelEelError('WORKER_CRASHED', 'The CV worker could not read this video.', { hint: result.stderr.trim().split('\n').at(-1) ?? undefined, @@ -348,79 +171,16 @@ export const proposeAthleteTracks = async ( ); } - /** - * Grow the athlete one fragment at a time, re-deriving their appearance after - * each addition. - * - * Iterative rather than a single pass because coverage compounds: the - * fragment that continues the athlete's *new* last track was not adjacent to - * anything before it was accepted. Re-averaging the signature as it goes also - * lets the reference follow a genuine change in lighting down the court, - * which a signature frozen at the first binding cannot. - */ - const threshold = options.threshold ?? COLOUR_FLOOR; - const frameWidth = video.probe?.video?.width ?? 1920; - const limit = options.limit ?? 40; - - const chosen = [...reference]; - const accepted: AthleteProposal[] = []; - const remaining = new Set(candidates); - - while (accepted.length < limit) { - const current = mergeSignatures( - chosen.map((track) => ({ - signature: signatures[track.id] ?? [], - weight: pixels[track.id] ?? 0, - })), - ); - if (current.length === 0) break; - - let best: { track: TrackSeries; colour: number; link: Link } | null = null; - for (const track of remaining) { - // One child, one place at a time — re-checked against everything accepted - // so far, not only the original binding. - if (chosen.some((known) => overlapsInTime(known, track))) { - remaining.delete(track); - continue; - } - - let link: Link | null = null; - for (const known of chosen) { - const found = linkBetween(known, track, frameWidth); - if (found !== null && (link === null || found.gapSeconds < link.gapSeconds)) link = found; - } - if (link === null) continue; - - const signature = signatures[track.id]; - if (signature === undefined || signature.length === 0) continue; - const colour = similarity(signature, current); - if (colour < threshold) continue; - - // Prefer the closest, cleanest link; colour has already done its only job. - const score = colour - link.distancePx / (frameWidth * 2); - const bestScore = - best === null ? -Infinity : best.colour - best.link.distancePx / (frameWidth * 2); - if (score > bestScore) best = { track, colour, link }; - } - - if (best === null) break; - remaining.delete(best.track); - chosen.push(best.track); - const { start, end } = spanOf(best.track); - accepted.push({ - trackId: best.track.id, - score: best.colour, - startTs: start, - endTs: end, - seconds: end - start, - samples: best.track.samples.length, - gapSeconds: best.link.gapSeconds, - distancePx: Math.round(best.link.distancePx), - }); - } - return { - proposals: accepted, + proposals: chooseAthleteTracks({ + reference, + candidates, + signatures, + pixels, + frameWidth, + threshold: options.threshold ?? COLOUR_FLOOR, + ...(options.limit === undefined ? {} : { limit: options.limit }), + }), referenceTrackIds: [...assigned], considered: candidates.length, }; diff --git a/packages/core/src/stitch.test.ts b/packages/core/src/stitch.test.ts new file mode 100644 index 0000000..68b7d7a --- /dev/null +++ b/packages/core/src/stitch.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, it } from 'vitest'; + +import { candidatesFrom, chooseAthleteTracks } from './stitch.js'; +import type { TrackSeries } from './scoring.js'; + +/** + * The decision itself, with no video and no database in the way. + * + * This module was split out of `appearance.ts` because the only way to try the + * matching against real footage had been to re-implement it in a probe — and a + * probe agreeing with a re-implementation is exactly how a version that + * returned zero matches on real footage passed every check. + */ + +const track = (id: string, from: number, to: number, x = 100): TrackSeries => ({ + id, + className: 'player', + samples: [ + { ts: from, x, y: 400, w: 40, h: 100, confidence: 0.9 }, + { ts: to, x, y: 400, w: 40, h: 100, confidence: 0.9 }, + ], +}); + +const RED = [1, 0, 0]; +const BLUE = [0, 0, 1]; +const FRAME = 1920; + +/** Everything measured, so no candidate is dropped for want of pixels. */ +const weights = (ids: string[]): Record => + Object.fromEntries(ids.map((id) => [id, 1000])); + +describe('growing an athlete out of fragments', () => { + it('chains onward, using each new fragment as the next anchor', () => { + // c only touches b, and b only touches a: a single pass would miss it. + const reference = [track('a', 10, 20)]; + const candidates = [track('c', 30.5, 40, 140), track('b', 20.5, 30, 120)]; + const accepted = chooseAthleteTracks({ + reference, + candidates, + signatures: { a: RED, b: RED, c: RED }, + pixels: weights(['a', 'b', 'c']), + frameWidth: FRAME, + }); + expect(accepted.map((p) => p.trackId)).toEqual(['b', 'c']); + }); + + it('refuses a fragment in the wrong shirt however well it lines up', () => { + const accepted = chooseAthleteTracks({ + reference: [track('a', 10, 20)], + candidates: [track('other', 20.5, 30, 110)], + signatures: { a: RED, other: BLUE }, + pixels: weights(['a', 'other']), + frameWidth: FRAME, + }); + expect(accepted).toEqual([]); + }); + + it('refuses a matching shirt that does not continue anything', () => { + // Same colour, but a hundred seconds later: that is a teammate. + const accepted = chooseAthleteTracks({ + reference: [track('a', 10, 20)], + candidates: [track('teammate', 120, 130)], + signatures: { a: RED, teammate: RED }, + pixels: weights(['a', 'teammate']), + frameWidth: FRAME, + }); + expect(accepted).toEqual([]); + }); + + it('never takes a fragment that is on screen with the athlete', () => { + const accepted = chooseAthleteTracks({ + reference: [track('a', 10, 20)], + candidates: [track('beside', 12, 18, 130)], + signatures: { a: RED, beside: RED }, + pixels: weights(['a', 'beside']), + frameWidth: FRAME, + }); + expect(accepted).toEqual([]); + }); + + it('reports the evidence behind each link, for a human to judge', () => { + const accepted = chooseAthleteTracks({ + reference: [track('a', 10, 20)], + candidates: [track('b', 20.5, 30, 160)], + signatures: { a: RED, b: RED }, + pixels: weights(['a', 'b']), + frameWidth: FRAME, + }); + expect(accepted[0]?.gapSeconds).toBeCloseTo(0.5); + expect(accepted[0]?.distancePx).toBe(60); + expect(accepted[0]?.score).toBeCloseTo(1); + }); + + it('stops at the limit it was given', () => { + const candidates = Array.from({ length: 6 }, (_u, i) => + track(`c${i}`, 20.5 + i * 10, 30 + i * 10, 100 + i * 20), + ); + const ids = ['a', ...candidates.map((c) => c.id)]; + const accepted = chooseAthleteTracks({ + reference: [track('a', 10, 20)], + candidates, + signatures: Object.fromEntries(ids.map((id) => [id, RED])), + pixels: weights(ids), + frameWidth: FRAME, + limit: 2, + }); + expect(accepted).toHaveLength(2); + }); + + it('ignores a candidate no frames could be read for', () => { + const accepted = chooseAthleteTracks({ + reference: [track('a', 10, 20)], + candidates: [track('b', 20.5, 30, 120)], + signatures: { a: RED }, + pixels: { a: 1000 }, + frameWidth: FRAME, + }); + expect(accepted).toEqual([]); + }); +}); + +describe('which tracks are worth comparing', () => { + const reference = [track('a', 10, 20)]; + const assigned = new Set(['a']); + + it('keeps the short fragments stitching depends on', () => { + /** + * The picker hides anything under 1.5s because a human cannot recognise a + * face in it. Five of the eight links that recovered a real athlete were + * shorter than that, so reusing the picker's floor here found nothing. + */ + const series = [...reference, track('brief', 20.5, 20.8, 120)]; + expect(candidatesFrom(series, reference, assigned, 0.25).map((t) => t.id)).toEqual(['brief']); + expect(candidatesFrom(series, reference, assigned, 1.5)).toEqual([]); + }); + + it('drops the ball, the rim and the officials', () => { + const series = [ + ...reference, + { id: 'ball', className: 'ball', samples: track('x', 21, 25).samples }, + { id: 'ref', className: 'referee', samples: track('x', 21, 25).samples }, + track('player', 21, 25, 120), + ]; + expect(candidatesFrom(series, reference, assigned, 0.25).map((t) => t.id)).toEqual(['player']); + }); + + it('never offers a track the athlete already has', () => { + expect(candidatesFrom(reference, reference, assigned, 0.25)).toEqual([]); + }); +}); diff --git a/packages/core/src/stitch.ts b/packages/core/src/stitch.ts new file mode 100644 index 0000000..a00b338 --- /dev/null +++ b/packages/core/src/stitch.ts @@ -0,0 +1,309 @@ +import type { TrackSeries } from './scoring.js'; + +/** + * Deciding which fragments are the same child — the whole of the judgement, and + * none of the plumbing. + * + * Separated from `appearance.ts` because that module reaches a database, a + * subprocess and a filesystem, and importing any of it drags in a native driver. + * The consequence was that the only way to try this against real footage was to + * *re-implement* it in a probe, and a probe that agrees with a re-implementation + * proves nothing about what ships. It shipped returning zero matches on the very + * game a probe had found eight in, and neither the tests nor the probe could + * have caught it. Everything here is pure, so the real code can be run against + * real data without the app around it. + */ + +export interface AthleteProposal { + trackId: string; + /** Colour-signature agreement with the athlete's known tracks, 0..1. */ + score: number; + startTs: number; + endTs: number; + seconds: number; + samples: number; + /** The link that justified it, so a person can judge the claim. */ + gapSeconds: number; + distancePx: number; +} + +/** + * Colour is a veto, never an identifier. + * + * Measured on a real game: at a 0.55 colour match, 661 of 1152 candidate tracks + * qualified — 2,306 seconds of "athlete" in a 300-second video. That is not a + * tuning failure, it is what a shirt means. Teammates wear the same one, so a + * colour signature identifies a *team*, and the children it wrongly volunteers + * are precisely the ones standing next to yours. + * + * So the identity claim rests on continuity, and colour only ever rules a link + * out. The same measurement, three ways: continuity alone recovered 120.3s + * across 56 tracks (too permissive — it links whoever is nearby); colour alone + * 2,306s; both together 51.2s across 14 tracks, up from 31.7s across 6, with + * every link under a second of gap and a few hundred pixels of travel. + */ +export const COLOUR_FLOOR = 0.7; + +/** Longest silence a link may be drawn across. */ +export const MAX_LINK_SECONDS = 2; + +/** + * How far a child may travel between two fragments, as a fraction of frame + * width per second, plus a fixed allowance for the tracker's own jitter. + * + * At four seconds and this speed the accepted links reached 894 pixels of a + * 1920-wide frame — most of the way across a court — for eight more seconds of + * coverage. The gap limit above is where that trade stops being worth taking. + */ +export const LINK_SPEED_FRACTION = 0.31; +export const LINK_SLACK_FRACTION = 0.03; + +/** Time span a track occupies. */ +export const spanOf = (series: TrackSeries): { start: number; end: number } => { + const first = series.samples[0]; + const last = series.samples[series.samples.length - 1]; + if (first === undefined || last === undefined) return { start: 0, end: 0 }; + return { start: first.ts, end: last.ts }; +}; + +/** + * Whether two tracks are ever on screen at the same moment. + * + * The hardest constraint available and the cheapest: one child cannot be in two + * places at once, so a candidate that coexists with a track already known to be + * the athlete is definitively somebody else — however similar their shirt. + * Teammates wear the same colour, so without this the strongest false matches + * would be exactly the children standing next to them. + */ +export const overlapsInTime = (a: TrackSeries, b: TrackSeries): boolean => { + const first = spanOf(a); + const second = spanOf(b); + return first.start <= second.end && second.start <= first.end; +}; + +/** + * A handful of boxes spread across a track's life, rather than all of them. + * + * A signature wants variety — different moments, poses and lighting — not + * volume. Sampling every half-second and capping keeps a thirty-second track + * from drowning out a three-second one in the reference average. + */ +export const sampleBoxes = ( + series: TrackSeries, + everySeconds = 0.5, + cap = 12, +): { ts: number; x: number; y: number; w: number; h: number }[] => { + const picked: { ts: number; x: number; y: number; w: number; h: number }[] = []; + let nextTs = Number.NEGATIVE_INFINITY; + for (const sample of series.samples) { + if (sample.ts < nextTs) continue; + picked.push({ ts: sample.ts, x: sample.x, y: sample.y, w: sample.w, h: sample.h }); + nextTs = sample.ts + everySeconds; + } + if (picked.length <= cap) return picked; + + // Thin evenly rather than truncating, so the tail of a long track is still + // represented — a child who changes ends of the court is still that child. + const step = picked.length / cap; + return Array.from({ length: cap }, (_unused, i) => picked[Math.floor(i * step)]).filter( + (box): box is { ts: number; x: number; y: number; w: number; h: number } => box !== undefined, + ); +}; + +/** Weighted mean of several signatures, renormalized. */ +export const mergeSignatures = (parts: { signature: number[]; weight: number }[]): number[] => { + const usable = parts.filter((part) => part.weight > 0 && part.signature.length > 0); + const first = usable[0]; + if (first === undefined) return []; + + const totals = new Array(first.signature.length).fill(0); + let weightSum = 0; + for (const part of usable) { + weightSum += part.weight; + for (let i = 0; i < totals.length; i += 1) { + totals[i] = (totals[i] ?? 0) + (part.signature[i] ?? 0) * part.weight; + } + } + if (weightSum <= 0) return []; + const scaled = totals.map((value) => value / weightSum); + const sum = scaled.reduce((a, b) => a + b, 0); + return sum > 0 ? scaled.map((value) => value / sum) : scaled; +}; + +const centreOf = (sample: { + x: number; + y: number; + w: number; + h: number; +}): { x: number; y: number } => ({ + x: sample.x + sample.w / 2, + y: sample.y + sample.h / 2, +}); + +export interface Link { + gapSeconds: number; + distancePx: number; +} + +/** + * Whether a candidate plausibly continues a known track — picking up where it + * left off, or leading into where it began. + * + * This is the part that actually claims identity, so it is deliberately mean: + * a short silence, and a distance a child could really have covered in it. Both + * directions, because a fragment can extend an athlete backwards just as + * usefully as forwards. + */ +export const linkBetween = ( + known: TrackSeries, + candidate: TrackSeries, + frameWidth: number, + maxSeconds = MAX_LINK_SECONDS, +): Link | null => { + const knownFirst = known.samples[0]; + const knownLast = known.samples[known.samples.length - 1]; + const otherFirst = candidate.samples[0]; + const otherLast = candidate.samples[candidate.samples.length - 1]; + if ( + knownFirst === undefined || + knownLast === undefined || + otherFirst === undefined || + otherLast === undefined + ) { + return null; + } + + const reach = (gap: number): number => + frameWidth * LINK_SPEED_FRACTION * gap + frameWidth * LINK_SLACK_FRACTION; + + const forward = otherFirst.ts - knownLast.ts; + if (forward > 0 && forward <= maxSeconds) { + const distance = Math.hypot( + centreOf(knownLast).x - centreOf(otherFirst).x, + centreOf(knownLast).y - centreOf(otherFirst).y, + ); + if (distance <= reach(forward)) return { gapSeconds: forward, distancePx: distance }; + } + + const backward = knownFirst.ts - otherLast.ts; + if (backward > 0 && backward <= maxSeconds) { + const distance = Math.hypot( + centreOf(knownFirst).x - centreOf(otherLast).x, + centreOf(knownFirst).y - centreOf(otherLast).y, + ); + if (distance <= reach(backward)) return { gapSeconds: backward, distancePx: distance }; + } + + return null; +}; + +/** Histogram intersection, mirroring the worker's own comparison. */ +export const similarity = (a: number[], b: number[]): number => { + const length = Math.min(a.length, b.length); + let shared = 0; + for (let i = 0; i < length; i += 1) shared += Math.min(a[i] ?? 0, b[i] ?? 0); + return shared; +}; + +export interface ChooseOptions { + reference: TrackSeries[]; + candidates: TrackSeries[]; + signatures: Record; + pixels: Record; + frameWidth: number; + threshold?: number; + limit?: number; +} + +/** + * Grow the athlete one fragment at a time, re-deriving their appearance after + * each addition. + * + * Iterative rather than a single pass because coverage compounds: the fragment + * that continues the athlete's *new* last track was not adjacent to anything + * before it was accepted. Re-averaging the signature as it goes also lets the + * reference follow a genuine change in lighting down the court, which a + * signature frozen at the first binding cannot. + */ +export const chooseAthleteTracks = (options: ChooseOptions): AthleteProposal[] => { + const { reference, signatures, pixels, frameWidth } = options; + const threshold = options.threshold ?? COLOUR_FLOOR; + const limit = options.limit ?? 40; + + const chosen = [...reference]; + const accepted: AthleteProposal[] = []; + const remaining = new Set(options.candidates); + + while (accepted.length < limit) { + const current = mergeSignatures( + chosen.map((track) => ({ + signature: signatures[track.id] ?? [], + weight: pixels[track.id] ?? 0, + })), + ); + if (current.length === 0) break; + + let best: { track: TrackSeries; colour: number; link: Link } | null = null; + for (const track of remaining) { + // One child, one place at a time — re-checked against everything accepted + // so far, not only the original binding. + if (chosen.some((known) => overlapsInTime(known, track))) { + remaining.delete(track); + continue; + } + + let link: Link | null = null; + for (const known of chosen) { + const found = linkBetween(known, track, frameWidth); + if (found !== null && (link === null || found.gapSeconds < link.gapSeconds)) link = found; + } + if (link === null) continue; + + const signature = signatures[track.id]; + if (signature === undefined || signature.length === 0) continue; + const colour = similarity(signature, current); + if (colour < threshold) continue; + + // Prefer the closest, cleanest link; colour has already done its only job. + const score = colour - link.distancePx / (frameWidth * 2); + const bestScore = + best === null ? -Infinity : best.colour - best.link.distancePx / (frameWidth * 2); + if (score > bestScore) best = { track, colour, link }; + } + + if (best === null) break; + remaining.delete(best.track); + chosen.push(best.track); + const { start, end } = spanOf(best.track); + accepted.push({ + trackId: best.track.id, + score: best.colour, + startTs: start, + endTs: end, + seconds: end - start, + samples: best.track.samples.length, + gapSeconds: best.link.gapSeconds, + distancePx: Math.round(best.link.distancePx), + }); + } + + return accepted; +}; + +/** + * Which tracks are worth comparing at all: people, long enough to measure, and + * never on screen at the same time as the athlete already is. + */ +export const candidatesFrom = ( + series: TrackSeries[], + reference: TrackSeries[], + assigned: ReadonlySet, + minSeconds: number, +): TrackSeries[] => + series.filter((track) => { + if (assigned.has(track.id)) return false; + if (track.className !== 'player') return false; + const { start, end } = spanOf(track); + if (end - start < minSeconds) return false; + return !reference.some((known) => overlapsInTime(known, track)); + }); diff --git a/scripts/stitch-probe.mjs b/scripts/stitch-probe.mjs index ac30a39..198a83b 100644 --- a/scripts/stitch-probe.mjs +++ b/scripts/stitch-probe.mjs @@ -1,92 +1,79 @@ /** - * Does continuity-plus-colour actually recover an athlete, on real footage? + * Runs the shipped matching against a real project, read-only. * - * Colour alone cannot: a shirt identifies a team, and on this game 661 of 1152 - * candidate tracks cleared a 0.55 colour match — 2306 seconds of "athlete" in a - * 300-second video. Teammates are the false positives, and they are the ones a - * parent would most object to. - * - * So colour becomes a veto, not an identifier, and the identity claim rests on - * continuity: a track that begins where and when another ended is the same - * person. This measures how much of the game that recovers before any of it is - * allowed near the product. + * This imports the same `chooseAthleteTracks` and `computeSignatures` the app + * calls — not a re-implementation. An earlier version of this probe agreed with + * a re-implementation and the product still returned zero matches, because the + * bug lived in the plumbing between the two halves and nothing exercised it. + * `stitch.js` touches no database precisely so this can import it. * * node --experimental-sqlite scripts/stitch-probe.mjs