fix: four reasons a five-minute game suggested nothing

packages/core/src/analyze.ts

@@ -9,7 +9,7 @@ import { run } from './ffmpeg.js';
 import { createJob, logJob, updateJob } from './jobs.js';
 import { getFocalAthlete } from './athletes.js';
 import { generateMoments } from './moments.js';
-import { generateProxy, generateThumbnails } from './media.js';
+import { generateProxy, generateThumbnails, PROXY_HEIGHT } from './media.js';
 import { readManifest } from './projects.js';
 import { clearTracks, createTrack, rebindAthletes, snapshotAthleteBindings } from './tracks.js';
 import type { Job, Preset } from './types.js';
@@ -51,6 +51,34 @@ export const PRESET_SETTINGS: Record<Exclude<Preset, 'custom'>, PresetSettings>
   thorough: { frameStride: 2, inferenceSize: 1280, minConfidence: 0.25, useProxy: false, tileGrid: 2 },
 };
 
+/**
+ * Which file the detector should actually read.
+ *
+ * The proxy is only worth detecting from when it is at least as tall as the
+ * frame the worker will hand the model. `useProxy` was obeyed unconditionally,
+ * and the 540p editing proxy is shorter than every inference size above `fast`.
+ * The worker decodes to its own input size regardless, so a 540p proxy was
+ * *upscaled* — the same inference cost for strictly less picture. Measured on a
+ * 1080p game, the identical preset found 145,975 detections across 3,948 tracks
+ * from the source against 67,985 across 1,415 from the proxy; the ball, a
+ * handful of pixels to begin with, is what goes first. The saving was only ever
+ * decode time.
+ */
+export const detectionInputFor = (
+  settings: PresetSettings,
+  video: { path: string; proxyPath: string | null },
+  proxyExists: boolean,
+): { input: string; usedProxy: boolean; proxyTooSmall: boolean } => {
+  const proxyTooSmall = settings.useProxy && settings.inferenceSize > PROXY_HEIGHT;
+  const usedProxy =
+    settings.useProxy && !proxyTooSmall && video.proxyPath !== null && proxyExists;
+  return {
+    input: usedProxy && video.proxyPath !== null ? video.proxyPath : video.path,
+    usedProxy,
+    proxyTooSmall,
+  };
+};
+
 export const settingsForPreset = (preset: Preset): PresetSettings => {
   /**
    * The web form and the API both cast whatever string arrives into `Preset`
@@ -289,10 +317,20 @@ export const analyzeProject = async (
         const share = (index + 1) / refreshed.length;
         await stage('detection', 0.2 + 0.5 * share, path.basename(video.path));
 
-        const input =
-          settings.useProxy && video.proxyPath !== null && existsSync(video.proxyPath)
-            ? video.proxyPath
-            : video.path;
+        const choice = detectionInputFor(
+          settings,
+          video,
+          video.proxyPath !== null && existsSync(video.proxyPath),
+        );
+        const input = choice.input;
+        if (choice.proxyTooSmall) {
+          await logJob(
+            root,
+            job.id,
+            `detecting from the original: the ${PROXY_HEIGHT}p proxy is smaller than the ` +
+              `${settings.inferenceSize}px this preset detects at, so it would only lose detail.`,
+          );
+        }
 
         /**
          * Detection is the long pole — minutes of CPU inference on a full game
@@ -558,7 +596,13 @@ export const analyzeProject = async (
           root,
           job.id,
           `what was seen: ${classes}; longest track ${diagnosis.longestTrackSeconds.toFixed(1)}s; ` +
-            `athlete identified: ${diagnosis.focalBound ? 'yes' : 'no'}`,
+            `athlete identified: ${
+              diagnosis.focalBound
+                ? `yes, on screen ${diagnosis.focalSeconds.toFixed(1)}s of ` +
+                  `${diagnosis.durationSeconds.toFixed(0)}s across ` +
+                  `${diagnosis.focalTrackCount} track(s)`
+                : 'no'
+            }`,
           'warn',
         );
         await logJob(
@@ -571,6 +615,32 @@ export const analyzeProject = async (
               : ` (no data for: ${diagnosis.unmeasurable.join(', ')})`),
           'warn',
         );
+        /**
+         * The binding is thin: said whether or not the threshold was reachable
+         * in principle.
+         *
+         * Reachability is computed over the whole footage, so a rim visible for
+         * half a minute can hold the ceiling above the threshold while the
+         * athlete every focal signal depends on is present for a fraction of a
+         * second. Production hit exactly that — a binding to a ten-frame
+         * fragment of a five-minute game — and every line here read plausibly:
+         * tracks found, athlete identified, threshold reachable, footage too
+         * dull. The one number that showed the problem was not among them.
+         */
+        const coverage =
+          diagnosis.durationSeconds > 0 ? diagnosis.focalSeconds / diagnosis.durationSeconds : 0;
+        if (diagnosis.focalBound && coverage < 0.05) {
+          await logJob(
+            root,
+            job.id,
+            `your athlete is only on screen for ${diagnosis.focalSeconds.toFixed(1)}s of ` +
+              `${diagnosis.durationSeconds.toFixed(0)}s (${(coverage * 100).toFixed(1)}%), so every ` +
+              'signal that follows them is dark for the rest of the game. That is almost certainly ' +
+              'the reason, not the footage. Open "Identify your athlete" and pick them again — ' +
+              'choose every fragment of them you can see, not just one.',
+            'warn',
+          );
+        }
         if (!diagnosis.reachable) {
           // The important case, and the one the old message got wrong.
           const because = !diagnosis.focalBound

packages/core/src/media.ts

@@ -77,8 +77,16 @@ export const generateThumbnails = async (
   return { dir, files: readdirSync(dir).sort() };
 };
 
+/**
+ * Proxy height in pixels. 540 keeps scrubbing smooth on a laptop.
+ *
+ * Exported because analysis has to be able to ask whether the proxy is big
+ * enough to detect from, rather than assuming it always is.
+ */
+export const PROXY_HEIGHT = 540;
+
 export interface ProxyOptions {
-  /** Proxy height in pixels. 540 keeps scrubbing smooth on a laptop. */
+  /** Proxy height in pixels. Defaults to {@link PROXY_HEIGHT}. */
   height?: number;
   crf?: number;
   signal?: AbortSignal;
@@ -100,7 +108,7 @@ export const generateProxy = async (
   }
 
   const ffmpeg = requireBinary('ffmpeg');
-  const height = options.height ?? 540;
+  const height = options.height ?? PROXY_HEIGHT;
   const dir = projectDir(root, 'proxies');
   mkdirSync(dir, { recursive: true });
   const output = path.join(dir, `${video.id}_${height}p.mp4`);

packages/core/src/rebindgrow.test.ts

@@ -0,0 +1,155 @@
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+/**
+ * A re-bind has to be able to *grow* an athlete's coverage, not merely survive.
+ *
+ * Matching took the single best new track per old track, so N fragments in gave
+ * at most N fragments out — however the new run happened to cut the same child
+ * up. In production a binding to one ten-frame fragment came back from
+ * re-detection as one ten-frame fragment, twice in a row, over runs that
+ * produced 3,948 and then 1,415 tracks. The athlete every focal signal depends
+ * on was therefore present for 0.3s of a 300s game, and the run suggested
+ * nothing while reporting "athlete identified: yes".
+ */
+
+let home: string;
+
+beforeAll(() => {
+  home = mkdtempSync(path.join(tmpdir(), 'reeleel-rebindgrow-'));
+  process.env['REELEEL_HOME'] = home;
+});
+
+afterAll(async () => {
+  const { resetDbCache } = await import('./db.js');
+  resetDbCache();
+  rmSync(home, { recursive: true, force: true });
+  delete process.env['REELEEL_HOME'];
+});
+
+const project = async (name: string): Promise<string> => {
+  const { createProject } = await import('./projects.js');
+  const created = await createProject({
+    name,
+    path: path.join(home, 'projects', `${name}-${process.hrtime.bigint()}`),
+  });
+  return created.path ?? created.root;
+};
+
+const video = async (root: string, id: string): Promise<void> => {
+  const { execute, projectDb } = await import('./db.js');
+  const db = await projectDb(root);
+  const now = new Date().toISOString();
+  await execute(
+    db,
+    'INSERT INTO source_videos (id, project_id, path, created_at, updated_at) VALUES (?, ?, ?, ?, ?)',
+    [id, 'prj_test', `/tmp/${id}.mp4`, now, now],
+  );
+};
+
+/** Dense samples along a straight walk, the way the tracker emits them. */
+const walk = (from: number, to: number, offset = 0) => {
+  const out = [];
+  for (let i = 0; i <= Math.round((to - from) * 4); i += 1) {
+    const ts = from + i / 4;
+    out.push({ ts, frame: Math.round(ts * 30), x: 100 + ts * 10 + offset, y: 300, w: 40, h: 100, confidence: 0.9 });
+  }
+  return out;
+};
+
+describe('re-identifying an athlete across a re-detection', () => {
+  it('picks up every new fragment of the athlete, not one per old fragment', async () => {
+    const root = await project('grow');
+    await video(root, 'vid_a');
+    const { createTrack, clearTracks, snapshotAthleteBindings, rebindAthletes, tracksForAthlete } =
+      await import('./tracks.js');
+    const { addAthlete, updateAthlete } = await import('./athletes.js');
+
+    // The old run saw the child as one long track.
+    const old = await createTrack(root, {
+      videoId: 'vid_a',
+      className: 'player',
+      confidence: 0.9,
+      samples: walk(0, 30),
+    });
+    const athlete = await addAthlete(root, { name: 'Kid' });
+    await updateAthlete(root, athlete.id, { focalTrackId: old.id, focal: true });
+
+    const remembered = await snapshotAthleteBindings(root, 'vid_a');
+    await clearTracks(root, 'vid_a');
+
+    // The new run cuts the same child into three consecutive pieces, and also
+    // sees a different child on the far side of the court throughout.
+    for (const [from, to] of [
+      [0, 9],
+      [10, 19],
+      [20, 30],
+    ] as const) {
+      await createTrack(root, {
+        videoId: 'vid_a',
+        className: 'player',
+        confidence: 0.9,
+        samples: walk(from, to, 2),
+      });
+    }
+    await createTrack(root, {
+      videoId: 'vid_a',
+      className: 'player',
+      confidence: 0.9,
+      samples: walk(0, 30, 900),
+    });
+
+    const restored = await rebindAthletes(root, 'vid_a', remembered);
+    expect(restored).toHaveLength(1);
+
+    /**
+     * All three pieces, which is the whole point. One-best-per-old-track would
+     * return exactly one here and silently drop two thirds of the athlete.
+     */
+    const assigned = await tracksForAthlete(root, athlete.id);
+    expect(assigned).toHaveLength(3);
+  });
+
+  it('still refuses a child who merely walked through the same space later', async () => {
+    const root = await project('stranger');
+    await video(root, 'vid_a');
+    const { createTrack, clearTracks, snapshotAthleteBindings, rebindAthletes, tracksForAthlete } =
+      await import('./tracks.js');
+    const { addAthlete, updateAthlete } = await import('./athletes.js');
+
+    const old = await createTrack(root, {
+      videoId: 'vid_a',
+      className: 'player',
+      confidence: 0.9,
+      samples: walk(0, 30),
+    });
+    const athlete = await addAthlete(root, { name: 'Kid' });
+    await updateAthlete(root, athlete.id, { focalTrackId: old.id, focal: true });
+
+    const remembered = await snapshotAthleteBindings(root, 'vid_a');
+    await clearTracks(root, 'vid_a');
+
+    // Same path, a different half of the game: never on screen together, so not
+    // the same person as far as anything here can tell.
+    await createTrack(root, {
+      videoId: 'vid_a',
+      className: 'player',
+      confidence: 0.9,
+      samples: walk(200, 230),
+    });
+    // And the real athlete.
+    await createTrack(root, {
+      videoId: 'vid_a',
+      className: 'player',
+      confidence: 0.9,
+      samples: walk(0, 30, 2),
+    });
+
+    await rebindAthletes(root, 'vid_a', remembered);
+    const assigned = await tracksForAthlete(root, athlete.id);
+    expect(assigned).toHaveLength(1);
+  });
+});