fix: one click should mark your child for the game, and the ball should be findable

apps/cv-worker/src/ballfloor.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, it } from 'vitest';
 
-import { parseClassConfidence } from './index.js';
+import { parseClassBuffer, parseClassConfidence } from './index.js';
 
 /**
  * A basketball is a handful of pixels and the model is right to be unsure about
@@ -58,3 +58,27 @@ describe('per-class detection floors', () => {
     expect(parseClassConfidence('ball=-0.2')).toEqual({});
   });
 });
+
+/**
+ * Buffers are a distance, not a probability, so they share the parser but not
+ * its ceiling — the useful values for a basketball are greater than 1, which is
+ * the whole reason the plain overlap test fails on it.
+ */
+describe('per-class association buffers', () => {
+  it('accepts the values a ball actually needs', () => {
+    expect(parseClassBuffer('ball=1.5')).toEqual({ ball: 1.5 });
+    expect(parseClassBuffer('ball=2.5,puck=1.5')).toEqual({ ball: 2.5, puck: 1.5 });
+  });
+
+  it('still refuses nonsense', () => {
+    expect(parseClassBuffer('ball=0')).toEqual({});
+    expect(parseClassBuffer('ball=-1')).toEqual({});
+    expect(parseClassBuffer('ball=abc')).toEqual({});
+    // Far enough that every ball on court would match every other one.
+    expect(parseClassBuffer('ball=99')).toEqual({});
+  });
+
+  it('is absent rather than empty when nothing is asked for', () => {
+    expect(parseClassBuffer(undefined)).toEqual({});
+  });
+});

apps/cv-worker/src/ballrecall.test.ts

@@ -0,0 +1,161 @@
+import { describe, expect, it } from 'vitest';
+
+import type { Detection } from './geometry.js';
+import { ByteTracker, inflate } from './tracker.js';
+
+/**
+ * Why the ball is hardly ever identified.
+ *
+ * Per-class confidence floors let a faint ball through the filter, and ball
+ * *positions* rose 69% — but the number of ball tracks barely moved, which is
+ * the tell. Only a confident detection may open a track; a faint one can extend
+ * a track that already exists. So every extra ball detection attached itself to
+ * the handful of tracks lucky enough to have started at 0.4, and the rest were
+ * discarded for having nothing to attach to.
+ *
+ * The second half is geometry. Overlap is a poor question to ask about a thrown
+ * ball: it is a small box that clears several of its own widths between sampled
+ * frames, and boxes that do not touch have an IoU of exactly zero however
+ * close they are.
+ *
+ * Measured against the deployed worker, same 20s of real game, same flags:
+ *
+ *                        ball        hoop      player      referee    time
+ *   deployed today     21 / 382    7 / 387   297 / 8604   39 / 803    89s
+ *   + birth at floor   71 / 831   15 / 425   297 / 8604   39 / 803    93s
+ *   + buffered IoU     66 / 891   15 / 425   297 / 8604   39 / 803    88s
+ *
+ * Ball positions rise 133% for no measurable runtime. The rim doubles too — its
+ * 0.15 floor was under the same 0.4 birth bar. People are identical to the
+ * digit, which is the invariant the tests below exist to hold: loosening the
+ * ball must never loosen anybody.
+ */
+
+const det = (x: number, y: number, w: number, h: number, score: number, classId = 0): Detection => ({
+  x,
+  y,
+  w,
+  h,
+  score,
+  classId,
+});
+
+const names = { 0: 'player', 32: 'ball' };
+
+describe('starting a track at the class its own floor', () => {
+  it('opens a ball track from detections no player detection could open', () => {
+    const tracker = new ByteTracker({
+      minLength: 1,
+      highThreshold: 0.5,
+      lowThreshold: 0.05,
+      classHighThreshold: { ball: 0.08 },
+    });
+    // A ball seen faintly and steadily — never once at the global bar of 0.5.
+    for (let f = 0; f < 6; f += 1) {
+      tracker.update([det(100 + f * 3, 200, 20, 20, 0.2, 32)], names, f, f / 30);
+    }
+
+    const tracks = tracker.results();
+    expect(tracks).toHaveLength(1);
+    expect(tracks[0]?.className).toBe('ball');
+    expect(tracks[0]?.points).toHaveLength(6);
+  });
+
+  it('still refuses to open a player track from the same weak evidence', () => {
+    const tracker = new ByteTracker({
+      minLength: 1,
+      highThreshold: 0.5,
+      lowThreshold: 0.05,
+      classHighThreshold: { ball: 0.08 },
+    });
+    for (let f = 0; f < 6; f += 1) {
+      tracker.update([det(100 + f * 3, 200, 40, 90, 0.2, 0)], names, f, f / 30);
+    }
+    // The floor is the ball's alone: loosening it must not loosen people.
+    expect(tracker.results()).toHaveLength(0);
+  });
+
+  it('leaves a class with no override on the global bar', () => {
+    const tracker = new ByteTracker({ minLength: 1, highThreshold: 0.5, lowThreshold: 0.05 });
+    for (let f = 0; f < 6; f += 1) {
+      tracker.update([det(100 + f * 3, 200, 20, 20, 0.2, 32)], names, f, f / 30);
+    }
+    expect(tracker.results()).toHaveLength(0);
+  });
+});
+
+describe('buffered overlap for something small and fast', () => {
+  it('grows a box by a fraction of its own size, and leaves a zero buffer alone', () => {
+    const original = { x: 100, y: 100, w: 20, h: 20 };
+    expect(inflate(original, 0)).toEqual(original);
+    expect(inflate(original, 0.5)).toEqual({ x: 90, y: 90, w: 40, h: 40 });
+  });
+
+  it('follows a ball that clears its own width between frames as one flight', () => {
+    const tracker = new ByteTracker({
+      minLength: 1,
+      highThreshold: 0.5,
+      lowThreshold: 0.05,
+      classHighThreshold: { ball: 0.08 },
+      classBuffer: { ball: 1.5 },
+    });
+    // 20px wide, moving 30px a frame: consecutive boxes never touch, so plain
+    // IoU is zero at every step.
+    for (let f = 0; f < 8; f += 1) {
+      tracker.update([det(100 + f * 30, 200, 20, 20, 0.3, 32)], names, f, f / 30);
+    }
+
+    const tracks = tracker.results();
+    expect(tracks).toHaveLength(1);
+    expect(tracks[0]?.points).toHaveLength(8);
+  });
+
+  it('is what makes the difference — the same flight without a buffer shatters', () => {
+    const tracker = new ByteTracker({
+      minLength: 1,
+      highThreshold: 0.5,
+      lowThreshold: 0.05,
+      classHighThreshold: { ball: 0.08 },
+    });
+    for (let f = 0; f < 8; f += 1) {
+      tracker.update([det(100 + f * 30, 200, 20, 20, 0.3, 32)], names, f, f / 30);
+    }
+
+    // Every frame starts a new track: this is the scatter seen in production.
+    expect(tracker.results().length).toBeGreaterThan(1);
+  });
+
+  it('does not let a buffered ball steal the ball beside it', () => {
+    const tracker = new ByteTracker({
+      minLength: 1,
+      highThreshold: 0.5,
+      lowThreshold: 0.05,
+      classHighThreshold: { ball: 0.08 },
+      classBuffer: { ball: 1.5 },
+    });
+    // Two balls, far apart, moving in opposite directions. Buffering must widen
+    // the net, not merge distinct objects.
+    for (let f = 0; f < 8; f += 1) {
+      tracker.update(
+        [det(100 + f * 20, 200, 20, 20, 0.3, 32), det(900 - f * 20, 600, 20, 20, 0.3, 32)],
+        names,
+        f,
+        f / 30,
+      );
+    }
+    expect(tracker.results()).toHaveLength(2);
+  });
+
+  it('leaves people associating on exactly the geometry they always did', () => {
+    const withBuffer = new ByteTracker({ minLength: 1, classBuffer: { ball: 1.5 } });
+    const without = new ByteTracker({ minLength: 1 });
+    for (let f = 0; f < 10; f += 1) {
+      const frame = [det(100 + f * 5, 200, 40, 90, 0.9), det(300 - f * 5, 210, 40, 90, 0.85)];
+      withBuffer.update(frame, names, f, f / 30);
+      without.update(frame, names, f, f / 30);
+    }
+    expect(withBuffer.results().map((t) => t.points.length)).toEqual(
+      without.results().map((t) => t.points.length),
+    );
+  });
+});

apps/cv-worker/src/index.ts

@@ -43,25 +43,41 @@ const number = (value: string | undefined, fallback: number): number => {
 };
 
 /**
- * `ball=0.08,hoop=0.15` — per-class detection floors.
+ * `ball=0.08,hoop=0.15` — a per-class number, whatever the number means.
  *
  * Malformed pairs are dropped rather than throwing: a typo in a preset should
  * cost the ball some recall, not take a twenty-minute detection pass down with
  * it.
  */
-export const parseClassConfidence = (value: string | undefined): Record<string, number> => {
+const parseClassNumbers = (value: string | undefined, max: number): Record<string, number> => {
   if (value === undefined || value.trim().length === 0) return {};
   const out: Record<string, number> = {};
   for (const pair of value.split(',')) {
     const [name, raw] = pair.split('=');
     const parsed = Number(raw);
     if (name === undefined || name.trim().length === 0) continue;
-    if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 1) continue;
+    if (!Number.isFinite(parsed) || parsed <= 0 || parsed > max) continue;
     out[name.trim()] = parsed;
   }
   return out;
 };
 
+/** Per-class detection floors. A confidence cannot exceed 1. */
+export const parseClassConfidence = (value: string | undefined): Record<string, number> =>
+  parseClassNumbers(value, 1);
+
+/**
+ * Per-class association buffers, as a fraction of the box's own size.
+ *
+ * Exposed as a flag for the same reason the floors are: the value that belongs
+ * here is a measurement, not an opinion, and a constant compiled into the
+ * pipeline cannot be swept against real footage without rebuilding it. Allowed
+ * well above 1 — a basketball travels several of its own widths between sampled
+ * frames, which is the entire problem being solved.
+ */
+export const parseClassBuffer = (value: string | undefined): Record<string, number> =>
+  parseClassNumbers(value, 8);
+
 /**
  * How many threads onnxruntime should use for one inference.
  *
@@ -187,6 +203,9 @@ const detectAndTrack = async (flags: Record<string, string>): Promise<void> => {
       tileGrid: number(flags['tile-grid'], 1),
       minConfidence: number(flags['min-confidence'], 0.3),
       classConfidence: parseClassConfidence(flags['class-confidence']),
+      ...(flags['class-buffer'] === undefined
+        ? {}
+        : { classBuffer: parseClassBuffer(flags['class-buffer']) }),
       iouThreshold: number(flags['iou'], 0.45),
       sourceWidth: width,
       sourceHeight: height,

apps/cv-worker/src/pipeline.ts

@@ -51,6 +51,8 @@ export interface PipelineOptions {
    * detections are the same ball seen more often rather than new phantoms.
    */
   classConfidence?: Record<string, number>;
+  /** Per-class association buffer; omitted means the built-in table. */
+  classBuffer?: Record<string, number>;
   iouThreshold: number;
   sourceWidth: number;
   sourceHeight: number;
@@ -72,6 +74,33 @@ export interface PipelineResult {
   unsupportedClasses: string[];
 }
 
+/**
+ * How far each class may move, relative to its own size, and still be
+ * recognised as the same object on the next sampled frame.
+ *
+ * People are left at 0: a player cannot cross their own width in a frame, so
+ * plain overlap already answers the question and buffering would only invite
+ * one player's box to capture the player beside them. A ball is the opposite —
+ * small, and fast enough to clear several of its own widths — which is why it
+ * arrives as a scatter of short tracks rather than a flight.
+ *
+ * Swept over 20s of a real game, at the shipped 2x2 grid and ball floor:
+ *
+ *   buffer   ball tracks / positions   positions per track
+ *   0             71 / 831                   11.7
+ *   0.5           67 / 869                   13.0
+ *   1.5           66 / 891                   13.5
+ *   3.0           59 / 900                   15.3
+ *
+ * Tracks fall while positions rise, which is fragments being joined into
+ * flights rather than new detections appearing. 3.0 joins more, but buys almost
+ * no extra coverage for it (+1% positions over 1.5) while roughly doubling the
+ * radius in which a stray ball-shaped blob can be mistaken for the ball — and a
+ * track that teleports is worse than two that stop. Player, referee and hoop
+ * counts were identical to the digit at every value.
+ */
+const SMALL_FAST: Record<string, number> = { ball: 1.5, puck: 1.5 };
+
 export const createSession = async (
   modelPath: string,
   threads: number,
@@ -203,6 +232,14 @@ export const runPipeline = async (options: PipelineOptions): Promise<PipelineRes
     // this lets through, so the tracker's floor follows the lowest class floor.
     lowThreshold: decodeFloor,
     iouThreshold: options.iouThreshold,
+    /**
+     * A class that was given its own confidence floor gets to start tracks at
+     * that floor too. Without this the floor only ever fed the rescue pass,
+     * which cannot open a track — so a ball that is never seen at 0.4 stays
+     * invisible no matter how far the floor drops.
+     */
+    classHighThreshold: options.classConfidence ?? {},
+    classBuffer: options.classBuffer ?? SMALL_FAST,
   });
 
   let framesProcessed = 0;