feat: judge a basketball by what a basketball looks like

apps/cv-worker/src/ballfloor.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, it } from 'vitest';
+
+import { parseClassConfidence } from './index.js';
+
+/**
+ * A basketball is a handful of pixels and the model is right to be unsure about
+ * it; a player fills a fifth of the frame and a 0.08 "person" is junk. Holding
+ * both to one number meant the ball was judged by the standard a person needs.
+ *
+ * Measured over 20s of a real game at the shipped 2x2 grid:
+ *
+ *   floor   ball tracks / positions   rim positions   player tracks
+ *   0.25         15 / 173                  316             279
+ *   0.18         16 / 206                  351             294
+ *   0.12         17 / 253                  379             285
+ *   0.08         17 / 293                  403             313
+ *   0.05         16 / 330                  427             294
+ *
+ * Track count stays flat while positions nearly double, which is the shape of
+ * better recall rather than new phantoms — a noisier detector would invent
+ * short tracks, not lengthen the ones already there.
+ */
+
+describe('per-class detection floors', () => {
+  it('reads the pairs a preset sends', () => {
+    expect(parseClassConfidence('ball=0.08,hoop=0.15')).toEqual({ ball: 0.08, hoop: 0.15 });
+  });
+
+  it('tolerates whitespace around names and values', () => {
+    expect(parseClassConfidence(' ball = 0.08 ')).toEqual({ ball: 0.08 });
+    expect(parseClassConfidence('ball=0.08, hoop=0.15')).toEqual({ ball: 0.08, hoop: 0.15 });
+  });
+
+  it('is absent rather than empty when nothing is asked for', () => {
+    expect(parseClassConfidence(undefined)).toEqual({});
+    expect(parseClassConfidence('')).toEqual({});
+  });
+
+  /**
+   * A typo in a preset should cost the ball some recall, not take a
+   * twenty-minute detection pass down with it.
+   */
+  it('drops malformed pairs instead of throwing', () => {
+    expect(parseClassConfidence('ball')).toEqual({});
+    expect(parseClassConfidence('ball=')).toEqual({});
+    expect(parseClassConfidence('ball=abc')).toEqual({});
+    expect(parseClassConfidence('=0.5')).toEqual({});
+    expect(parseClassConfidence('ball=0.08,broken,hoop=0.15')).toEqual({
+      ball: 0.08,
+      hoop: 0.15,
+    });
+  });
+
+  it('refuses values outside a probability', () => {
+    // A floor above 1 detects nothing; a floor of 0 or below detects everything.
+    expect(parseClassConfidence('ball=1.5')).toEqual({});
+    expect(parseClassConfidence('ball=0')).toEqual({});
+    expect(parseClassConfidence('ball=-0.2')).toEqual({});
+  });
+});

apps/cv-worker/src/index.ts

@@ -42,6 +42,26 @@ const number = (value: string | undefined, fallback: number): number => {
   return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
 };
 
+/**
+ * `ball=0.08,hoop=0.15` — per-class detection floors.
+ *
+ * Malformed pairs are dropped rather than throwing: a typo in a preset should
+ * cost the ball some recall, not take a twenty-minute detection pass down with
+ * it.
+ */
+export const parseClassConfidence = (value: string | undefined): Record<string, number> => {
+  if (value === undefined || value.trim().length === 0) return {};
+  const out: Record<string, number> = {};
+  for (const pair of value.split(',')) {
+    const [name, raw] = pair.split('=');
+    const parsed = Number(raw);
+    if (name === undefined || name.trim().length === 0) continue;
+    if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 1) continue;
+    out[name.trim()] = parsed;
+  }
+  return out;
+};
+
 /**
  * How many threads onnxruntime should use for one inference.
  *
@@ -166,6 +186,7 @@ const detectAndTrack = async (flags: Record<string, string>): Promise<void> => {
       // Off unless asked for: it costs grid^2 + 1 inferences per frame.
       tileGrid: number(flags['tile-grid'], 1),
       minConfidence: number(flags['min-confidence'], 0.3),
+      classConfidence: parseClassConfidence(flags['class-confidence']),
       iouThreshold: number(flags['iou'], 0.45),
       sourceWidth: width,
       sourceHeight: height,

apps/cv-worker/src/pipeline.ts

@@ -39,6 +39,18 @@ export interface PipelineOptions {
    */
   tileGrid?: number;
   minConfidence: number;
+  /**
+   * Per-class overrides of `minConfidence`, by sport class name.
+   *
+   * A basketball is a handful of pixels and the model is right to be unsure
+   * about it; a person fills a fifth of the frame and a 0.08 "person" is junk.
+   * One threshold for both meant the ball was held to a standard set by what
+   * players need. Measured over 20s of a real game, dropping only the ball's
+   * floor from 0.25 to 0.08 took it from 173 sampled positions to 293 — while
+   * the number of ball *tracks* stayed at 16, which is what says the extra
+   * detections are the same ball seen more often rather than new phantoms.
+   */
+  classConfidence?: Record<string, number>;
   iouThreshold: number;
   sourceWidth: number;
   sourceHeight: number;
@@ -174,9 +186,22 @@ export const runPipeline = async (options: PipelineOptions): Promise<PipelineRes
     fps: options.fps,
   });
 
+  /**
+   * Decode at the most permissive floor any class asks for; each detection is
+   * then held to its own class's threshold once we know what it is. The decoder
+   * cannot do this itself — it works in class indices, before the sport's
+   * mapping has been applied.
+   */
+  const decodeFloor = Math.min(
+    options.minConfidence,
+    ...Object.values(options.classConfidence ?? {}),
+  );
+
   const tracker = new ByteTracker({
     highThreshold: Math.max(options.minConfidence, 0.4),
-    lowThreshold: options.minConfidence,
+    // ByteTrack's second pass exists to re-attach exactly the faint detections
+    // this lets through, so the tracker's floor follows the lowest class floor.
+    lowThreshold: decodeFloor,
     iouThreshold: options.iouThreshold,
   });
 
@@ -213,11 +238,11 @@ export const runPipeline = async (options: PipelineOptions): Promise<PipelineRes
       // returns a full set of plausible boxes in the wrong places.
       const decodedHead =
         headKindFor(head.dims) === 'yolov8'
-          ? decodeYolov8(raw, head.dims, options.minConfidence)
+          ? decodeYolov8(raw, head.dims, decodeFloor)
           : decodeYolox(raw, head.dims[head.dims.length - 1] ?? 0, {
               inputWidth: size,
               inputHeight: size,
-              scoreThreshold: options.minConfidence,
+              scoreThreshold: decodeFloor,
             });
       return decodedHead.map((d) => ({
         ...d,
@@ -258,7 +283,15 @@ export const runPipeline = async (options: PipelineOptions): Promise<PipelineRes
     // Drop classes this sport does not care about before NMS, so a stray
     // "bench" never suppresses a player. NMS also fuses the duplicates that
     // overlapping tiles and the full-frame pass necessarily produce.
-    const relevant = found.filter((d) => mapping.byIndex[d.classId] !== undefined);
+    //
+    // The per-class floor is applied here rather than in the decoder: the model
+    // is run once at the lowest floor any class asks for, and each detection is
+    // then held to its own class's standard.
+    const relevant = found.filter((d) => {
+      const className = mapping.byIndex[d.classId];
+      if (className === undefined) return false;
+      return d.score >= (options.classConfidence?.[className] ?? options.minConfidence);
+    });
     const kept = nonMaxSuppression(relevant, options.iouThreshold);
 
     const inSourceSpace: Detection[] = kept.map((detection) => {

packages/core/src/analyze.ts

@@ -36,19 +36,72 @@ export interface PresetSettings {
    * because it costs tiles^2 + 1 inferences per frame.
    */
   tileGrid: number;
+  /**
+   * Detection floors for classes that need a different standard from people,
+   * as `class: confidence`.
+   *
+   * A basketball is a handful of pixels; a player fills a fifth of the frame.
+   * Holding both to one threshold meant the ball was judged by what a person
+   * needs. Measured over 20s of a real game at the shipped tile grid: dropping
+   * only the ball to 0.08 took it from 173 sampled positions to 293, and the
+   * rim from 316 to 403, at identical cost — while ball *track* count stayed
+   * at 16, which is what distinguishes better recall from new phantoms.
+   */
+  classConfidence?: Record<string, number>;
 }
 
+/**
+ * Small, fast-moving and low-contrast: the things a detector is legitimately
+ * unsure about, and the two classes scoring most depends on.
+ */
+const SMALL_OBJECTS: Record<string, number> = { ball: 0.08, puck: 0.08, hoop: 0.15, net: 0.15 };
+
 export const PRESET_SETTINGS: Record<Exclude<Preset, 'custom'>, PresetSettings> = {
   // CPU-only is a hard requirement, so "fast" has to be genuinely cheap.
-  fast: { frameStride: 5, inferenceSize: 512, minConfidence: 0.35, useProxy: true, tileGrid: 1 },
-  balanced: { frameStride: 2, inferenceSize: 768, minConfidence: 0.3, useProxy: true, tileGrid: 1 },
-  accurate: { frameStride: 1, inferenceSize: 1280, minConfidence: 0.25, useProxy: false, tileGrid: 1 },
+  fast: {
+    frameStride: 5,
+    inferenceSize: 512,
+    minConfidence: 0.35,
+    useProxy: true,
+    tileGrid: 1,
+    classConfidence: SMALL_OBJECTS,
+  },
+  balanced: {
+    frameStride: 2,
+    inferenceSize: 768,
+    minConfidence: 0.3,
+    useProxy: true,
+    tileGrid: 1,
+    classConfidence: SMALL_OBJECTS,
+  },
+  accurate: {
+    frameStride: 1,
+    inferenceSize: 1280,
+    minConfidence: 0.25,
+    useProxy: false,
+    tileGrid: 1,
+    classConfidence: SMALL_OBJECTS,
+  },
   /**
    * The one that can see the ball. Five inferences per frame instead of one,
    * so it is minutes rather than seconds — offered as a choice, not a default,
    * because nobody's existing runtime should regress silently.
    */
-  thorough: { frameStride: 2, inferenceSize: 1280, minConfidence: 0.25, useProxy: false, tileGrid: 2 },
+  thorough: {
+    frameStride: 2,
+    inferenceSize: 1280,
+    minConfidence: 0.25,
+    useProxy: false,
+    /**
+     * Two, not three. A 3x3 grid was measured against this same footage and was
+     * *worse* for the thing tiling exists to find: 127 ball positions against
+     * 173, and 100 rim positions against 316, for twice the runtime. Tiles get
+     * smaller but the whole-frame pass downsamples further to feed them, and
+     * the rim loses more than the ball gains.
+     */
+    tileGrid: 2,
+    classConfidence: SMALL_OBJECTS,
+  },
 };
 
 /**
@@ -97,6 +150,7 @@ export const settingsForPreset = (preset: Preset): PresetSettings => {
       minConfidence: 0.3,
       useProxy: true,
       tileGrid: 1,
+      classConfidence: SMALL_OBJECTS,
     };
   }
   return PRESET_SETTINGS[preset];
@@ -428,6 +482,14 @@ export const analyzeProject = async (
             String(settings.minConfidence),
             '--tile-grid',
             String(settings.tileGrid),
+            ...(settings.classConfidence === undefined
+              ? []
+              : [
+                  '--class-confidence',
+                  Object.entries(settings.classConfidence)
+                    .map(([name, value]) => `${name}=${value}`)
+                    .join(','),
+                ]),
             '--tracker',
             plugin.tracker.algorithm,
             '--backend',