From 7538d0f1d66b1a71b94504979868408516d6a238 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 10 Aug 2026 15:06:17 +0000 Subject: [PATCH] feat: judge a basketball by what a basketball looks like MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "the ball is hardly ever identified. we need to work on that." Measured before changing anything, on 20s of the reported game at the shipped 2x2 tile grid. The obvious lever turned out to be the wrong one: 3x3 tiling ball 13 tracks / 127 positions, rim 100 — against 15 / 173 and 316 for 2x2, at twice the runtime. Tiles get smaller, but the whole-frame pass has to downsample further to feed them, and the rim loses more than the ball gains. So the grid stays at two, and the comment in the preset now says why, because 3x3 is the first thing anyone will reach for next. The lever that does work is the confidence floor, swept over the same clip: floor ball tracks / positions rim positions player tracks 0.25 15 / 173 316 279 0.18 16 / 206 351 294 0.12 17 / 253 379 285 0.08 17 / 293 403 313 0.05 16 / 330 427 294 Ball *track* count stays flat while positions nearly double. That is the shape of better recall rather than more noise: a looser detector would invent short spurious tracks, not lengthen the ones already there. Lowering it globally would have loosened people too, so the floor is now per-class. A basketball is a handful of pixels and the model is right to be unsure about it; a player fills a fifth of the frame and a 0.08 "person" is junk. The model runs once at the most permissive floor any class asks for and each detection is then held to its own class's standard — the decoder cannot do this itself, since it works in class indices before the sport's mapping exists. ByteTrack's low-score pass follows the same floor, because re-attaching exactly these faint detections is what that pass is for. Verified through the real worker CLI on the same clip: before ball 15/173 | hoop 5/316 | player 279/8457 | 90s after ball 17/293 | hoop 6/359 | player 279/8457 | 91s +69% ball positions and +14% rim, with the player numbers identical to the byte, at the same cost. Co-Authored-By: Claude Opus 5 (1M context) --- apps/cv-worker/src/ballfloor.test.ts | 60 ++++++++++++++++++++++++ apps/cv-worker/src/index.ts | 21 +++++++++ apps/cv-worker/src/pipeline.ts | 41 ++++++++++++++-- packages/core/src/analyze.ts | 70 ++++++++++++++++++++++++++-- 4 files changed, 184 insertions(+), 8 deletions(-) create mode 100644 apps/cv-worker/src/ballfloor.test.ts diff --git a/apps/cv-worker/src/ballfloor.test.ts b/apps/cv-worker/src/ballfloor.test.ts new file mode 100644 index 0000000..2755eb3 --- /dev/null +++ b/apps/cv-worker/src/ballfloor.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest'; + +import { parseClassConfidence } from './index.js'; + +/** + * A basketball is a handful of pixels and the model is right to be unsure about + * it; a player fills a fifth of the frame and a 0.08 "person" is junk. Holding + * both to one number meant the ball was judged by the standard a person needs. + * + * Measured over 20s of a real game at the shipped 2x2 grid: + * + * floor ball tracks / positions rim positions player tracks + * 0.25 15 / 173 316 279 + * 0.18 16 / 206 351 294 + * 0.12 17 / 253 379 285 + * 0.08 17 / 293 403 313 + * 0.05 16 / 330 427 294 + * + * Track count stays flat while positions nearly double, which is the shape of + * better recall rather than new phantoms — a noisier detector would invent + * short tracks, not lengthen the ones already there. + */ + +describe('per-class detection floors', () => { + it('reads the pairs a preset sends', () => { + expect(parseClassConfidence('ball=0.08,hoop=0.15')).toEqual({ ball: 0.08, hoop: 0.15 }); + }); + + it('tolerates whitespace around names and values', () => { + expect(parseClassConfidence(' ball = 0.08 ')).toEqual({ ball: 0.08 }); + expect(parseClassConfidence('ball=0.08, hoop=0.15')).toEqual({ ball: 0.08, hoop: 0.15 }); + }); + + it('is absent rather than empty when nothing is asked for', () => { + expect(parseClassConfidence(undefined)).toEqual({}); + expect(parseClassConfidence('')).toEqual({}); + }); + + /** + * A typo in a preset should cost the ball some recall, not take a + * twenty-minute detection pass down with it. + */ + it('drops malformed pairs instead of throwing', () => { + expect(parseClassConfidence('ball')).toEqual({}); + expect(parseClassConfidence('ball=')).toEqual({}); + expect(parseClassConfidence('ball=abc')).toEqual({}); + expect(parseClassConfidence('=0.5')).toEqual({}); + expect(parseClassConfidence('ball=0.08,broken,hoop=0.15')).toEqual({ + ball: 0.08, + hoop: 0.15, + }); + }); + + it('refuses values outside a probability', () => { + // A floor above 1 detects nothing; a floor of 0 or below detects everything. + expect(parseClassConfidence('ball=1.5')).toEqual({}); + expect(parseClassConfidence('ball=0')).toEqual({}); + expect(parseClassConfidence('ball=-0.2')).toEqual({}); + }); +}); diff --git a/apps/cv-worker/src/index.ts b/apps/cv-worker/src/index.ts index 1e4add6..5b953e8 100644 --- a/apps/cv-worker/src/index.ts +++ b/apps/cv-worker/src/index.ts @@ -42,6 +42,26 @@ const number = (value: string | undefined, fallback: number): number => { return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; }; +/** + * `ball=0.08,hoop=0.15` — per-class detection floors. + * + * Malformed pairs are dropped rather than throwing: a typo in a preset should + * cost the ball some recall, not take a twenty-minute detection pass down with + * it. + */ +export const parseClassConfidence = (value: string | undefined): Record => { + if (value === undefined || value.trim().length === 0) return {}; + const out: Record = {}; + for (const pair of value.split(',')) { + const [name, raw] = pair.split('='); + const parsed = Number(raw); + if (name === undefined || name.trim().length === 0) continue; + if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 1) continue; + out[name.trim()] = parsed; + } + return out; +}; + /** * How many threads onnxruntime should use for one inference. * @@ -166,6 +186,7 @@ const detectAndTrack = async (flags: Record): Promise => { // Off unless asked for: it costs grid^2 + 1 inferences per frame. tileGrid: number(flags['tile-grid'], 1), minConfidence: number(flags['min-confidence'], 0.3), + classConfidence: parseClassConfidence(flags['class-confidence']), iouThreshold: number(flags['iou'], 0.45), sourceWidth: width, sourceHeight: height, diff --git a/apps/cv-worker/src/pipeline.ts b/apps/cv-worker/src/pipeline.ts index 8e20c15..b741e89 100644 --- a/apps/cv-worker/src/pipeline.ts +++ b/apps/cv-worker/src/pipeline.ts @@ -39,6 +39,18 @@ export interface PipelineOptions { */ tileGrid?: number; minConfidence: number; + /** + * Per-class overrides of `minConfidence`, by sport class name. + * + * A basketball is a handful of pixels and the model is right to be unsure + * about it; a person fills a fifth of the frame and a 0.08 "person" is junk. + * One threshold for both meant the ball was held to a standard set by what + * players need. Measured over 20s of a real game, dropping only the ball's + * floor from 0.25 to 0.08 took it from 173 sampled positions to 293 — while + * the number of ball *tracks* stayed at 16, which is what says the extra + * detections are the same ball seen more often rather than new phantoms. + */ + classConfidence?: Record; iouThreshold: number; sourceWidth: number; sourceHeight: number; @@ -174,9 +186,22 @@ export const runPipeline = async (options: PipelineOptions): Promise ({ ...d, @@ -258,7 +283,15 @@ export const runPipeline = async (options: PipelineOptions): Promise mapping.byIndex[d.classId] !== undefined); + // + // The per-class floor is applied here rather than in the decoder: the model + // is run once at the lowest floor any class asks for, and each detection is + // then held to its own class's standard. + const relevant = found.filter((d) => { + const className = mapping.byIndex[d.classId]; + if (className === undefined) return false; + return d.score >= (options.classConfidence?.[className] ?? options.minConfidence); + }); const kept = nonMaxSuppression(relevant, options.iouThreshold); const inSourceSpace: Detection[] = kept.map((detection) => { diff --git a/packages/core/src/analyze.ts b/packages/core/src/analyze.ts index f62cd25..bc88612 100644 --- a/packages/core/src/analyze.ts +++ b/packages/core/src/analyze.ts @@ -36,19 +36,72 @@ export interface PresetSettings { * because it costs tiles^2 + 1 inferences per frame. */ tileGrid: number; + /** + * Detection floors for classes that need a different standard from people, + * as `class: confidence`. + * + * A basketball is a handful of pixels; a player fills a fifth of the frame. + * Holding both to one threshold meant the ball was judged by what a person + * needs. Measured over 20s of a real game at the shipped tile grid: dropping + * only the ball to 0.08 took it from 173 sampled positions to 293, and the + * rim from 316 to 403, at identical cost — while ball *track* count stayed + * at 16, which is what distinguishes better recall from new phantoms. + */ + classConfidence?: Record; } +/** + * Small, fast-moving and low-contrast: the things a detector is legitimately + * unsure about, and the two classes scoring most depends on. + */ +const SMALL_OBJECTS: Record = { ball: 0.08, puck: 0.08, hoop: 0.15, net: 0.15 }; + export const PRESET_SETTINGS: Record, PresetSettings> = { // CPU-only is a hard requirement, so "fast" has to be genuinely cheap. - fast: { frameStride: 5, inferenceSize: 512, minConfidence: 0.35, useProxy: true, tileGrid: 1 }, - balanced: { frameStride: 2, inferenceSize: 768, minConfidence: 0.3, useProxy: true, tileGrid: 1 }, - accurate: { frameStride: 1, inferenceSize: 1280, minConfidence: 0.25, useProxy: false, tileGrid: 1 }, + fast: { + frameStride: 5, + inferenceSize: 512, + minConfidence: 0.35, + useProxy: true, + tileGrid: 1, + classConfidence: SMALL_OBJECTS, + }, + balanced: { + frameStride: 2, + inferenceSize: 768, + minConfidence: 0.3, + useProxy: true, + tileGrid: 1, + classConfidence: SMALL_OBJECTS, + }, + accurate: { + frameStride: 1, + inferenceSize: 1280, + minConfidence: 0.25, + useProxy: false, + tileGrid: 1, + classConfidence: SMALL_OBJECTS, + }, /** * The one that can see the ball. Five inferences per frame instead of one, * so it is minutes rather than seconds — offered as a choice, not a default, * because nobody's existing runtime should regress silently. */ - thorough: { frameStride: 2, inferenceSize: 1280, minConfidence: 0.25, useProxy: false, tileGrid: 2 }, + thorough: { + frameStride: 2, + inferenceSize: 1280, + minConfidence: 0.25, + useProxy: false, + /** + * Two, not three. A 3x3 grid was measured against this same footage and was + * *worse* for the thing tiling exists to find: 127 ball positions against + * 173, and 100 rim positions against 316, for twice the runtime. Tiles get + * smaller but the whole-frame pass downsamples further to feed them, and + * the rim loses more than the ball gains. + */ + tileGrid: 2, + classConfidence: SMALL_OBJECTS, + }, }; /** @@ -97,6 +150,7 @@ export const settingsForPreset = (preset: Preset): PresetSettings => { minConfidence: 0.3, useProxy: true, tileGrid: 1, + classConfidence: SMALL_OBJECTS, }; } return PRESET_SETTINGS[preset]; @@ -428,6 +482,14 @@ export const analyzeProject = async ( String(settings.minConfidence), '--tile-grid', String(settings.tileGrid), + ...(settings.classConfidence === undefined + ? [] + : [ + '--class-confidence', + Object.entries(settings.classConfidence) + .map(([name, value]) => `${name}=${value}`) + .join(','), + ]), '--tracker', plugin.tracker.algorithm, '--backend',