diff --git a/apps/cv-worker/src/ballfloor.test.ts b/apps/cv-worker/src/ballfloor.test.ts new file mode 100644 index 0000000..2755eb3 --- /dev/null +++ b/apps/cv-worker/src/ballfloor.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest'; + +import { parseClassConfidence } from './index.js'; + +/** + * A basketball is a handful of pixels and the model is right to be unsure about + * it; a player fills a fifth of the frame and a 0.08 "person" is junk. Holding + * both to one number meant the ball was judged by the standard a person needs. + * + * Measured over 20s of a real game at the shipped 2x2 grid: + * + * floor ball tracks / positions rim positions player tracks + * 0.25 15 / 173 316 279 + * 0.18 16 / 206 351 294 + * 0.12 17 / 253 379 285 + * 0.08 17 / 293 403 313 + * 0.05 16 / 330 427 294 + * + * Track count stays flat while positions nearly double, which is the shape of + * better recall rather than new phantoms — a noisier detector would invent + * short tracks, not lengthen the ones already there. + */ + +describe('per-class detection floors', () => { + it('reads the pairs a preset sends', () => { + expect(parseClassConfidence('ball=0.08,hoop=0.15')).toEqual({ ball: 0.08, hoop: 0.15 }); + }); + + it('tolerates whitespace around names and values', () => { + expect(parseClassConfidence(' ball = 0.08 ')).toEqual({ ball: 0.08 }); + expect(parseClassConfidence('ball=0.08, hoop=0.15')).toEqual({ ball: 0.08, hoop: 0.15 }); + }); + + it('is absent rather than empty when nothing is asked for', () => { + expect(parseClassConfidence(undefined)).toEqual({}); + expect(parseClassConfidence('')).toEqual({}); + }); + + /** + * A typo in a preset should cost the ball some recall, not take a + * twenty-minute detection pass down with it. + */ + it('drops malformed pairs instead of throwing', () => { + expect(parseClassConfidence('ball')).toEqual({}); + expect(parseClassConfidence('ball=')).toEqual({}); + expect(parseClassConfidence('ball=abc')).toEqual({}); + expect(parseClassConfidence('=0.5')).toEqual({}); + expect(parseClassConfidence('ball=0.08,broken,hoop=0.15')).toEqual({ + ball: 0.08, + hoop: 0.15, + }); + }); + + it('refuses values outside a probability', () => { + // A floor above 1 detects nothing; a floor of 0 or below detects everything. + expect(parseClassConfidence('ball=1.5')).toEqual({}); + expect(parseClassConfidence('ball=0')).toEqual({}); + expect(parseClassConfidence('ball=-0.2')).toEqual({}); + }); +}); diff --git a/apps/cv-worker/src/index.ts b/apps/cv-worker/src/index.ts index 1e4add6..5b953e8 100644 --- a/apps/cv-worker/src/index.ts +++ b/apps/cv-worker/src/index.ts @@ -42,6 +42,26 @@ const number = (value: string | undefined, fallback: number): number => { return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; }; +/** + * `ball=0.08,hoop=0.15` — per-class detection floors. + * + * Malformed pairs are dropped rather than throwing: a typo in a preset should + * cost the ball some recall, not take a twenty-minute detection pass down with + * it. + */ +export const parseClassConfidence = (value: string | undefined): Record => { + if (value === undefined || value.trim().length === 0) return {}; + const out: Record = {}; + for (const pair of value.split(',')) { + const [name, raw] = pair.split('='); + const parsed = Number(raw); + if (name === undefined || name.trim().length === 0) continue; + if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 1) continue; + out[name.trim()] = parsed; + } + return out; +}; + /** * How many threads onnxruntime should use for one inference. * @@ -166,6 +186,7 @@ const detectAndTrack = async (flags: Record): Promise => { // Off unless asked for: it costs grid^2 + 1 inferences per frame. tileGrid: number(flags['tile-grid'], 1), minConfidence: number(flags['min-confidence'], 0.3), + classConfidence: parseClassConfidence(flags['class-confidence']), iouThreshold: number(flags['iou'], 0.45), sourceWidth: width, sourceHeight: height, diff --git a/apps/cv-worker/src/pipeline.ts b/apps/cv-worker/src/pipeline.ts index 8e20c15..b741e89 100644 --- a/apps/cv-worker/src/pipeline.ts +++ b/apps/cv-worker/src/pipeline.ts @@ -39,6 +39,18 @@ export interface PipelineOptions { */ tileGrid?: number; minConfidence: number; + /** + * Per-class overrides of `minConfidence`, by sport class name. + * + * A basketball is a handful of pixels and the model is right to be unsure + * about it; a person fills a fifth of the frame and a 0.08 "person" is junk. + * One threshold for both meant the ball was held to a standard set by what + * players need. Measured over 20s of a real game, dropping only the ball's + * floor from 0.25 to 0.08 took it from 173 sampled positions to 293 — while + * the number of ball *tracks* stayed at 16, which is what says the extra + * detections are the same ball seen more often rather than new phantoms. + */ + classConfidence?: Record; iouThreshold: number; sourceWidth: number; sourceHeight: number; @@ -174,9 +186,22 @@ export const runPipeline = async (options: PipelineOptions): Promise ({ ...d, @@ -258,7 +283,15 @@ export const runPipeline = async (options: PipelineOptions): Promise mapping.byIndex[d.classId] !== undefined); + // + // The per-class floor is applied here rather than in the decoder: the model + // is run once at the lowest floor any class asks for, and each detection is + // then held to its own class's standard. + const relevant = found.filter((d) => { + const className = mapping.byIndex[d.classId]; + if (className === undefined) return false; + return d.score >= (options.classConfidence?.[className] ?? options.minConfidence); + }); const kept = nonMaxSuppression(relevant, options.iouThreshold); const inSourceSpace: Detection[] = kept.map((detection) => { diff --git a/packages/core/src/analyze.ts b/packages/core/src/analyze.ts index f62cd25..bc88612 100644 --- a/packages/core/src/analyze.ts +++ b/packages/core/src/analyze.ts @@ -36,19 +36,72 @@ export interface PresetSettings { * because it costs tiles^2 + 1 inferences per frame. */ tileGrid: number; + /** + * Detection floors for classes that need a different standard from people, + * as `class: confidence`. + * + * A basketball is a handful of pixels; a player fills a fifth of the frame. + * Holding both to one threshold meant the ball was judged by what a person + * needs. Measured over 20s of a real game at the shipped tile grid: dropping + * only the ball to 0.08 took it from 173 sampled positions to 293, and the + * rim from 316 to 403, at identical cost — while ball *track* count stayed + * at 16, which is what distinguishes better recall from new phantoms. + */ + classConfidence?: Record; } +/** + * Small, fast-moving and low-contrast: the things a detector is legitimately + * unsure about, and the two classes scoring most depends on. + */ +const SMALL_OBJECTS: Record = { ball: 0.08, puck: 0.08, hoop: 0.15, net: 0.15 }; + export const PRESET_SETTINGS: Record, PresetSettings> = { // CPU-only is a hard requirement, so "fast" has to be genuinely cheap. - fast: { frameStride: 5, inferenceSize: 512, minConfidence: 0.35, useProxy: true, tileGrid: 1 }, - balanced: { frameStride: 2, inferenceSize: 768, minConfidence: 0.3, useProxy: true, tileGrid: 1 }, - accurate: { frameStride: 1, inferenceSize: 1280, minConfidence: 0.25, useProxy: false, tileGrid: 1 }, + fast: { + frameStride: 5, + inferenceSize: 512, + minConfidence: 0.35, + useProxy: true, + tileGrid: 1, + classConfidence: SMALL_OBJECTS, + }, + balanced: { + frameStride: 2, + inferenceSize: 768, + minConfidence: 0.3, + useProxy: true, + tileGrid: 1, + classConfidence: SMALL_OBJECTS, + }, + accurate: { + frameStride: 1, + inferenceSize: 1280, + minConfidence: 0.25, + useProxy: false, + tileGrid: 1, + classConfidence: SMALL_OBJECTS, + }, /** * The one that can see the ball. Five inferences per frame instead of one, * so it is minutes rather than seconds — offered as a choice, not a default, * because nobody's existing runtime should regress silently. */ - thorough: { frameStride: 2, inferenceSize: 1280, minConfidence: 0.25, useProxy: false, tileGrid: 2 }, + thorough: { + frameStride: 2, + inferenceSize: 1280, + minConfidence: 0.25, + useProxy: false, + /** + * Two, not three. A 3x3 grid was measured against this same footage and was + * *worse* for the thing tiling exists to find: 127 ball positions against + * 173, and 100 rim positions against 316, for twice the runtime. Tiles get + * smaller but the whole-frame pass downsamples further to feed them, and + * the rim loses more than the ball gains. + */ + tileGrid: 2, + classConfidence: SMALL_OBJECTS, + }, }; /** @@ -97,6 +150,7 @@ export const settingsForPreset = (preset: Preset): PresetSettings => { minConfidence: 0.3, useProxy: true, tileGrid: 1, + classConfidence: SMALL_OBJECTS, }; } return PRESET_SETTINGS[preset]; @@ -428,6 +482,14 @@ export const analyzeProject = async ( String(settings.minConfidence), '--tile-grid', String(settings.tileGrid), + ...(settings.classConfidence === undefined + ? [] + : [ + '--class-confidence', + Object.entries(settings.classConfidence) + .map(([name, value]) => `${name}=${value}`) + .join(','), + ]), '--tracker', plugin.tracker.algorithm, '--backend',