feat: follow an athlete past the fragment you pointed at

apps/cv-worker/src/appearance.test.ts

@@ -0,0 +1,135 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+  accumulate,
+  BIN_COUNT,
+  binFor,
+  normalize,
+  similarity,
+  toHsv,
+  torsoRect,
+} from './appearance.js';
+
+/**
+ * The safety property under all of this: a signature that cannot tell two teams
+ * apart is useless, and a signature that confidently equates them is dangerous.
+ * These pin both directions.
+ */
+
+/** A solid-colour BGR frame, so a crop's expected histogram is known exactly. */
+const solidFrame = (width: number, height: number, [b, g, r]: [number, number, number]): Buffer => {
+  const buffer = Buffer.alloc(width * height * 3);
+  for (let i = 0; i < width * height; i += 1) {
+    buffer[i * 3] = b;
+    buffer[i * 3 + 1] = g;
+    buffer[i * 3 + 2] = r;
+  }
+  return buffer;
+};
+
+const signatureOf = (frame: Buffer, width: number, height: number): number[] => {
+  const bins = new Float64Array(BIN_COUNT);
+  const rect = torsoRect({ x: 0, y: 0, w: width, h: height }, 1, width, height);
+  expect(rect).not.toBeNull();
+  accumulate(frame, width, rect!, bins);
+  return normalize(bins);
+};
+
+describe('colour conversion', () => {
+  it('reads primaries at the hues they belong to', () => {
+    expect(toHsv(255, 0, 0).h).toBeCloseTo(0);
+    expect(toHsv(0, 255, 0).h).toBeCloseTo(120);
+    expect(toHsv(0, 0, 255).h).toBeCloseTo(240);
+  });
+
+  it('reports grey as unsaturated whatever its lightness', () => {
+    expect(toHsv(128, 128, 128).s).toBe(0);
+    expect(toHsv(255, 255, 255).s).toBe(0);
+    expect(toHsv(0, 0, 0).s).toBe(0);
+  });
+});
+
+describe('binning', () => {
+  it('sends washed-out and near-black pixels to the lightness bins, not a random hue', () => {
+    // A white shirt has a hue, arithmetically; it means nothing. Binning it by
+    // hue would scatter white jerseys across the spectrum at random.
+    const white = binFor(toHsv(250, 250, 250));
+    const black = binFor(toHsv(4, 4, 6));
+    expect(white).toBeGreaterThanOrEqual(24);
+    expect(black).toBeGreaterThanOrEqual(24);
+    expect(white).not.toBe(black);
+  });
+
+  it('keeps saturated colours apart', () => {
+    expect(binFor(toHsv(255, 0, 0))).not.toBe(binFor(toHsv(0, 0, 255)));
+  });
+
+  it('never returns a bin outside the histogram', () => {
+    for (let r = 0; r <= 255; r += 17) {
+      for (let g = 0; g <= 255; g += 17) {
+        for (let b = 0; b <= 255; b += 17) {
+          const bin = binFor(toHsv(r, g, b));
+          expect(bin).toBeGreaterThanOrEqual(0);
+          expect(bin).toBeLessThan(BIN_COUNT);
+        }
+      }
+    }
+  });
+});
+
+describe('the torso crop', () => {
+  it('takes the shirt, not the head, the legs or the air beside them', () => {
+    const rect = torsoRect({ x: 100, y: 200, w: 100, h: 200 }, 1, 1920, 1080);
+    expect(rect).toEqual({ x0: 120, x1: 180, y0: 230, y1: 300 });
+  });
+
+  it('scales into decoded-frame pixels', () => {
+    // Boxes arrive in source pixels; the frame is decoded smaller.
+    const rect = torsoRect({ x: 100, y: 200, w: 100, h: 200 }, 0.5, 960, 540);
+    expect(rect).toEqual({ x0: 60, x1: 90, y0: 115, y1: 150 });
+  });
+
+  it('refuses a box with nothing left in frame rather than inventing a sliver', () => {
+    expect(torsoRect({ x: -500, y: 0, w: 100, h: 200 }, 1, 960, 540)).toBeNull();
+    expect(torsoRect({ x: 0, y: 0, w: 1, h: 1 }, 1, 960, 540)).toBeNull();
+  });
+});
+
+describe('comparing two players', () => {
+  it('matches a shirt against itself', () => {
+    const red = signatureOf(solidFrame(40, 80, [30, 30, 200]), 40, 80);
+    expect(similarity(red, red)).toBeCloseTo(1);
+  });
+
+  it('separates two teams wearing different colours', () => {
+    const red = signatureOf(solidFrame(40, 80, [30, 30, 200]), 40, 80);
+    const blue = signatureOf(solidFrame(40, 80, [200, 30, 30]), 40, 80);
+    // The whole point. If this ever creeps up, the wrong child ends up in a reel.
+    expect(similarity(red, blue)).toBeLessThan(0.1);
+  });
+
+  it('separates a white shirt from a black one', () => {
+    const white = signatureOf(solidFrame(40, 80, [245, 245, 245]), 40, 80);
+    const black = signatureOf(solidFrame(40, 80, [12, 12, 12]), 40, 80);
+    expect(similarity(white, black)).toBeLessThan(0.1);
+  });
+
+  it('still recognises a shirt through a shading change', () => {
+    // Same jersey, one player in sun and one in shadow: value drops, hue holds.
+    const lit = signatureOf(solidFrame(40, 80, [40, 40, 220]), 40, 80);
+    const shaded = signatureOf(solidFrame(40, 80, [26, 26, 140]), 40, 80);
+    expect(similarity(lit, shaded)).toBeGreaterThan(0.8);
+  });
+
+  it('normalises away crop size, so a close-up matches a distant shot', () => {
+    const near = signatureOf(solidFrame(80, 160, [30, 180, 40]), 80, 160);
+    const far = signatureOf(solidFrame(12, 24, [30, 180, 40]), 12, 24);
+    expect(similarity(near, far)).toBeCloseTo(1, 1);
+  });
+
+  it('gives an empty signature no similarity to anything', () => {
+    const nothing = normalize(new Float64Array(BIN_COUNT));
+    const red = signatureOf(solidFrame(40, 80, [30, 30, 200]), 40, 80);
+    expect(similarity(nothing, red)).toBe(0);
+  });
+});

apps/cv-worker/src/appearance.ts

@@ -0,0 +1,160 @@
+/**
+ * What a player looks like, reduced to something two tracks can be compared on.
+ *
+ * Re-identification matched on box overlap alone, which by construction only
+ * finds an athlete where they were already known to be: it can confirm a
+ * binding across a re-detection, but it can never discover the same child
+ * somewhere else in the game. On real footage that left an athlete identified
+ * for 31.7s of a 300s match, with every signal that follows them dark for the
+ * other 90%.
+ *
+ * A jersey is the one thing about a child that a detector can see and that
+ * stays the same all afternoon, so that is what this measures: a coarse colour
+ * histogram of the torso. Deliberately coarse — the point is to tell one team's
+ * shirt from the other's and one shirt from the floor, not to recognise a face.
+ * Anything finer would invite false confidence, and the cost of a confident
+ * wrong answer here is another family's child in your highlight reel.
+ */
+
+/** Where the shirt is, as fractions of a player's box. */
+const TORSO = { x0: 0.2, x1: 0.8, y0: 0.15, y1: 0.5 };
+
+/** 12 hues x 2 saturations for colour, plus 4 lightness bins for grey. */
+export const HUE_BINS = 12;
+export const SAT_BINS = 2;
+export const GREY_BINS = 4;
+export const BIN_COUNT = HUE_BINS * SAT_BINS + GREY_BINS;
+
+/**
+ * Below these a pixel has no usable hue — a white shirt, a black shoe, a shadow
+ * — and binning it by hue would scatter it at random across the spectrum. Those
+ * pixels carry their lightness instead, which is what actually distinguishes a
+ * white jersey from a dark one.
+ */
+const MIN_SATURATION = 0.2;
+const MIN_VALUE = 0.15;
+
+export interface Box {
+  x: number;
+  y: number;
+  w: number;
+  h: number;
+}
+
+export interface Rect {
+  x0: number;
+  y0: number;
+  x1: number;
+  y1: number;
+}
+
+/**
+ * The torso of a box, scaled into decoded-frame pixels and clamped to the
+ * frame. Returns null when nothing usable is left — an off-screen or
+ * sub-pixel box has no appearance to measure, and inventing one from a
+ * clamped sliver would be worse than skipping it.
+ */
+export const torsoRect = (
+  box: Box,
+  scale: number,
+  frameWidth: number,
+  frameHeight: number,
+): Rect | null => {
+  const x0 = Math.round((box.x + box.w * TORSO.x0) * scale);
+  const x1 = Math.round((box.x + box.w * TORSO.x1) * scale);
+  const y0 = Math.round((box.y + box.h * TORSO.y0) * scale);
+  const y1 = Math.round((box.y + box.h * TORSO.y1) * scale);
+
+  const clamped: Rect = {
+    x0: Math.max(0, Math.min(frameWidth, x0)),
+    x1: Math.max(0, Math.min(frameWidth, x1)),
+    y0: Math.max(0, Math.min(frameHeight, y0)),
+    y1: Math.max(0, Math.min(frameHeight, y1)),
+  };
+  if (clamped.x1 - clamped.x0 < 2 || clamped.y1 - clamped.y0 < 2) return null;
+  return clamped;
+};
+
+export interface Hsv {
+  /** Degrees, 0..360. */
+  h: number;
+  s: number;
+  v: number;
+}
+
+/** Standard conversion, on 0..255 channels. */
+export const toHsv = (r: number, g: number, b: number): Hsv => {
+  const rn = r / 255;
+  const gn = g / 255;
+  const bn = b / 255;
+  const max = Math.max(rn, gn, bn);
+  const min = Math.min(rn, gn, bn);
+  const delta = max - min;
+
+  let h = 0;
+  if (delta > 0) {
+    if (max === rn) h = 60 * (((gn - bn) / delta) % 6);
+    else if (max === gn) h = 60 * ((bn - rn) / delta + 2);
+    else h = 60 * ((rn - gn) / delta + 4);
+  }
+  if (h < 0) h += 360;
+  return { h, s: max === 0 ? 0 : delta / max, v: max };
+};
+
+/** Which bin a colour belongs in; grey and near-black go to the lightness bins. */
+export const binFor = ({ h, s, v }: Hsv): number => {
+  if (s < MIN_SATURATION || v < MIN_VALUE) {
+    const bin = Math.min(GREY_BINS - 1, Math.floor(v * GREY_BINS));
+    return HUE_BINS * SAT_BINS + Math.max(0, bin);
+  }
+  const hue = Math.min(HUE_BINS - 1, Math.floor(h / (360 / HUE_BINS)));
+  const sat = s < 0.5 ? 0 : 1;
+  return hue * SAT_BINS + sat;
+};
+
+/**
+ * Adds one crop's colours into an accumulator. BGR because that is the order
+ * the frame decoder emits, matching what the detector was trained on.
+ */
+export const accumulate = (
+  pixels: Buffer | Uint8Array,
+  frameWidth: number,
+  rect: Rect,
+  into: Float64Array,
+): number => {
+  let counted = 0;
+  for (let y = rect.y0; y < rect.y1; y += 1) {
+    const row = y * frameWidth;
+    for (let x = rect.x0; x < rect.x1; x += 1) {
+      const at = (row + x) * 3;
+      const b = pixels[at] ?? 0;
+      const g = pixels[at + 1] ?? 0;
+      const r = pixels[at + 2] ?? 0;
+      const bin = binFor(toHsv(r, g, b));
+      into[bin] = (into[bin] ?? 0) + 1;
+      counted += 1;
+    }
+  }
+  return counted;
+};
+
+/** Sum to one, so signatures from crops of different sizes are comparable. */
+export const normalize = (histogram: Float64Array | number[]): number[] => {
+  let total = 0;
+  for (const value of histogram) total += value;
+  if (total <= 0) return Array.from({ length: histogram.length }, () => 0);
+  return Array.from(histogram, (value) => value / total);
+};
+
+/**
+ * Histogram intersection: 1 for identical signatures, 0 for no shared colour at
+ * all. Chosen over a Euclidean distance because it degrades gracefully when a
+ * crop catches some background — the extra mass simply fails to overlap,
+ * rather than dominating the distance.
+ */
+export const similarity = (a: number[], b: number[]): number => {
+  const length = Math.min(a.length, b.length);
+  let shared = 0;
+  for (let i = 0; i < length; i += 1) shared += Math.min(a[i] ?? 0, b[i] ?? 0);
+  return shared;
+};