impr(weakspot): score within-word bigrams instead of characters (@JMLegere)

frontend/__tests__/test/weak-spot.spec.ts

@@ -0,0 +1,192 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { Wordset } from "../../src/ts/test/wordset";
+import type { InputEventData } from "../../src/ts/test/events/types";
+
+type WeakSpotModule = typeof import("../../src/ts/test/weak-spot");
+type LiveCacheModule = typeof import("../../src/ts/test/events/live-cache");
+
+function wordsetReturning(first: string, second: string): Wordset {
+  return {
+    randomWord: vi
+      .fn()
+      .mockReturnValue(first)
+      .mockReturnValueOnce(first)
+      .mockReturnValueOnce(second),
+  } as unknown as Wordset;
+}
+
+describe("weak-spot", () => {
+  let WeakSpot: WeakSpotModule;
+  let liveCache: LiveCacheModule;
+  let now: number;
+
+  beforeEach(async () => {
+    vi.resetModules();
+    WeakSpot = await import("../../src/ts/test/weak-spot");
+    liveCache = await import("../../src/ts/test/events/live-cache");
+    liveCache.resetLiveCache();
+    now = 0;
+  });
+
+  function input(word: string, data: InputEventData, spacing = 100): void {
+    now += spacing;
+    liveCache.recordEventForCache({
+      type: "input",
+      ms: now,
+      testMs: now,
+      data,
+    });
+    if (data.inputType === "insertText") WeakSpot.updateScore(word);
+  }
+
+  function letter(
+    word: string,
+    char: string,
+    charIndex: number,
+    spacing = 100,
+    extra: Partial<InputEventData> = {},
+  ): void {
+    input(
+      word,
+      {
+        inputType: "insertText",
+        wordIndex: 0,
+        charIndex,
+        inputValue: word.slice(0, charIndex) + char,
+        data: char,
+        correct: word.slice(charIndex).startsWith(char),
+        ...extra,
+      },
+      spacing,
+    );
+  }
+
+  function pair(word: string, spacing: number, correct = true): void {
+    liveCache.resetLiveCache();
+    letter(word, word[0] as string, 0);
+    letter(word, correct ? (word[1] as string) : "x", 1, spacing);
+  }
+
+  it("selects the slow transition without treating its letters as weak", () => {
+    pair("th", 200);
+    pair("sh", 80);
+    expect(WeakSpot.getWord(wordsetReturning("ship", "thing"))).toBe("thing");
+  });
+
+  it("penalizes the intended pair rather than the typo", () => {
+    pair("ab", 1000);
+    pair("th", 1, false);
+    expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("th");
+    expect(WeakSpot.getWord(wordsetReturning("ab", "tx"))).toBe("ab");
+  });
+
+  it("keeps the 5000 ms error penalty", () => {
+    pair("th", 100, false);
+    pair("ab", 5099);
+    expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("th");
+    pair("cd", 5101);
+    expect(WeakSpot.getWord(wordsetReturning("th", "cd"))).toBe("cd");
+  });
+
+  it("does not learn a transition immediately after an incorrect key", () => {
+    letter("the", "t", 0);
+    letter("the", "x", 1);
+    letter("the", "e", 2);
+    expect(WeakSpot.getWord(wordsetReturning("ab", "xe"))).toBe("ab");
+  });
+
+  it("scores a stopped error but not its retry as a new transition", () => {
+    letter("th", "t", 0);
+    letter("th", "x", 1, 100, { inputStopped: true });
+    letter("th", "h", 1);
+    expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("th");
+    expect(WeakSpot.getWord(wordsetReturning("ab", "xh"))).toBe("ab");
+  });
+
+  it("does not bridge whitespace or word changes", () => {
+    letter("t ", "t", 0);
+    letter("t ", " ", 1, 5000, { commitsWord: true });
+    letter("he", "h", 0, 5000, { wordIndex: 1 });
+    expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("ab");
+    expect(WeakSpot.getWord(wordsetReturning("ab", "t "))).toBe("ab");
+    expect(WeakSpot.getWord(wordsetReturning("ab", " h"))).toBe("ab");
+  });
+
+  it("does not bridge words even when no space was typed", () => {
+    letter("t", "t", 0);
+    letter("he", "h", 0, 5000, { wordIndex: 1 });
+    expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("ab");
+  });
+
+  it("ignores overtyping beyond the target word", () => {
+    letter("t", "t", 0);
+    letter("t", "h", 1);
+    expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("ab");
+  });
+
+  it("resets transition context, but keeps learned scores, across tests", () => {
+    pair("ab", 100);
+    liveCache.resetLiveCache();
+    letter("th", "t", 0);
+    expect(WeakSpot.getWord(wordsetReturning("ab", "bt"))).toBe("ab");
+    letter("th", "h", 1, 200);
+    expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("th");
+  });
+
+  it("does not use deletion timing as a letter transition", () => {
+    letter("the", "t", 0);
+    letter("the", "h", 1);
+    input("the", {
+      inputType: "deleteContentBackward",
+      wordIndex: 0,
+      charIndex: 2,
+      inputValue: "t",
+    });
+    letter("the", "h", 1, 5000);
+    pair("ab", 200);
+    expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("ab");
+    expect(WeakSpot.getWord(wordsetReturning("ab", "hh"))).toBe("ab");
+  });
+
+  it.each([
+    { automatic: true as const },
+    { isCompositionEnding: true as const },
+  ])("ignores generated or composed input: %j", (extra) => {
+    letter("th", "t", 0);
+    letter("th", "h", 1, 5000, extra);
+    letter("the", "e", 2);
+    expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("ab");
+    expect(WeakSpot.getWord(wordsetReturning("ab", "he"))).toBe("ab");
+  });
+
+  it("forms candidate pairs from Unicode code points", () => {
+    letter("𐐀a", "𐐀", 0);
+    letter("𐐀a", "a", 2);
+    expect(WeakSpot.getWord(wordsetReturning("ab", "𐐀a"))).toBe("𐐀a");
+  });
+
+  it("averages only observed bigrams", () => {
+    pair("th", 300);
+    pair("he", 100);
+    pair("ab", 150);
+    expect(WeakSpot.getWord(wordsetReturning("ab", "them"))).toBe("them");
+    pair("cd", 250);
+    expect(WeakSpot.getWord(wordsetReturning("them", "cd"))).toBe("cd");
+  });
+
+  it("retains the capped 50-observation moving average", () => {
+    for (let i = 0; i < 50; i++) pair("th", 100);
+    pair("th", 5100); // 5100 / 50 + 100 * 49 / 50 = 200
+    pair("ab", 199);
+    pair("cd", 201);
+    expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("th");
+    expect(WeakSpot.getWord(wordsetReturning("th", "cd"))).toBe("cd");
+  });
+
+  it("samples 20 words, keeping the first on ties or no observations", () => {
+    const wordset = wordsetReturning("a", "b");
+    expect(WeakSpot.getWord(wordset)).toBe("a");
+    expect(wordset.randomWord).toHaveBeenCalledTimes(20);
+    expect(wordset.randomWord).toHaveBeenCalledWith("normal");
+  });
+});

frontend/src/ts/input/handlers/insert-text.ts

@@ -296,7 +296,7 @@ export async function onInsertText(options: OnInsertTextParams): Promise<void> {
   });
 
   // this needs to be called after event logging
-  WeakSpot.updateScore(data, correct);
+  WeakSpot.updateScore(currentWord);
 
   // delete on error
   // skipped when the input was stopped - nothing was inserted to delete

frontend/src/ts/test/events/live-cache.ts

@@ -1,12 +1,14 @@
 import { roundTo2 } from "@monkeytype/util/numbers";
-import { TestEvent } from "./types";
+import { InputEvent, TestEvent } from "./types";
 
 // Running tallies maintained as events arrive, so live readers don't rescan
 // the event log. For replay, derive from the event log directly.
 const cache = {
   correctInputs: 0,
   totalInputs: 0,
   timerStartMs: null as number | null,
+  previousInput: null as InputEvent | null,
+  lastInput: null as InputEvent | null,
   msSinceLastInputEvent: {
     value: null as number | null,
     lastEventMs: null as number | null,
@@ -17,12 +19,16 @@ export function resetLiveCache(): void {
   cache.correctInputs = 0;
   cache.totalInputs = 0;
   cache.timerStartMs = null;
+  cache.previousInput = null;
+  cache.lastInput = null;
   cache.msSinceLastInputEvent.value = null;
   cache.msSinceLastInputEvent.lastEventMs = null;
 }
 
 export function recordEventForCache(event: TestEvent): void {
   if (event.type === "input") {
+    cache.previousInput = cache.lastInput;
+    cache.lastInput = event;
     if ("correct" in event.data) {
       cache.totalInputs++;
       if (event.data.correct) cache.correctInputs++;
@@ -48,6 +54,13 @@ export function getLiveCachedMsSinceLastInputEvent(): number | null {
   return cache.msSinceLastInputEvent.value;
 }
 
+export function getLiveCachedInputPair(): [
+  InputEvent | null,
+  InputEvent | null,
+] {
+  return [cache.previousInput, cache.lastInput];
+}
+
 export function getLiveCachedTimerStartMs(): number | null {
   return cache.timerStartMs;
 }

frontend/src/ts/test/weak-spot.ts

@@ -1,15 +1,18 @@
-import { getLiveCachedMsSinceLastInputEvent } from "./events/live-cache";
+import {
+  getLiveCachedInputPair,
+  getLiveCachedMsSinceLastInputEvent,
+} from "./events/live-cache";
 import { Wordset } from "./wordset";
 
-// Changes how quickly it 'learns' scores - very roughly the score for a char
-// is based on last perCharCount occurrences. Make it smaller to adjust faster.
-const perCharCount = 50;
+// Changes how quickly it 'learns' scores - very roughly the score for a bigram
+// is based on last perBigramCount occurrences. Make it smaller to adjust faster.
+const perBigramCount = 50;
 
 // Choose the highest scoring word from this many random words. Higher values
-// will choose words with more weak letters on average.
+// will choose words with more weak bigrams on average.
 const wordSamples = 20;
 
-// Score penatly (in milliseconds) for getting a letter wrong.
+// Score penalty (in milliseconds) for getting the second letter wrong.
 const incorrectPenalty = 5000;
 
 const scores: Record<string, Score> = {};
@@ -23,7 +26,7 @@ class Score {
   }
 
   update(score: number): void {
-    if (this.count < perCharCount) {
+    if (this.count < perBigramCount) {
       this.count++;
     }
     const adjustRate = 1.0 / this.count;
@@ -32,31 +35,62 @@ class Score {
   }
 }
 
-export function updateScore(char: string, isCorrect: boolean): void {
+export function updateScore(word: string): void {
+  const [previous, current] = getLiveCachedInputPair();
   const spacing = getLiveCachedMsSinceLastInputEvent();
-  if (spacing === null) {
+  if (previous === null || current === null || spacing === null) {
     return;
   }
+
+  const before = previous.data;
+  const after = current.data;
+  // Only consecutive, manually typed characters in the same word have a
+  // meaningful transition time. Deletions and corrections break the pair.
+  if (
+    before.inputType !== "insertText" ||
+    after.inputType !== "insertText" ||
+    before.automatic ||
+    after.automatic ||
+    before.isCompositionEnding ||
+    after.isCompositionEnding ||
+    !before.correct ||
+    before.inputStopped ||
+    before.commitsWord ||
+    before.wordIndex !== after.wordIndex ||
+    before.charIndex + before.data.length !== after.charIndex ||
+    [...before.data].length !== 1 ||
+    [...after.data].length !== 1 ||
+    /\s/u.test(before.data + after.data)
+  ) {
+    return;
+  }
+
+  const char = [...word.slice(after.charIndex)][0];
+  if (char === undefined || /\s/u.test(char)) return;
+  // Penalize the intended pair, not the typo (e.g. th, not tx).
+  const bigram = before.data + char;
   let score = spacing;
-  if (!isCorrect) {
+  if (!after.correct) {
     score += incorrectPenalty;
   }
-  if (!(char in scores)) {
-    scores[char] = new Score();
+  if (!(bigram in scores)) {
+    scores[bigram] = new Score();
   }
-  scores[char]?.update(score);
+  scores[bigram]?.update(score);
 }
 
 function score(word: string): number {
   let total = 0.0;
-  let numChars = 0;
-  for (const c of word) {
-    if (c in scores) {
-      total += (scores[c] as Score).average;
-      numChars++;
+  let numBigrams = 0;
+  const chars = [...word];
+  for (let i = 1; i < chars.length; i++) {
+    const bigram = (chars[i - 1] as string) + (chars[i] as string);
+    if (bigram in scores) {
+      total += (scores[bigram] as Score).average;
+      numBigrams++;
     }
   }
-  return numChars === 0 ? 0.0 : total / numChars;
+  return numBigrams === 0 ? 0.0 : total / numBigrams;
 }
 
 export function getWord(wordset: Wordset): string {

packages/funbox/src/list.ts

@@ -314,7 +314,7 @@ const list: Record<FunboxName, FunboxMetadata> = {
     name: "wikipedia",
   },
   weakspot: {
-    description: "Focus on slow and mistyped letters.",
+    description: "Focus on slow and mistyped letter pairs.",
     canGetPb: false,
     difficultyLevel: 0,
     properties: ["changesWordsFrequency"],