From 75ab9ba14dfa6a799ecff1e428c6472811cd2bac Mon Sep 17 00:00:00 2001 From: Jeremy Legere Date: Fri, 11 Sep 2026 20:33:26 -0400 Subject: [PATCH] impr(weakspot): score within-word bigrams instead of characters --- frontend/__tests__/test/weak-spot.spec.ts | 192 ++++++++++++++++++ frontend/src/ts/input/handlers/insert-text.ts | 2 +- frontend/src/ts/test/events/live-cache.ts | 15 +- frontend/src/ts/test/weak-spot.ts | 72 +++++-- packages/funbox/src/list.ts | 2 +- 5 files changed, 261 insertions(+), 22 deletions(-) create mode 100644 frontend/__tests__/test/weak-spot.spec.ts diff --git a/frontend/__tests__/test/weak-spot.spec.ts b/frontend/__tests__/test/weak-spot.spec.ts new file mode 100644 index 000000000000..9ff55dd370fb --- /dev/null +++ b/frontend/__tests__/test/weak-spot.spec.ts @@ -0,0 +1,192 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { Wordset } from "../../src/ts/test/wordset"; +import type { InputEventData } from "../../src/ts/test/events/types"; + +type WeakSpotModule = typeof import("../../src/ts/test/weak-spot"); +type LiveCacheModule = typeof import("../../src/ts/test/events/live-cache"); + +function wordsetReturning(first: string, second: string): Wordset { + return { + randomWord: vi + .fn() + .mockReturnValue(first) + .mockReturnValueOnce(first) + .mockReturnValueOnce(second), + } as unknown as Wordset; +} + +describe("weak-spot", () => { + let WeakSpot: WeakSpotModule; + let liveCache: LiveCacheModule; + let now: number; + + beforeEach(async () => { + vi.resetModules(); + WeakSpot = await import("../../src/ts/test/weak-spot"); + liveCache = await import("../../src/ts/test/events/live-cache"); + liveCache.resetLiveCache(); + now = 0; + }); + + function input(word: string, data: InputEventData, spacing = 100): void { + now += spacing; + liveCache.recordEventForCache({ + type: "input", + ms: now, + testMs: now, + data, + }); + if (data.inputType === "insertText") WeakSpot.updateScore(word); + } + + function letter( + word: string, + char: string, + charIndex: number, + spacing = 100, + extra: Partial = {}, + ): void { + input( + word, + { + inputType: "insertText", + wordIndex: 0, + charIndex, + inputValue: word.slice(0, charIndex) + char, + data: char, + correct: word.slice(charIndex).startsWith(char), + ...extra, + }, + spacing, + ); + } + + function pair(word: string, spacing: number, correct = true): void { + liveCache.resetLiveCache(); + letter(word, word[0] as string, 0); + letter(word, correct ? (word[1] as string) : "x", 1, spacing); + } + + it("selects the slow transition without treating its letters as weak", () => { + pair("th", 200); + pair("sh", 80); + expect(WeakSpot.getWord(wordsetReturning("ship", "thing"))).toBe("thing"); + }); + + it("penalizes the intended pair rather than the typo", () => { + pair("ab", 1000); + pair("th", 1, false); + expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("th"); + expect(WeakSpot.getWord(wordsetReturning("ab", "tx"))).toBe("ab"); + }); + + it("keeps the 5000 ms error penalty", () => { + pair("th", 100, false); + pair("ab", 5099); + expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("th"); + pair("cd", 5101); + expect(WeakSpot.getWord(wordsetReturning("th", "cd"))).toBe("cd"); + }); + + it("does not learn a transition immediately after an incorrect key", () => { + letter("the", "t", 0); + letter("the", "x", 1); + letter("the", "e", 2); + expect(WeakSpot.getWord(wordsetReturning("ab", "xe"))).toBe("ab"); + }); + + it("scores a stopped error but not its retry as a new transition", () => { + letter("th", "t", 0); + letter("th", "x", 1, 100, { inputStopped: true }); + letter("th", "h", 1); + expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("th"); + expect(WeakSpot.getWord(wordsetReturning("ab", "xh"))).toBe("ab"); + }); + + it("does not bridge whitespace or word changes", () => { + letter("t ", "t", 0); + letter("t ", " ", 1, 5000, { commitsWord: true }); + letter("he", "h", 0, 5000, { wordIndex: 1 }); + expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("ab"); + expect(WeakSpot.getWord(wordsetReturning("ab", "t "))).toBe("ab"); + expect(WeakSpot.getWord(wordsetReturning("ab", " h"))).toBe("ab"); + }); + + it("does not bridge words even when no space was typed", () => { + letter("t", "t", 0); + letter("he", "h", 0, 5000, { wordIndex: 1 }); + expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("ab"); + }); + + it("ignores overtyping beyond the target word", () => { + letter("t", "t", 0); + letter("t", "h", 1); + expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("ab"); + }); + + it("resets transition context, but keeps learned scores, across tests", () => { + pair("ab", 100); + liveCache.resetLiveCache(); + letter("th", "t", 0); + expect(WeakSpot.getWord(wordsetReturning("ab", "bt"))).toBe("ab"); + letter("th", "h", 1, 200); + expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("th"); + }); + + it("does not use deletion timing as a letter transition", () => { + letter("the", "t", 0); + letter("the", "h", 1); + input("the", { + inputType: "deleteContentBackward", + wordIndex: 0, + charIndex: 2, + inputValue: "t", + }); + letter("the", "h", 1, 5000); + pair("ab", 200); + expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("ab"); + expect(WeakSpot.getWord(wordsetReturning("ab", "hh"))).toBe("ab"); + }); + + it.each([ + { automatic: true as const }, + { isCompositionEnding: true as const }, + ])("ignores generated or composed input: %j", (extra) => { + letter("th", "t", 0); + letter("th", "h", 1, 5000, extra); + letter("the", "e", 2); + expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("ab"); + expect(WeakSpot.getWord(wordsetReturning("ab", "he"))).toBe("ab"); + }); + + it("forms candidate pairs from Unicode code points", () => { + letter("𐐀a", "𐐀", 0); + letter("𐐀a", "a", 2); + expect(WeakSpot.getWord(wordsetReturning("ab", "𐐀a"))).toBe("𐐀a"); + }); + + it("averages only observed bigrams", () => { + pair("th", 300); + pair("he", 100); + pair("ab", 150); + expect(WeakSpot.getWord(wordsetReturning("ab", "them"))).toBe("them"); + pair("cd", 250); + expect(WeakSpot.getWord(wordsetReturning("them", "cd"))).toBe("cd"); + }); + + it("retains the capped 50-observation moving average", () => { + for (let i = 0; i < 50; i++) pair("th", 100); + pair("th", 5100); // 5100 / 50 + 100 * 49 / 50 = 200 + pair("ab", 199); + pair("cd", 201); + expect(WeakSpot.getWord(wordsetReturning("ab", "th"))).toBe("th"); + expect(WeakSpot.getWord(wordsetReturning("th", "cd"))).toBe("cd"); + }); + + it("samples 20 words, keeping the first on ties or no observations", () => { + const wordset = wordsetReturning("a", "b"); + expect(WeakSpot.getWord(wordset)).toBe("a"); + expect(wordset.randomWord).toHaveBeenCalledTimes(20); + expect(wordset.randomWord).toHaveBeenCalledWith("normal"); + }); +}); diff --git a/frontend/src/ts/input/handlers/insert-text.ts b/frontend/src/ts/input/handlers/insert-text.ts index 104f381c7822..048ecd6242d4 100644 --- a/frontend/src/ts/input/handlers/insert-text.ts +++ b/frontend/src/ts/input/handlers/insert-text.ts @@ -296,7 +296,7 @@ export async function onInsertText(options: OnInsertTextParams): Promise { }); // this needs to be called after event logging - WeakSpot.updateScore(data, correct); + WeakSpot.updateScore(currentWord); // delete on error // skipped when the input was stopped - nothing was inserted to delete diff --git a/frontend/src/ts/test/events/live-cache.ts b/frontend/src/ts/test/events/live-cache.ts index 52372ab297b6..348a2d22ed59 100644 --- a/frontend/src/ts/test/events/live-cache.ts +++ b/frontend/src/ts/test/events/live-cache.ts @@ -1,5 +1,5 @@ import { roundTo2 } from "@monkeytype/util/numbers"; -import { TestEvent } from "./types"; +import { InputEvent, TestEvent } from "./types"; // Running tallies maintained as events arrive, so live readers don't rescan // the event log. For replay, derive from the event log directly. @@ -7,6 +7,8 @@ const cache = { correctInputs: 0, totalInputs: 0, timerStartMs: null as number | null, + previousInput: null as InputEvent | null, + lastInput: null as InputEvent | null, msSinceLastInputEvent: { value: null as number | null, lastEventMs: null as number | null, @@ -17,12 +19,16 @@ export function resetLiveCache(): void { cache.correctInputs = 0; cache.totalInputs = 0; cache.timerStartMs = null; + cache.previousInput = null; + cache.lastInput = null; cache.msSinceLastInputEvent.value = null; cache.msSinceLastInputEvent.lastEventMs = null; } export function recordEventForCache(event: TestEvent): void { if (event.type === "input") { + cache.previousInput = cache.lastInput; + cache.lastInput = event; if ("correct" in event.data) { cache.totalInputs++; if (event.data.correct) cache.correctInputs++; @@ -48,6 +54,13 @@ export function getLiveCachedMsSinceLastInputEvent(): number | null { return cache.msSinceLastInputEvent.value; } +export function getLiveCachedInputPair(): [ + InputEvent | null, + InputEvent | null, +] { + return [cache.previousInput, cache.lastInput]; +} + export function getLiveCachedTimerStartMs(): number | null { return cache.timerStartMs; } diff --git a/frontend/src/ts/test/weak-spot.ts b/frontend/src/ts/test/weak-spot.ts index 6dd2270fd8da..1eca3e1d5e4c 100644 --- a/frontend/src/ts/test/weak-spot.ts +++ b/frontend/src/ts/test/weak-spot.ts @@ -1,15 +1,18 @@ -import { getLiveCachedMsSinceLastInputEvent } from "./events/live-cache"; +import { + getLiveCachedInputPair, + getLiveCachedMsSinceLastInputEvent, +} from "./events/live-cache"; import { Wordset } from "./wordset"; -// Changes how quickly it 'learns' scores - very roughly the score for a char -// is based on last perCharCount occurrences. Make it smaller to adjust faster. -const perCharCount = 50; +// Changes how quickly it 'learns' scores - very roughly the score for a bigram +// is based on last perBigramCount occurrences. Make it smaller to adjust faster. +const perBigramCount = 50; // Choose the highest scoring word from this many random words. Higher values -// will choose words with more weak letters on average. +// will choose words with more weak bigrams on average. const wordSamples = 20; -// Score penatly (in milliseconds) for getting a letter wrong. +// Score penalty (in milliseconds) for getting the second letter wrong. const incorrectPenalty = 5000; const scores: Record = {}; @@ -23,7 +26,7 @@ class Score { } update(score: number): void { - if (this.count < perCharCount) { + if (this.count < perBigramCount) { this.count++; } const adjustRate = 1.0 / this.count; @@ -32,31 +35,62 @@ class Score { } } -export function updateScore(char: string, isCorrect: boolean): void { +export function updateScore(word: string): void { + const [previous, current] = getLiveCachedInputPair(); const spacing = getLiveCachedMsSinceLastInputEvent(); - if (spacing === null) { + if (previous === null || current === null || spacing === null) { return; } + + const before = previous.data; + const after = current.data; + // Only consecutive, manually typed characters in the same word have a + // meaningful transition time. Deletions and corrections break the pair. + if ( + before.inputType !== "insertText" || + after.inputType !== "insertText" || + before.automatic || + after.automatic || + before.isCompositionEnding || + after.isCompositionEnding || + !before.correct || + before.inputStopped || + before.commitsWord || + before.wordIndex !== after.wordIndex || + before.charIndex + before.data.length !== after.charIndex || + [...before.data].length !== 1 || + [...after.data].length !== 1 || + /\s/u.test(before.data + after.data) + ) { + return; + } + + const char = [...word.slice(after.charIndex)][0]; + if (char === undefined || /\s/u.test(char)) return; + // Penalize the intended pair, not the typo (e.g. th, not tx). + const bigram = before.data + char; let score = spacing; - if (!isCorrect) { + if (!after.correct) { score += incorrectPenalty; } - if (!(char in scores)) { - scores[char] = new Score(); + if (!(bigram in scores)) { + scores[bigram] = new Score(); } - scores[char]?.update(score); + scores[bigram]?.update(score); } function score(word: string): number { let total = 0.0; - let numChars = 0; - for (const c of word) { - if (c in scores) { - total += (scores[c] as Score).average; - numChars++; + let numBigrams = 0; + const chars = [...word]; + for (let i = 1; i < chars.length; i++) { + const bigram = (chars[i - 1] as string) + (chars[i] as string); + if (bigram in scores) { + total += (scores[bigram] as Score).average; + numBigrams++; } } - return numChars === 0 ? 0.0 : total / numChars; + return numBigrams === 0 ? 0.0 : total / numBigrams; } export function getWord(wordset: Wordset): string { diff --git a/packages/funbox/src/list.ts b/packages/funbox/src/list.ts index 866decc7d7e7..bb54ae5d21a0 100644 --- a/packages/funbox/src/list.ts +++ b/packages/funbox/src/list.ts @@ -314,7 +314,7 @@ const list: Record = { name: "wikipedia", }, weakspot: { - description: "Focus on slow and mistyped letters.", + description: "Focus on slow and mistyped letter pairs.", canGetPb: false, difficultyLevel: 0, properties: ["changesWordsFrequency"],