diff --git a/core/linguistics/BUILD.bazel b/core/linguistics/BUILD.bazel index bea67e1..0bd2c5d 100644 --- a/core/linguistics/BUILD.bazel +++ b/core/linguistics/BUILD.bazel @@ -18,3 +18,14 @@ kt_jvm_test( "@maven//:junit_junit", ], ) + +kt_jvm_test( + name = "sentence_scorer_test", + size = "small", + srcs = ["src/test/kotlin/com/anysoftkeyboard/calque/linguistics/SentenceScorerTest.kt"], + test_class = "com.anysoftkeyboard.calque.linguistics.SentenceScorerTest", + deps = [ + ":linguistics", + "@maven//:junit_junit", + ], +) diff --git a/core/linguistics/src/main/kotlin/com/anysoftkeyboard/calque/linguistics/SentenceScorer.kt b/core/linguistics/src/main/kotlin/com/anysoftkeyboard/calque/linguistics/SentenceScorer.kt new file mode 100644 index 0000000..9cfe284 --- /dev/null +++ b/core/linguistics/src/main/kotlin/com/anysoftkeyboard/calque/linguistics/SentenceScorer.kt @@ -0,0 +1,177 @@ +package com.anysoftkeyboard.calque.linguistics + +import java.text.BreakIterator +import java.util.Locale + +/** Difficulty tier for curated reading lessons. */ +enum class SentenceLevel { + LEVEL_0, + LEVEL_1, + LEVEL_2, +} + +/** + * Word properties required for sentence scoring. + * + * @property frequencyRank Subtitle frequency rank (lower number = more common, e.g. 1..1500). + * @property isCognate Whether this word is a transparent cognate with English. + * @property grammar Morphological and grammatical tags (e.g., tense, mood). + */ +data class ScoredWordInfo( + val frequencyRank: Int? = null, + val isCognate: Boolean = false, + val grammar: String? = null, +) + +/** Result of scoring a Spanish sentence for lesson suitability. */ +data class SentenceScoreResult( + val level: SentenceLevel?, + val score: Double, + val wordCount: Int, + val rareWordRatio: Double, + val cognateRatio: Double, + val hasComplexTense: Boolean, +) + +/** + * Sentence readability scorer and level bucketer. + * + * Implements Step 4 from INGESTION.md to evaluate sentence difficulty based on length, vocabulary + * rarity (SUBTLEX-ESP frequency), cognate transparency, and grammatical complexity. + */ +object SentenceScorer { + const val MIN_WORD_COUNT = 4 + const val MAX_WORD_COUNT = 14 + const val RARE_WORD_FREQ_THRESHOLD = 1500 + + const val LEVEL_0_MAX_SCORE = 2.0 + const val LEVEL_1_MAX_SCORE = 4.0 + const val LEVEL_2_MAX_SCORE = 6.5 + + private const val LENGTH_WEIGHT = 0.25 + private const val RARE_RATIO_WEIGHT = 5.0 + private const val COGNATE_RATIO_WEIGHT = 2.5 + private const val COMPLEX_TENSE_PENALTY = 3.0 + + private val COMPLEX_TENSE_KEYWORDS = + listOf("subjunctive", "conditional", "subjuntivo", "condicional") + + /** + * Tokenizes a Spanish sentence into lowercased word tokens using Unicode-aware BreakIterator. + */ + fun tokenizeWords(sentence: String, locale: Locale = Locale("es", "ES")): List { + val iterator = BreakIterator.getWordInstance(locale) + iterator.setText(sentence) + val words = mutableListOf() + + var start = iterator.first() + var end = iterator.next() + while (end != BreakIterator.DONE) { + val token = sentence.substring(start, end).trim() + if (token.isNotEmpty() && token.any { it.isLetter() }) { + words.add(token.lowercase(locale)) + } + start = end + end = iterator.next() + } + return words + } + + /** + * Scores a Spanish sentence using the provided word lookup function. + * + * @param sentence The raw Spanish sentence to evaluate. + * @param wordLookup Function providing [ScoredWordInfo] for a given lowercased word. + * @return [SentenceScoreResult] containing the computed score, metrics, and level (or null if + * discarded). + */ + fun scoreSentence( + sentence: String, + wordLookup: (String) -> ScoredWordInfo?, + ): SentenceScoreResult { + val words = tokenizeWords(sentence) + val wordCount = words.size + + if (wordCount < MIN_WORD_COUNT || wordCount > MAX_WORD_COUNT) { + return SentenceScoreResult( + level = null, + score = 0.0, + wordCount = wordCount, + rareWordRatio = 0.0, + cognateRatio = 0.0, + hasComplexTense = false, + ) + } + + val metrics = calculateMetrics(words, wordLookup) + var score = + (wordCount * LENGTH_WEIGHT) + (metrics.rareWordRatio * RARE_RATIO_WEIGHT) - + (metrics.cognateRatio * COGNATE_RATIO_WEIGHT) + + if (metrics.hasComplexTense) { + score += COMPLEX_TENSE_PENALTY + } + + return SentenceScoreResult( + level = determineLevel(score), + score = score, + wordCount = wordCount, + rareWordRatio = metrics.rareWordRatio, + cognateRatio = metrics.cognateRatio, + hasComplexTense = metrics.hasComplexTense, + ) + } + + private fun calculateMetrics( + words: List, + wordLookup: (String) -> ScoredWordInfo?, + ): IntermediateMetrics { + var rareWordCount = 0 + var cognateCount = 0 + var hasComplexTense = false + + for (word in words) { + val info = wordLookup(word) + if (info == null || isRare(info.frequencyRank)) { + rareWordCount++ + } + if (info?.isCognate == true) { + cognateCount++ + } + if (info?.grammar != null && isComplexTense(info.grammar)) { + hasComplexTense = true + } + } + + return IntermediateMetrics( + rareWordRatio = rareWordCount.toDouble() / words.size, + cognateRatio = cognateCount.toDouble() / words.size, + hasComplexTense = hasComplexTense, + ) + } + + private fun isRare(frequencyRank: Int?): Boolean { + return frequencyRank == null || frequencyRank > RARE_WORD_FREQ_THRESHOLD + } + + private fun determineLevel(score: Double): SentenceLevel? { + return when { + score <= LEVEL_0_MAX_SCORE -> SentenceLevel.LEVEL_0 + score <= LEVEL_1_MAX_SCORE -> SentenceLevel.LEVEL_1 + score <= LEVEL_2_MAX_SCORE -> SentenceLevel.LEVEL_2 + else -> null + } + } + + private fun isComplexTense(grammar: String): Boolean { + return COMPLEX_TENSE_KEYWORDS.any { keyword -> + grammar.contains(keyword, ignoreCase = true) + } + } + + private data class IntermediateMetrics( + val rareWordRatio: Double, + val cognateRatio: Double, + val hasComplexTense: Boolean, + ) +} diff --git a/core/linguistics/src/test/kotlin/com/anysoftkeyboard/calque/linguistics/SentenceScorerTest.kt b/core/linguistics/src/test/kotlin/com/anysoftkeyboard/calque/linguistics/SentenceScorerTest.kt new file mode 100644 index 0000000..3dc374b --- /dev/null +++ b/core/linguistics/src/test/kotlin/com/anysoftkeyboard/calque/linguistics/SentenceScorerTest.kt @@ -0,0 +1,185 @@ +package com.anysoftkeyboard.calque.linguistics + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class SentenceScorerTest { + private val delta = 0.0001 + + @Test + fun tokenizeWords_handlesSpanishPunctuationAndInvertedMarks() { + val sentence = "¡Hola! ¿Cómo estás, amigo mío?" + val tokens = SentenceScorer.tokenizeWords(sentence) + + assertEquals(listOf("hola", "cómo", "estás", "amigo", "mío"), tokens) + } + + @Test + fun scoreSentence_discardsSentencesWithFewerThanFourWords() { + val shortSentence = "Hola, mi amigo." + val result = + SentenceScorer.scoreSentence(shortSentence) { + ScoredWordInfo(frequencyRank = 100, isCognate = false, grammar = null) + } + + assertEquals(3, result.wordCount) + assertNull(result.level) + } + + @Test + fun scoreSentence_discardsSentencesWithMoreThanFourteenWords() { + val longSentence = + "Uno dos tres cuatro cinco seis siete ocho nueve diez once doce trece catorce quince." + val result = + SentenceScorer.scoreSentence(longSentence) { + ScoredWordInfo(frequencyRank = 100, isCognate = false, grammar = null) + } + + assertEquals(15, result.wordCount) + assertNull(result.level) + } + + @Test + fun scoreSentence_assignsLevel0ForSimpleCognateSentences() { + // 4 words, all common (freq rank <= 1500), 2 cognates, no complex tense: + // score = 4 * 0.25 + 0 * 5.0 - (2/4) * 2.5 = 1.0 - 1.25 = -0.25 <= 2.0 -> LEVEL_0 + val sentence = "La música es fantástica." + val dict = + mapOf( + "la" to ScoredWordInfo(frequencyRank = 5, isCognate = false), + "música" to ScoredWordInfo(frequencyRank = 500, isCognate = true), + "es" to ScoredWordInfo(frequencyRank = 10, isCognate = false), + "fantástica" to ScoredWordInfo(frequencyRank = 1200, isCognate = true), + ) + + val result = SentenceScorer.scoreSentence(sentence) { dict[it] } + + assertEquals(4, result.wordCount) + assertEquals(SentenceLevel.LEVEL_0, result.level) + assertEquals(-0.25, result.score, delta) + assertEquals(0.0, result.rareWordRatio, delta) + assertEquals(0.5, result.cognateRatio, delta) + assertFalse(result.hasComplexTense) + } + + @Test + fun scoreSentence_assignsLevel1ForModerateVocabulary() { + // 8 words, 3 rare words (rank > 1500), 1 cognate: + // rare ratio = 3/8 = 0.375 + // cognate ratio = 1/8 = 0.125 + // score = (8 * 0.25) + (0.375 * 5.0) - (0.125 * 2.5) = 2.0 + 1.875 - 0.3125 = 3.5625 <= 4.0 + // -> LEVEL_1 + val words = listOf("uno", "dos", "tres", "cuatro", "cinco", "seis", "siete", "ocho") + val sentence = words.joinToString(" ") + "." + val dict = + mapOf( + "uno" to ScoredWordInfo(frequencyRank = 2000, isCognate = false), + "dos" to ScoredWordInfo(frequencyRank = 3000, isCognate = false), + "tres" to ScoredWordInfo(frequencyRank = 4000, isCognate = false), + "cuatro" to ScoredWordInfo(frequencyRank = 100, isCognate = true), + "cinco" to ScoredWordInfo(frequencyRank = 200, isCognate = false), + "seis" to ScoredWordInfo(frequencyRank = 300, isCognate = false), + "siete" to ScoredWordInfo(frequencyRank = 400, isCognate = false), + "ocho" to ScoredWordInfo(frequencyRank = 500, isCognate = false), + ) + + val result = SentenceScorer.scoreSentence(sentence) { dict[it] } + + assertEquals(8, result.wordCount) + assertEquals(SentenceLevel.LEVEL_1, result.level) + assertEquals(3.5625, result.score, delta) + } + + @Test + fun scoreSentence_assignsLevel2ForAdvancedVocabularyWithoutComplexTense() { + // 10 words, 5 rare (0.5), 0 cognates: + // score = (10 * 0.25) + (0.5 * 5.0) - 0 = 2.5 + 2.5 = 5.0 <= 6.5 -> LEVEL_2 + val words = (1..10).map { "palabra$it" } + val sentence = words.joinToString(" ") + "." + val dict = + words + .mapIndexed { index, word -> + word to + ScoredWordInfo( + frequencyRank = if (index < 5) 5000 else 100, + isCognate = false, + ) + } + .toMap() + + val result = SentenceScorer.scoreSentence(sentence) { dict[it] } + + assertEquals(10, result.wordCount) + assertEquals(SentenceLevel.LEVEL_2, result.level) + assertEquals(5.0, result.score, delta) + } + + @Test + fun scoreSentence_addsPenaltyForSubjunctiveOrConditionalTense() { + // 5 words, 0 rare, 0 cognates: + // base score = 5 * 0.25 = 1.25 (would be LEVEL_0) + // with subjunctive: + 3.0 = 4.25 -> LEVEL_2 (since 4.0 < 4.25 <= 6.5) + val sentence = "Espero que vengas pronto mañana." + val dict = + mapOf( + "espero" to ScoredWordInfo(frequencyRank = 50, isCognate = false), + "que" to ScoredWordInfo(frequencyRank = 2, isCognate = false), + "vengas" to + ScoredWordInfo( + frequencyRank = 100, + isCognate = false, + grammar = "present subjunctive", + ), + "pronto" to ScoredWordInfo(frequencyRank = 80, isCognate = false), + "mañana" to ScoredWordInfo(frequencyRank = 60, isCognate = false), + ) + + val result = SentenceScorer.scoreSentence(sentence) { dict[it] } + + assertEquals(5, result.wordCount) + assertTrue(result.hasComplexTense) + assertEquals(4.25, result.score, delta) + assertEquals(SentenceLevel.LEVEL_2, result.level) + } + + @Test + fun scoreSentence_discardsSentencesWithScoreExceedingMaxLevel() { + // 14 words, all rare, no cognates, with complex tense: + // score = 14 * 0.25 + 1.0 * 5.0 - 0 + 3.0 = 3.5 + 5.0 + 3.0 = 11.5 > 6.5 -> DISCARD + val words = (1..14).map { "rara$it" } + val sentence = words.joinToString(" ") + "." + val dict = + words + .mapIndexed { index, word -> + word to + ScoredWordInfo( + frequencyRank = 10000, + isCognate = false, + grammar = if (index == 0) "conditional" else null, + ) + } + .toMap() + + val result = SentenceScorer.scoreSentence(sentence) { dict[it] } + + assertNull(result.level) + assertEquals(11.5, result.score, delta) + } + + @Test + fun scoreSentence_treatsUnrecognizedWordsAsRare() { + // Words missing from lookup should default to rare + val sentence = "Palabra desconocida misteriosa extraña." + val result = SentenceScorer.scoreSentence(sentence) { null } + + assertEquals(4, result.wordCount) + assertEquals(1.0, result.rareWordRatio, delta) + assertEquals(0.0, result.cognateRatio, delta) + // score = 4 * 0.25 + 1.0 * 5.0 = 6.0 <= 6.5 -> LEVEL_2 + assertEquals(6.0, result.score, delta) + assertEquals(SentenceLevel.LEVEL_2, result.level) + } +}