From aa09132bfc5e3f85457198ba47ce5e1637863b69 Mon Sep 17 00:00:00 2001 From: Menny Even Danan Date: Thu, 15 Oct 2026 18:46:25 +0000 Subject: [PATCH] Implement Algorithmic CognateDetector with unit tests Add pure-JVM CognateDetector in //core/linguistics implementing Step 2 of INGESTION.md. Features accent stripping, gloss candidate extraction, suffix transformation mappings, and stem Levenshtein distance matching. Includes comprehensive JUnit unit tests. --- .editorconfig | 1 + core/linguistics/BUILD.bazel | 19 ++ .../calque/linguistics/CognateDetector.kt | 176 ++++++++++++++++ .../calque/linguistics/CognateDetectorTest.kt | 188 ++++++++++++++++++ 4 files changed, 384 insertions(+) create mode 100644 core/linguistics/src/main/kotlin/com/anysoftkeyboard/calque/linguistics/CognateDetector.kt create mode 100644 core/linguistics/src/test/kotlin/com/anysoftkeyboard/calque/linguistics/CognateDetectorTest.kt diff --git a/.editorconfig b/.editorconfig index f1f636d..f68d576 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,6 +11,7 @@ trim_trailing_whitespace = true [*.{kt,kts}] indent_size = 4 max_line_length = 120 +ktlint_code_style = intellij_idea ij_kotlin_allow_trailing_comma = true ij_kotlin_allow_trailing_comma_on_call_site = true diff --git a/core/linguistics/BUILD.bazel b/core/linguistics/BUILD.bazel index ffd0fb0..bea67e1 100644 --- a/core/linguistics/BUILD.bazel +++ b/core/linguistics/BUILD.bazel @@ -1 +1,20 @@ +load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library", "kt_jvm_test") + package(default_visibility = ["//visibility:public"]) + +kt_jvm_library( + name = "linguistics", + srcs = glob(["src/main/kotlin/**/*.kt"]), + visibility = ["//visibility:public"], +) + +kt_jvm_test( + name = "cognate_detector_test", + size = "small", + srcs = ["src/test/kotlin/com/anysoftkeyboard/calque/linguistics/CognateDetectorTest.kt"], + test_class = "com.anysoftkeyboard.calque.linguistics.CognateDetectorTest", + deps = [ + ":linguistics", + "@maven//:junit_junit", + ], +) diff --git a/core/linguistics/src/main/kotlin/com/anysoftkeyboard/calque/linguistics/CognateDetector.kt b/core/linguistics/src/main/kotlin/com/anysoftkeyboard/calque/linguistics/CognateDetector.kt new file mode 100644 index 0000000..733d821 --- /dev/null +++ b/core/linguistics/src/main/kotlin/com/anysoftkeyboard/calque/linguistics/CognateDetector.kt @@ -0,0 +1,176 @@ +package com.anysoftkeyboard.calque.linguistics + +import java.text.Normalizer +import java.util.Locale +import kotlin.math.min + +/** + * Algorithmic cognate detector that determines whether a Spanish word and its English definition + * share a transparent etymological root. + * + * Implements Step 2 from INGESTION.md using suffix transformation mapping, accent stripping, and + * Levenshtein distance on word stems. + */ +object CognateDetector { + /** Suffix conversion pairs between Spanish and English. */ + private val SUFFIX_MAP: List> = + listOf( + "ción" to "tion", + "sión" to "sion", + "dad" to "ty", + "mente" to "ly", + "oso" to "ous", + "osa" to "ous", + "ico" to "ic", + "ica" to "ic", + "al" to "al", + "cto" to "ct", + "cta" to "ct", + "ncia" to "nce", + "ble" to "ble", + "ario" to "ary", + "aria" to "ary", + "ivo" to "ive", + "iva" to "ive", + "ido" to "id", + "ida" to "id", + "ura" to "ure", + "or" to "or", + "ar" to "ate", + "ar" to "e", + "er" to "e", + "ir" to "e", + "izar" to "ize", + "izar" to "ise", + ) + + private val PARENTHETICAL_REGEX = Regex("""\([^)]*\)|\[[^\]]*\]""") + private val GLOSS_DELIMITERS_REGEX = Regex("""[,;/|\n]+""") + private val LEADING_PARTICLES_REGEX = Regex("""^(?:to|a|an|the)\s+""") + private val WORD_TOKEN_REGEX = Regex("""[\p{L}'-]+""") + private val COMBINING_MARKS_REGEX = Regex("""\p{M}+""") + + /** + * Detects whether [spanishWord] and [englishGloss] are cognates. + * + * @param spanishWord The Spanish word or surface token (e.g., "comunicación", "natural"). + * @param englishGloss The English definition or gloss (e.g., "communication", "to decide", + * "rapidly, quickly"). + * @return `true` if a transparent cognate relationship is detected, `false` otherwise. + */ + fun detectCognate(spanishWord: String, englishGloss: String): Boolean { + val es = normalize(spanishWord) + if (es.length <= 2) return false + + val candidates = extractCandidateWords(englishGloss) + if (candidates.isEmpty()) return false + + return candidates.any { en -> matchesCognate(es, en) } + } + + /** Checks if a normalized Spanish word and a candidate English word are cognates. */ + fun matchesCognate(spanishWord: String, englishWord: String): Boolean { + val es = normalize(spanishWord) + val en = normalize(englishWord) + + if (es.length <= 2 || en.length <= 2) return false + + // Direct identical match for words > 3 characters (e.g. "natural" == "natural") + if (es.length > 3 && es == en) { + return true + } + + // Direct match after accent normalization (e.g. "café" -> "cafe") + val esUnaccented = stripAccents(es) + if (esUnaccented.length > 3 && esUnaccented == en) { + return true + } + + // Suffix mapping and stem comparison + for ((esSuffix, enSuffix) in SUFFIX_MAP) { + if (es.endsWith(esSuffix) && en.endsWith(enSuffix)) { + val esStem = stripAccents(es.removeSuffix(esSuffix)) + val enStem = en.removeSuffix(enSuffix) + + // Exact match on short stems (e.g. "nación" -> "na" == "nation" -> "na") + if (esStem == enStem && esStem.length >= 2) { + return true + } + + // Levenshtein distance <= 1 for stems of length >= 3 + if (esStem.length >= 3 && levenshteinDistance(esStem, enStem) <= 1) { + return true + } + } + } + + return false + } + + /** Normalizes a word by lowercasing and trimming whitespace and non-letter edge characters. */ + fun normalize(word: String): String { + return word.trim().lowercase(Locale.ROOT).trim { !it.isLetter() } + } + + /** Strips combining diacritical marks (e.g., accents, tildes) from a string. */ + fun stripAccents(input: String): String { + val decomposed = Normalizer.normalize(input, Normalizer.Form.NFD) + return decomposed.replace(COMBINING_MARKS_REGEX, "") + } + + /** Extracts the primary word from an English gloss. */ + fun extractPrimaryWord(englishGloss: String): String { + return extractCandidateWords(englishGloss).firstOrNull().orEmpty() + } + + /** + * Extracts candidate English words from a gloss, stripping parenthetical notes, leading + * grammatical markers ("to ", "a ", "an ", "the "), and splitting synonyms. + */ + fun extractCandidateWords(englishGloss: String): List { + val cleanGloss = englishGloss.replace(PARENTHETICAL_REGEX, " ") + val parts = cleanGloss.split(GLOSS_DELIMITERS_REGEX) + val result = mutableListOf() + + for (part in parts) { + val trimmed = part.trim().lowercase(Locale.ROOT) + val withoutParticle = trimmed.replace(LEADING_PARTICLES_REGEX, "") + val words = WORD_TOKEN_REGEX.findAll(withoutParticle).map { it.value } + for (word in words) { + val normalized = normalize(word) + if (normalized.length >= 2 && normalized !in result) { + result.add(normalized) + } + } + } + return result + } + + /** Computes the Levenshtein distance between two strings using an O(min(N, M)) space buffer. */ + fun levenshteinDistance(s1: String, s2: String): Int { + if (s1 == s2) return 0 + if (s1.isEmpty()) return s2.length + if (s2.isEmpty()) return s1.length + + val (shorter, longer) = if (s1.length < s2.length) s1 to s2 else s2 to s1 + var prev = IntArray(shorter.length + 1) { it } + var curr = IntArray(shorter.length + 1) + + for (i in 1..longer.length) { + curr[0] = i + val ch2 = longer[i - 1] + for (j in 1..shorter.length) { + val cost = if (ch2 == shorter[j - 1]) 0 else 1 + curr[j] = + min( + curr[j - 1] + 1, + min(prev[j] + 1, prev[j - 1] + cost), + ) + } + val temp = prev + prev = curr + curr = temp + } + return prev[shorter.length] + } +} diff --git a/core/linguistics/src/test/kotlin/com/anysoftkeyboard/calque/linguistics/CognateDetectorTest.kt b/core/linguistics/src/test/kotlin/com/anysoftkeyboard/calque/linguistics/CognateDetectorTest.kt new file mode 100644 index 0000000..9c724ee --- /dev/null +++ b/core/linguistics/src/test/kotlin/com/anysoftkeyboard/calque/linguistics/CognateDetectorTest.kt @@ -0,0 +1,188 @@ +package com.anysoftkeyboard.calque.linguistics + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class CognateDetectorTest { + + @Test + fun testExactCognates() { + assertTrue(CognateDetector.detectCognate("natural", "natural")) + assertTrue(CognateDetector.detectCognate("animal", "animal")) + assertTrue(CognateDetector.detectCognate("hotel", "hotel")) + assertTrue(CognateDetector.detectCognate("radio", "radio")) + assertTrue(CognateDetector.detectCognate("color", "color")) + assertTrue(CognateDetector.detectCognate("drama", "drama")) + assertTrue(CognateDetector.detectCognate("error", "error")) + } + + @Test + fun testAccentNormalizedExactMatches() { + assertTrue(CognateDetector.detectCognate("café", "cafe")) + assertTrue(CognateDetector.detectCognate("menú", "menu")) + } + + @Test + fun testCionToTion() { + assertTrue(CognateDetector.detectCognate("comunicación", "communication")) + assertTrue(CognateDetector.detectCognate("información", "information")) + assertTrue(CognateDetector.detectCognate("nación", "nation")) + assertTrue(CognateDetector.detectCognate("estación", "station")) + assertTrue(CognateDetector.detectCognate("educación", "education")) + } + + @Test + fun testSionToSion() { + assertTrue(CognateDetector.detectCognate("decisión", "decision")) + assertTrue(CognateDetector.detectCognate("televisión", "television")) + assertTrue(CognateDetector.detectCognate("conclusión", "conclusion")) + assertTrue(CognateDetector.detectCognate("profesión", "profession")) + } + + @Test + fun testDadToTy() { + assertTrue(CognateDetector.detectCognate("actividad", "activity")) + assertTrue(CognateDetector.detectCognate("universidad", "university")) + assertTrue(CognateDetector.detectCognate("ciudad", "city")) + assertTrue(CognateDetector.detectCognate("comunidad", "community")) + assertTrue(CognateDetector.detectCognate("sociedad", "society")) + } + + @Test + fun testMenteToLy() { + assertTrue(CognateDetector.detectCognate("naturalmente", "naturally")) + assertTrue(CognateDetector.detectCognate("rápidamente", "rapidly")) + assertTrue(CognateDetector.detectCognate("totalmente", "totally")) + } + + @Test + fun testOsoToOus() { + assertTrue(CognateDetector.detectCognate("curioso", "curious")) + assertTrue(CognateDetector.detectCognate("famosa", "famous")) + assertTrue(CognateDetector.detectCognate("religioso", "religious")) + assertTrue(CognateDetector.detectCognate("nervioso", "nervous")) + } + + @Test + fun testIcoToIc() { + assertTrue(CognateDetector.detectCognate("básico", "basic")) + assertTrue(CognateDetector.detectCognate("romántico", "romantic")) + assertTrue(CognateDetector.detectCognate("música", "music")) + assertTrue(CognateDetector.detectCognate("dramático", "dramatic")) + } + + @Test + fun testAlToAl() { + assertTrue(CognateDetector.detectCognate("oficial", "official")) + assertTrue(CognateDetector.detectCognate("personal", "personal")) + assertTrue(CognateDetector.detectCognate("central", "central")) + } + + @Test + fun testCtoToCt() { + assertTrue(CognateDetector.detectCognate("perfecto", "perfect")) + assertTrue(CognateDetector.detectCognate("exacto", "exact")) + assertTrue(CognateDetector.detectCognate("directa", "direct")) + assertTrue(CognateDetector.detectCognate("contacto", "contact")) + } + + @Test + fun testNciaToNce() { + assertTrue(CognateDetector.detectCognate("importancia", "importance")) + assertTrue(CognateDetector.detectCognate("diferencia", "difference")) + assertTrue(CognateDetector.detectCognate("distancia", "distance")) + assertTrue(CognateDetector.detectCognate("inteligencia", "intelligence")) + } + + @Test + fun testBleToBle() { + assertTrue(CognateDetector.detectCognate("posible", "possible")) + assertTrue(CognateDetector.detectCognate("terrible", "terrible")) + } + + @Test + fun testArioToAry() { + assertTrue(CognateDetector.detectCognate("ordinario", "ordinary")) + assertTrue(CognateDetector.detectCognate("necesaria", "necessary")) + } + + @Test + fun testIvoToIve() { + assertTrue(CognateDetector.detectCognate("activo", "active")) + assertTrue(CognateDetector.detectCognate("creativa", "creative")) + } + + @Test + fun testOrToOr() { + assertTrue(CognateDetector.detectCognate("actor", "actor")) + assertTrue(CognateDetector.detectCognate("profesor", "professor")) + } + + @Test + fun testVerbCognates() { + assertTrue(CognateDetector.detectCognate("comunicar", "to communicate")) + assertTrue(CognateDetector.detectCognate("participar", "to participate")) + assertTrue(CognateDetector.detectCognate("separar", "to separate")) + assertTrue(CognateDetector.detectCognate("decidir", "to decide")) + assertTrue(CognateDetector.detectCognate("preparar", "to prepare")) + assertTrue(CognateDetector.detectCognate("organizar", "to organize")) + } + + @Test + fun testComplexEnglishGlossExtraction() { + // Leading infinitive particle with verb + assertTrue(CognateDetector.detectCognate("comunicar", "to communicate")) + // Leading article + assertTrue(CognateDetector.detectCognate("doctor", "a doctor")) + // Parenthetical qualifier + assertTrue(CognateDetector.detectCognate("natural", "(anatomy, biology) natural")) + // Comma-separated synonyms where cognate is later + assertTrue(CognateDetector.detectCognate("rápido", "quick, rapid")) + // Semicolon-separated definitions + assertTrue(CognateDetector.detectCognate("exacto", "precise; exact")) + } + + @Test + fun testNonCognates() { + assertFalse(CognateDetector.detectCognate("perro", "dog")) + assertFalse(CognateDetector.detectCognate("gato", "cat")) + assertFalse(CognateDetector.detectCognate("mesa", "table")) + assertFalse(CognateDetector.detectCognate("agua", "water")) + assertFalse(CognateDetector.detectCognate("hacer", "to do")) + assertFalse(CognateDetector.detectCognate("comer", "to eat")) + assertFalse(CognateDetector.detectCognate("ir", "to go")) + assertFalse(CognateDetector.detectCognate("libro", "book")) + assertFalse(CognateDetector.detectCognate("mano", "hand")) + assertFalse(CognateDetector.detectCognate("casa", "house")) + } + + @Test + fun testEdgeCases() { + assertFalse(CognateDetector.detectCognate("", "")) + assertFalse(CognateDetector.detectCognate(" ", " ")) + assertFalse(CognateDetector.detectCognate("a", "to")) + assertFalse(CognateDetector.detectCognate("de", "of")) + assertFalse(CognateDetector.detectCognate("el", "the")) + assertFalse(CognateDetector.detectCognate("la", "the")) + } + + @Test + fun testLevenshteinDistance() { + assertEquals(0, CognateDetector.levenshteinDistance("", "")) + assertEquals(3, CognateDetector.levenshteinDistance("abc", "")) + assertEquals(3, CognateDetector.levenshteinDistance("", "abc")) + assertEquals(0, CognateDetector.levenshteinDistance("test", "test")) + assertEquals(1, CognateDetector.levenshteinDistance("ofici", "offici")) + assertEquals(3, CognateDetector.levenshteinDistance("kitten", "sitting")) + } + + @Test + fun testStripAccents() { + assertEquals("comunicacion", CognateDetector.stripAccents("comunicación")) + assertEquals("facilmente", CognateDetector.stripAccents("fácilmente")) + assertEquals("rapido", CognateDetector.stripAccents("rápido")) + assertEquals("Espana", CognateDetector.stripAccents("España")) + } +}