Implement Algorithmic CognateDetector with unit tests

.editorconfig

@@ -11,6 +11,7 @@ trim_trailing_whitespace = true
 [*.{kt,kts}]
 indent_size = 4
 max_line_length = 120
+ktlint_code_style = intellij_idea
 ij_kotlin_allow_trailing_comma = true
 ij_kotlin_allow_trailing_comma_on_call_site = true
 

core/linguistics/BUILD.bazel

@@ -1 +1,20 @@
+load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library", "kt_jvm_test")
+
 package(default_visibility = ["//visibility:public"])
+
+kt_jvm_library(
+    name = "linguistics",
+    srcs = glob(["src/main/kotlin/**/*.kt"]),
+    visibility = ["//visibility:public"],
+)
+
+kt_jvm_test(
+    name = "cognate_detector_test",
+    size = "small",
+    srcs = ["src/test/kotlin/com/anysoftkeyboard/calque/linguistics/CognateDetectorTest.kt"],
+    test_class = "com.anysoftkeyboard.calque.linguistics.CognateDetectorTest",
+    deps = [
+        ":linguistics",
+        "@maven//:junit_junit",
+    ],
+)

core/linguistics/src/main/kotlin/com/anysoftkeyboard/calque/linguistics/CognateDetector.kt

@@ -0,0 +1,176 @@
+package com.anysoftkeyboard.calque.linguistics
+
+import java.text.Normalizer
+import java.util.Locale
+import kotlin.math.min
+
+/**
+ * Algorithmic cognate detector that determines whether a Spanish word and its English definition
+ * share a transparent etymological root.
+ *
+ * Implements Step 2 from INGESTION.md using suffix transformation mapping, accent stripping, and
+ * Levenshtein distance on word stems.
+ */
+object CognateDetector {
+    /** Suffix conversion pairs between Spanish and English. */
+    private val SUFFIX_MAP: List<Pair<String, String>> =
+        listOf(
+            "ción" to "tion",
+            "sión" to "sion",
+            "dad" to "ty",
+            "mente" to "ly",
+            "oso" to "ous",
+            "osa" to "ous",
+            "ico" to "ic",
+            "ica" to "ic",
+            "al" to "al",
+            "cto" to "ct",
+            "cta" to "ct",
+            "ncia" to "nce",
+            "ble" to "ble",
+            "ario" to "ary",
+            "aria" to "ary",
+            "ivo" to "ive",
+            "iva" to "ive",
+            "ido" to "id",
+            "ida" to "id",
+            "ura" to "ure",
+            "or" to "or",
+            "ar" to "ate",
+            "ar" to "e",
+            "er" to "e",
+            "ir" to "e",
+            "izar" to "ize",
+            "izar" to "ise",
+        )
+
+    private val PARENTHETICAL_REGEX = Regex("""\([^)]*\)|\[[^\]]*\]""")
+    private val GLOSS_DELIMITERS_REGEX = Regex("""[,;/|\n]+""")
+    private val LEADING_PARTICLES_REGEX = Regex("""^(?:to|a|an|the)\s+""")
+    private val WORD_TOKEN_REGEX = Regex("""[\p{L}'-]+""")
+    private val COMBINING_MARKS_REGEX = Regex("""\p{M}+""")
+
+    /**
+     * Detects whether [spanishWord] and [englishGloss] are cognates.
+     *
+     * @param spanishWord The Spanish word or surface token (e.g., "comunicación", "natural").
+     * @param englishGloss The English definition or gloss (e.g., "communication", "to decide",
+     *   "rapidly, quickly").
+     * @return `true` if a transparent cognate relationship is detected, `false` otherwise.
+     */
+    fun detectCognate(spanishWord: String, englishGloss: String): Boolean {
+        val es = normalize(spanishWord)
+        if (es.length <= 2) return false
+
+        val candidates = extractCandidateWords(englishGloss)
+        if (candidates.isEmpty()) return false
+
+        return candidates.any { en -> matchesCognate(es, en) }
+    }
+
+    /** Checks if a normalized Spanish word and a candidate English word are cognates. */
+    fun matchesCognate(spanishWord: String, englishWord: String): Boolean {
+        val es = normalize(spanishWord)
+        val en = normalize(englishWord)
+
+        if (es.length <= 2 || en.length <= 2) return false
+
+        // Direct identical match for words > 3 characters (e.g. "natural" == "natural")
+        if (es.length > 3 && es == en) {
+            return true
+        }
+
+        // Direct match after accent normalization (e.g. "café" -> "cafe")
+        val esUnaccented = stripAccents(es)
+        if (esUnaccented.length > 3 && esUnaccented == en) {
+            return true
+        }
+
+        // Suffix mapping and stem comparison
+        for ((esSuffix, enSuffix) in SUFFIX_MAP) {
+            if (es.endsWith(esSuffix) && en.endsWith(enSuffix)) {
+                val esStem = stripAccents(es.removeSuffix(esSuffix))
+                val enStem = en.removeSuffix(enSuffix)
+
+                // Exact match on short stems (e.g. "nación" -> "na" == "nation" -> "na")
+                if (esStem == enStem && esStem.length >= 2) {
+                    return true
+                }
+
+                // Levenshtein distance <= 1 for stems of length >= 3
+                if (esStem.length >= 3 && levenshteinDistance(esStem, enStem) <= 1) {
+                    return true
+                }
+            }
+        }
+
+        return false
+    }
+
+    /** Normalizes a word by lowercasing and trimming whitespace and non-letter edge characters. */
+    fun normalize(word: String): String {
+        return word.trim().lowercase(Locale.ROOT).trim { !it.isLetter() }
+    }
+
+    /** Strips combining diacritical marks (e.g., accents, tildes) from a string. */
+    fun stripAccents(input: String): String {
+        val decomposed = Normalizer.normalize(input, Normalizer.Form.NFD)
+        return decomposed.replace(COMBINING_MARKS_REGEX, "")
+    }
+
+    /** Extracts the primary word from an English gloss. */
+    fun extractPrimaryWord(englishGloss: String): String {
+        return extractCandidateWords(englishGloss).firstOrNull().orEmpty()
+    }
+
+    /**
+     * Extracts candidate English words from a gloss, stripping parenthetical notes, leading
+     * grammatical markers ("to ", "a ", "an ", "the "), and splitting synonyms.
+     */
+    fun extractCandidateWords(englishGloss: String): List<String> {
+        val cleanGloss = englishGloss.replace(PARENTHETICAL_REGEX, " ")
+        val parts = cleanGloss.split(GLOSS_DELIMITERS_REGEX)
+        val result = mutableListOf<String>()
+
+        for (part in parts) {
+            val trimmed = part.trim().lowercase(Locale.ROOT)
+            val withoutParticle = trimmed.replace(LEADING_PARTICLES_REGEX, "")
+            val words = WORD_TOKEN_REGEX.findAll(withoutParticle).map { it.value }
+            for (word in words) {
+                val normalized = normalize(word)
+                if (normalized.length >= 2 && normalized !in result) {
+                    result.add(normalized)
+                }
+            }
+        }
+        return result
+    }
+
+    /** Computes the Levenshtein distance between two strings using an O(min(N, M)) space buffer. */
+    fun levenshteinDistance(s1: String, s2: String): Int {
+        if (s1 == s2) return 0
+        if (s1.isEmpty()) return s2.length
+        if (s2.isEmpty()) return s1.length
+
+        val (shorter, longer) = if (s1.length < s2.length) s1 to s2 else s2 to s1
+        var prev = IntArray(shorter.length + 1) { it }
+        var curr = IntArray(shorter.length + 1)
+
+        for (i in 1..longer.length) {
+            curr[0] = i
+            val ch2 = longer[i - 1]
+            for (j in 1..shorter.length) {
+                val cost = if (ch2 == shorter[j - 1]) 0 else 1
+                curr[j] =
+                    min(
+                        curr[j - 1] + 1,
+                        min(prev[j] + 1, prev[j - 1] + cost),
+                    )
+            }
+            val temp = prev
+            prev = curr
+            curr = temp
+        }
+        return prev[shorter.length]
+    }
+}

core/linguistics/src/test/kotlin/com/anysoftkeyboard/calque/linguistics/CognateDetectorTest.kt

@@ -0,0 +1,188 @@
+package com.anysoftkeyboard.calque.linguistics
+
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class CognateDetectorTest {
+
+    @Test
+    fun testExactCognates() {
+        assertTrue(CognateDetector.detectCognate("natural", "natural"))
+        assertTrue(CognateDetector.detectCognate("animal", "animal"))
+        assertTrue(CognateDetector.detectCognate("hotel", "hotel"))
+        assertTrue(CognateDetector.detectCognate("radio", "radio"))
+        assertTrue(CognateDetector.detectCognate("color", "color"))
+        assertTrue(CognateDetector.detectCognate("drama", "drama"))
+        assertTrue(CognateDetector.detectCognate("error", "error"))
+    }
+
+    @Test
+    fun testAccentNormalizedExactMatches() {
+        assertTrue(CognateDetector.detectCognate("café", "cafe"))
+        assertTrue(CognateDetector.detectCognate("menú", "menu"))
+    }
+
+    @Test
+    fun testCionToTion() {
+        assertTrue(CognateDetector.detectCognate("comunicación", "communication"))
+        assertTrue(CognateDetector.detectCognate("información", "information"))
+        assertTrue(CognateDetector.detectCognate("nación", "nation"))
+        assertTrue(CognateDetector.detectCognate("estación", "station"))
+        assertTrue(CognateDetector.detectCognate("educación", "education"))
+    }
+
+    @Test
+    fun testSionToSion() {
+        assertTrue(CognateDetector.detectCognate("decisión", "decision"))
+        assertTrue(CognateDetector.detectCognate("televisión", "television"))
+        assertTrue(CognateDetector.detectCognate("conclusión", "conclusion"))
+        assertTrue(CognateDetector.detectCognate("profesión", "profession"))
+    }
+
+    @Test
+    fun testDadToTy() {
+        assertTrue(CognateDetector.detectCognate("actividad", "activity"))
+        assertTrue(CognateDetector.detectCognate("universidad", "university"))
+        assertTrue(CognateDetector.detectCognate("ciudad", "city"))
+        assertTrue(CognateDetector.detectCognate("comunidad", "community"))
+        assertTrue(CognateDetector.detectCognate("sociedad", "society"))
+    }
+
+    @Test
+    fun testMenteToLy() {
+        assertTrue(CognateDetector.detectCognate("naturalmente", "naturally"))
+        assertTrue(CognateDetector.detectCognate("rápidamente", "rapidly"))
+        assertTrue(CognateDetector.detectCognate("totalmente", "totally"))
+    }
+
+    @Test
+    fun testOsoToOus() {
+        assertTrue(CognateDetector.detectCognate("curioso", "curious"))
+        assertTrue(CognateDetector.detectCognate("famosa", "famous"))
+        assertTrue(CognateDetector.detectCognate("religioso", "religious"))
+        assertTrue(CognateDetector.detectCognate("nervioso", "nervous"))
+    }
+
+    @Test
+    fun testIcoToIc() {
+        assertTrue(CognateDetector.detectCognate("básico", "basic"))
+        assertTrue(CognateDetector.detectCognate("romántico", "romantic"))
+        assertTrue(CognateDetector.detectCognate("música", "music"))
+        assertTrue(CognateDetector.detectCognate("dramático", "dramatic"))
+    }
+
+    @Test
+    fun testAlToAl() {
+        assertTrue(CognateDetector.detectCognate("oficial", "official"))
+        assertTrue(CognateDetector.detectCognate("personal", "personal"))
+        assertTrue(CognateDetector.detectCognate("central", "central"))
+    }
+
+    @Test
+    fun testCtoToCt() {
+        assertTrue(CognateDetector.detectCognate("perfecto", "perfect"))
+        assertTrue(CognateDetector.detectCognate("exacto", "exact"))
+        assertTrue(CognateDetector.detectCognate("directa", "direct"))
+        assertTrue(CognateDetector.detectCognate("contacto", "contact"))
+    }
+
+    @Test
+    fun testNciaToNce() {
+        assertTrue(CognateDetector.detectCognate("importancia", "importance"))
+        assertTrue(CognateDetector.detectCognate("diferencia", "difference"))
+        assertTrue(CognateDetector.detectCognate("distancia", "distance"))
+        assertTrue(CognateDetector.detectCognate("inteligencia", "intelligence"))
+    }
+
+    @Test
+    fun testBleToBle() {
+        assertTrue(CognateDetector.detectCognate("posible", "possible"))
+        assertTrue(CognateDetector.detectCognate("terrible", "terrible"))
+    }
+
+    @Test
+    fun testArioToAry() {
+        assertTrue(CognateDetector.detectCognate("ordinario", "ordinary"))
+        assertTrue(CognateDetector.detectCognate("necesaria", "necessary"))
+    }
+
+    @Test
+    fun testIvoToIve() {
+        assertTrue(CognateDetector.detectCognate("activo", "active"))
+        assertTrue(CognateDetector.detectCognate("creativa", "creative"))
+    }
+
+    @Test
+    fun testOrToOr() {
+        assertTrue(CognateDetector.detectCognate("actor", "actor"))
+        assertTrue(CognateDetector.detectCognate("profesor", "professor"))
+    }
+
+    @Test
+    fun testVerbCognates() {
+        assertTrue(CognateDetector.detectCognate("comunicar", "to communicate"))
+        assertTrue(CognateDetector.detectCognate("participar", "to participate"))
+        assertTrue(CognateDetector.detectCognate("separar", "to separate"))
+        assertTrue(CognateDetector.detectCognate("decidir", "to decide"))
+        assertTrue(CognateDetector.detectCognate("preparar", "to prepare"))
+        assertTrue(CognateDetector.detectCognate("organizar", "to organize"))
+    }
+
+    @Test
+    fun testComplexEnglishGlossExtraction() {
+        // Leading infinitive particle with verb
+        assertTrue(CognateDetector.detectCognate("comunicar", "to communicate"))
+        // Leading article
+        assertTrue(CognateDetector.detectCognate("doctor", "a doctor"))
+        // Parenthetical qualifier
+        assertTrue(CognateDetector.detectCognate("natural", "(anatomy, biology) natural"))
+        // Comma-separated synonyms where cognate is later
+        assertTrue(CognateDetector.detectCognate("rápido", "quick, rapid"))
+        // Semicolon-separated definitions
+        assertTrue(CognateDetector.detectCognate("exacto", "precise; exact"))
+    }
+
+    @Test
+    fun testNonCognates() {
+        assertFalse(CognateDetector.detectCognate("perro", "dog"))
+        assertFalse(CognateDetector.detectCognate("gato", "cat"))
+        assertFalse(CognateDetector.detectCognate("mesa", "table"))
+        assertFalse(CognateDetector.detectCognate("agua", "water"))
+        assertFalse(CognateDetector.detectCognate("hacer", "to do"))
+        assertFalse(CognateDetector.detectCognate("comer", "to eat"))
+        assertFalse(CognateDetector.detectCognate("ir", "to go"))
+        assertFalse(CognateDetector.detectCognate("libro", "book"))
+        assertFalse(CognateDetector.detectCognate("mano", "hand"))
+        assertFalse(CognateDetector.detectCognate("casa", "house"))
+    }
+
+    @Test
+    fun testEdgeCases() {
+        assertFalse(CognateDetector.detectCognate("", ""))
+        assertFalse(CognateDetector.detectCognate("   ", "   "))
+        assertFalse(CognateDetector.detectCognate("a", "to"))
+        assertFalse(CognateDetector.detectCognate("de", "of"))
+        assertFalse(CognateDetector.detectCognate("el", "the"))
+        assertFalse(CognateDetector.detectCognate("la", "the"))
+    }
+
+    @Test
+    fun testLevenshteinDistance() {
+        assertEquals(0, CognateDetector.levenshteinDistance("", ""))
+        assertEquals(3, CognateDetector.levenshteinDistance("abc", ""))
+        assertEquals(3, CognateDetector.levenshteinDistance("", "abc"))
+        assertEquals(0, CognateDetector.levenshteinDistance("test", "test"))
+        assertEquals(1, CognateDetector.levenshteinDistance("ofici", "offici"))
+        assertEquals(3, CognateDetector.levenshteinDistance("kitten", "sitting"))
+    }
+
+    @Test
+    fun testStripAccents() {
+        assertEquals("comunicacion", CognateDetector.stripAccents("comunicación"))
+        assertEquals("facilmente", CognateDetector.stripAccents("fácilmente"))
+        assertEquals("rapido", CognateDetector.stripAccents("rápido"))
+        assertEquals("Espana", CognateDetector.stripAccents("España"))
+    }
+}