From 3fccf54c2e62d4e468740f3ab1f5273b7424fc09 Mon Sep 17 00:00:00 2001 From: Menny Even Danan Date: Fri, 16 Oct 2026 14:32:40 +0000 Subject: [PATCH] [LLM] Add curated cognate rules seed dataset and validation tests Adds the checked-in data/seeds/cognate_rules_seed.json dataset covering the 12 core English-to-Spanish suffix transformation rules for the Word Transformer toolbox module, along with structured example parsing, database query support, and unit tests verifying CognateDetector parity. --- data/seeds/BUILD.bazel | 12 + data/seeds/cognate_rules_seed.json | 434 ++++++++++++++++++ tools/preprocessor/BUILD.bazel | 3 + .../preprocessor/CognateRulesProcessor.kt | 87 +++- .../calque/preprocessor/Models.kt | 6 + .../calque/preprocessor/SeedDatabase.kt | 31 ++ .../calque/preprocessor/PreprocessorTest.kt | 78 ++++ 7 files changed, 628 insertions(+), 23 deletions(-) create mode 100644 data/seeds/BUILD.bazel create mode 100644 data/seeds/cognate_rules_seed.json diff --git a/data/seeds/BUILD.bazel b/data/seeds/BUILD.bazel new file mode 100644 index 0000000..94db817 --- /dev/null +++ b/data/seeds/BUILD.bazel @@ -0,0 +1,12 @@ +package(default_visibility = ["//visibility:public"]) + +exports_files([ + "cognate_rules_seed.json", +]) + +filegroup( + name = "seed_files", + srcs = [ + "cognate_rules_seed.json", + ], +) diff --git a/data/seeds/cognate_rules_seed.json b/data/seeds/cognate_rules_seed.json new file mode 100644 index 0000000..7d5ecca --- /dev/null +++ b/data/seeds/cognate_rules_seed.json @@ -0,0 +1,434 @@ +[ + { + "rule_id": "tion_to_cion", + "english_suffix": "-tion", + "spanish_suffix": "-ción", + "gender_rule": "Always feminine (la)", + "spelling_note": "Replace -tion with -ción (written accent on the o). Double consonants like -tt- or -mm- simplify to a single consonant.", + "examples": [ + { + "english": "nation", + "spanish": "nación" + }, + { + "english": "information", + "spanish": "información" + }, + { + "english": "situation", + "spanish": "situación" + }, + { + "english": "conversation", + "spanish": "conversación" + }, + { + "english": "attention", + "spanish": "atención" + }, + { + "english": "communication", + "spanish": "comunicación" + }, + { + "english": "education", + "spanish": "educación" + } + ] + }, + { + "rule_id": "sion_to_sion", + "english_suffix": "-sion", + "spanish_suffix": "-sión", + "gender_rule": "Always feminine (la)", + "spelling_note": "Keep -sion and add a written accent on the o (-sión). English double -ss- becomes a single -s-.", + "examples": [ + { + "english": "version", + "spanish": "versión" + }, + { + "english": "decision", + "spanish": "decisión" + }, + { + "english": "vision", + "spanish": "visión" + }, + { + "english": "confusion", + "spanish": "confusión" + }, + { + "english": "expression", + "spanish": "expresión" + }, + { + "english": "impression", + "spanish": "impresión" + }, + { + "english": "conclusion", + "spanish": "conclusión" + } + ] + }, + { + "rule_id": "ty_to_dad", + "english_suffix": "-ty", + "spanish_suffix": "-dad", + "gender_rule": "Always feminine (la)", + "spelling_note": "Abstract nouns ending in -ty (usually -ity) map to -dad (-idad) with stress on the final syllable.", + "examples": [ + { + "english": "university", + "spanish": "universidad" + }, + { + "english": "activity", + "spanish": "actividad" + }, + { + "english": "reality", + "spanish": "realidad" + }, + { + "english": "capacity", + "spanish": "capacidad" + }, + { + "english": "identity", + "spanish": "identidad" + }, + { + "english": "curiosity", + "spanish": "curiosidad" + }, + { + "english": "opportunity", + "spanish": "oportunidad" + } + ] + }, + { + "rule_id": "ly_to_mente", + "english_suffix": "-ly", + "spanish_suffix": "-mente", + "gender_rule": null, + "spelling_note": "Attach -mente to the feminine singular form of the adjective (or directly to invariable adjectives ending in -al or -e). Any written accent on the base adjective is preserved.", + "examples": [ + { + "english": "naturally", + "spanish": "naturalmente" + }, + { + "english": "totally", + "spanish": "totalmente" + }, + { + "english": "finally", + "spanish": "finalmente" + }, + { + "english": "rapidly", + "spanish": "rápidamente" + }, + { + "english": "exactly", + "spanish": "exactamente" + }, + { + "english": "completely", + "spanish": "completamente" + }, + { + "english": "constantly", + "spanish": "constantemente" + } + ] + }, + { + "rule_id": "ous_to_oso", + "english_suffix": "-ous", + "spanish_suffix": "-oso", + "gender_rule": "Matches noun gender (-oso masculine, -osa feminine)", + "spelling_note": "Replace -ous with -oso (or -osa for feminine nouns).", + "examples": [ + { + "english": "famous", + "spanish": "famoso" + }, + { + "english": "curious", + "spanish": "curioso" + }, + { + "english": "generous", + "spanish": "generoso" + }, + { + "english": "mysterious", + "spanish": "misterioso" + }, + { + "english": "furious", + "spanish": "furioso" + }, + { + "english": "ambitious", + "spanish": "ambicioso" + }, + { + "english": "delicious", + "spanish": "delicioso" + } + ] + }, + { + "rule_id": "ic_to_ico", + "english_suffix": "-ic", + "spanish_suffix": "-ico", + "gender_rule": "Matches noun gender (-ico masculine, -ica feminine)", + "spelling_note": "Add -o or -a to -ic and place a written accent on the preceding vowel (esdrújula stress pattern).", + "examples": [ + { + "english": "public", + "spanish": "público" + }, + { + "english": "basic", + "spanish": "básico" + }, + { + "english": "fantastic", + "spanish": "fantástico" + }, + { + "english": "classic", + "spanish": "clásico" + }, + { + "english": "romantic", + "spanish": "romántico" + }, + { + "english": "electric", + "spanish": "eléctrico" + }, + { + "english": "historic", + "spanish": "histórico" + } + ] + }, + { + "rule_id": "al_to_al", + "english_suffix": "-al", + "spanish_suffix": "-al", + "gender_rule": "Invariable for gender in singular (el/la); plural adds -es (-ales)", + "spelling_note": "Spelling is often identical to English, but spoken stress shifts to the final syllable (-al).", + "examples": [ + { + "english": "natural", + "spanish": "natural" + }, + { + "english": "original", + "spanish": "original" + }, + { + "english": "central", + "spanish": "central" + }, + { + "english": "normal", + "spanish": "normal" + }, + { + "english": "cultural", + "spanish": "cultural" + }, + { + "english": "personal", + "spanish": "personal" + }, + { + "english": "general", + "spanish": "general" + } + ] + }, + { + "rule_id": "ct_to_cto", + "english_suffix": "-ct", + "spanish_suffix": "-cto", + "gender_rule": "Masculine (el) for nouns; matches gender (-cto/-cta) for adjectives", + "spelling_note": "Spanish words do not end in -ct; add -o (or -a for feminine adjectives).", + "examples": [ + { + "english": "exact", + "spanish": "exacto" + }, + { + "english": "perfect", + "spanish": "perfecto" + }, + { + "english": "direct", + "spanish": "directo" + }, + { + "english": "product", + "spanish": "producto" + }, + { + "english": "contact", + "spanish": "contacto" + }, + { + "english": "conflict", + "spanish": "conflicto" + }, + { + "english": "correct", + "spanish": "correcto" + } + ] + }, + { + "rule_id": "nce_to_ncia", + "english_suffix": "-nce", + "spanish_suffix": "-ncia", + "gender_rule": "Always feminine (la)", + "spelling_note": "English -ance and -ence map directly to -ancia and -encia.", + "examples": [ + { + "english": "distance", + "spanish": "distancia" + }, + { + "english": "importance", + "spanish": "importancia" + }, + { + "english": "difference", + "spanish": "diferencia" + }, + { + "english": "experience", + "spanish": "experiencia" + }, + { + "english": "patience", + "spanish": "paciencia" + }, + { + "english": "presence", + "spanish": "presencia" + }, + { + "english": "influence", + "spanish": "influencia" + } + ] + }, + { + "rule_id": "ary_to_ario", + "english_suffix": "-ary", + "spanish_suffix": "-ario", + "gender_rule": "Masculine (el) for nouns; matches gender (-ario/-aria) for adjectives", + "spelling_note": "Replace -ary with -ario (or -aria). Double consonants like -ss- or -cc- simplify or shift to -cc-.", + "examples": [ + { + "english": "necessary", + "spanish": "necesario" + }, + { + "english": "ordinary", + "spanish": "ordinario" + }, + { + "english": "dictionary", + "spanish": "diccionario" + }, + { + "english": "vocabulary", + "spanish": "vocabulario" + }, + { + "english": "primary", + "spanish": "primario" + }, + { + "english": "contrary", + "spanish": "contrario" + } + ] + }, + { + "rule_id": "ive_to_ivo", + "english_suffix": "-ive", + "spanish_suffix": "-ivo", + "gender_rule": "Matches noun gender (-ivo masculine, -iva feminine)", + "spelling_note": "Replace -ive with -ivo (or -iva). Double consonants like -ff- simplify to a single -f-.", + "examples": [ + { + "english": "active", + "spanish": "activo" + }, + { + "english": "positive", + "spanish": "positivo" + }, + { + "english": "negative", + "spanish": "negativo" + }, + { + "english": "creative", + "spanish": "creativo" + }, + { + "english": "exclusive", + "spanish": "exclusivo" + }, + { + "english": "effective", + "spanish": "efectivo" + } + ] + }, + { + "rule_id": "ble_to_ble", + "english_suffix": "-ble", + "spanish_suffix": "-ble", + "gender_rule": "Invariable for gender in singular; plural adds -s (-bles)", + "spelling_note": "Adjectives ending in -able and -ible keep the same suffix in Spanish, dropping double consonants like -ss-.", + "examples": [ + { + "english": "possible", + "spanish": "posible" + }, + { + "english": "visible", + "spanish": "visible" + }, + { + "english": "flexible", + "spanish": "flexible" + }, + { + "english": "probable", + "spanish": "probable" + }, + { + "english": "notable", + "spanish": "notable" + }, + { + "english": "terrible", + "spanish": "terrible" + } + ] + } +] diff --git a/tools/preprocessor/BUILD.bazel b/tools/preprocessor/BUILD.bazel index 730850e..5273509 100644 --- a/tools/preprocessor/BUILD.bazel +++ b/tools/preprocessor/BUILD.bazel @@ -26,6 +26,9 @@ kt_jvm_test( name = "preprocessor_test", size = "small", srcs = glob(["src/test/kotlin/**/*.kt"]), + data = [ + "//data/seeds:cognate_rules_seed.json", + ], test_class = "com.anysoftkeyboard.calque.preprocessor.PreprocessorTest", deps = [ ":preprocessor_lib", diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/CognateRulesProcessor.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/CognateRulesProcessor.kt index 96613a2..3bf481d 100644 --- a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/CognateRulesProcessor.kt +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/CognateRulesProcessor.kt @@ -4,12 +4,18 @@ import java.io.InputStream import java.io.InputStreamReader import java.nio.charset.StandardCharsets import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.contentOrNull import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive /** Seeds the cognate_rules table from a JSON definition. */ object CognateRulesProcessor { + private const val ARROW_DELIMITER = "->" + private const val PAIR_PARTS_COUNT = 2 /** * Reads cognate rules from [inputStream] and inserts them into [database]. @@ -17,32 +23,67 @@ object CognateRulesProcessor { * @return Count of cognate rules inserted. */ fun processRules(inputStream: InputStream, database: SeedDatabase): Int { + val rows = parseRules(inputStream) + database.insertCognateRulesBatch(rows) + return rows.size + } + + /** Parses the cognate rules seed JSON stream into a list of [CognateRuleRow] objects. */ + fun parseRules(inputStream: InputStream): List { val jsonText = InputStreamReader(inputStream, StandardCharsets.UTF_8).use { it.readText() } val array = Json.parseToJsonElement(jsonText).jsonArray - val rows = mutableListOf() - - for (elem in array) { - val obj = elem.jsonObject - val ruleId = obj["rule_id"]?.jsonPrimitive?.content ?: continue - val englishSuffix = obj["english_suffix"]?.jsonPrimitive?.content.orEmpty() - val spanishSuffix = obj["spanish_suffix"]?.jsonPrimitive?.content.orEmpty() - val genderRule = obj["gender_rule"]?.jsonPrimitive?.content - val spellingNote = obj["spelling_note"]?.jsonPrimitive?.content.orEmpty() - val examplesJson = obj["examples"]?.toString() ?: "[]" - - rows.add( - CognateRuleRow( - ruleId = ruleId, - englishSuffix = englishSuffix, - spanishSuffix = spanishSuffix, - genderRule = genderRule, - spellingNote = spellingNote, - examplesJson = examplesJson, - ), - ) + return array.mapNotNull { parseRuleElement(it.jsonObject) } + } + + private fun parseRuleElement(obj: JsonObject): CognateRuleRow? { + val ruleId = obj["rule_id"]?.jsonPrimitive?.contentOrNull ?: return null + val englishSuffix = obj["english_suffix"]?.jsonPrimitive?.contentOrNull.orEmpty() + val spanishSuffix = obj["spanish_suffix"]?.jsonPrimitive?.contentOrNull.orEmpty() + val genderRule = obj["gender_rule"]?.jsonPrimitive?.contentOrNull + val spellingNote = obj["spelling_note"]?.jsonPrimitive?.contentOrNull.orEmpty() + val examplesJson = obj["examples"]?.toString() ?: "[]" + + return CognateRuleRow( + ruleId = ruleId, + englishSuffix = englishSuffix, + spanishSuffix = spanishSuffix, + genderRule = genderRule, + spellingNote = spellingNote, + examplesJson = examplesJson, + ) + } + + /** + * Parses the serialized `examples_json` array into structured [CognateRuleExample] pairs. + * Supports both object items (`{"english": "...", "spanish": "..."}`) and arrow-delimited + * strings (`"english -> spanish"`). + */ + fun parseExamples(examplesJson: String): List { + val array = Json.parseToJsonElement(examplesJson).jsonArray + return array.mapNotNull { parseExampleElement(it) } + } + + private fun parseExampleElement(elem: JsonElement): CognateRuleExample? { + return when (elem) { + is JsonObject -> parseObjectExample(elem) + is JsonPrimitive -> parseArrowExample(elem.contentOrNull.orEmpty()) + else -> null } + } - database.insertCognateRulesBatch(rows) - return rows.size + private fun parseObjectExample(obj: JsonObject): CognateRuleExample? { + val english = obj["english"]?.jsonPrimitive?.contentOrNull?.trim().orEmpty() + val spanish = obj["spanish"]?.jsonPrimitive?.contentOrNull?.trim().orEmpty() + if (english.isEmpty() || spanish.isEmpty()) return null + return CognateRuleExample(english = english, spanish = spanish) + } + + private fun parseArrowExample(raw: String): CognateRuleExample? { + val parts = raw.split(ARROW_DELIMITER) + if (parts.size != PAIR_PARTS_COUNT) return null + val english = parts[0].trim() + val spanish = parts[1].trim() + if (english.isEmpty() || spanish.isEmpty()) return null + return CognateRuleExample(english = english, spanish = spanish) } } diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Models.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Models.kt index 8b15790..850870c 100644 --- a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Models.kt +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Models.kt @@ -55,3 +55,9 @@ data class CognateRuleRow( val spellingNote: String, val examplesJson: String, ) + +/** Represents a single English-to-Spanish word pair in a cognate transformation rule. */ +data class CognateRuleExample( + val english: String, + val spanish: String, +) diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt index 301a2d4..764708f 100644 --- a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt @@ -210,6 +210,26 @@ class SeedDatabase(private val connection: Connection) : AutoCloseable { } } + /** Queries all cognate rules stored in the database. */ + fun queryCognateRules(): List { + val sql = + """ + SELECT rule_id, english_suffix, spanish_suffix, gender_rule, spelling_note, examples_json + FROM cognate_rules + ORDER BY rowid ASC + """ + .trimIndent() + val rules = mutableListOf() + connection.createStatement().use { stmt -> + stmt.executeQuery(sql).use { rs -> + while (rs.next()) { + rules.add(DatabaseBinders.mapCognateRuleRow(rs)) + } + } + } + return rules + } + override fun close() { connection.close() } @@ -324,4 +344,15 @@ internal object DatabaseBinders { pstmt.setString(PARAM_RULE_SPELLING, entry.spellingNote) pstmt.setString(PARAM_RULE_EXAMPLES, entry.examplesJson) } + + fun mapCognateRuleRow(rs: ResultSet): CognateRuleRow { + return CognateRuleRow( + ruleId = rs.getString("rule_id"), + englishSuffix = rs.getString("english_suffix"), + spanishSuffix = rs.getString("spanish_suffix"), + genderRule = rs.getString("gender_rule"), + spellingNote = rs.getString("spelling_note"), + examplesJson = rs.getString("examples_json"), + ) + } } diff --git a/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt b/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt index 8ca1a7b..df7e93c 100644 --- a/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt +++ b/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt @@ -1,10 +1,14 @@ package com.anysoftkeyboard.calque.preprocessor +import com.anysoftkeyboard.calque.linguistics.CognateDetector import java.io.File +import java.io.FileInputStream import java.nio.charset.StandardCharsets import kotlin.io.path.createTempDirectory import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull import org.junit.Assert.assertTrue import org.junit.Test @@ -173,6 +177,76 @@ class PreprocessorTest { val rulesCount = CognateRulesProcessor.processRules(rulesJson.byteInputStream(), db) assertEquals(1, rulesCount) + + val storedRules = db.queryCognateRules() + assertEquals(1, storedRules.size) + assertEquals("tion_to_cion", storedRules[0].ruleId) + val examples = CognateRulesProcessor.parseExamples(storedRules[0].examplesJson) + assertEquals(2, examples.size) + assertEquals(CognateRuleExample("action", "acción"), examples[0]) + } + } + + @Test + fun curatedCognateRulesSeed_isValidAndAlignedWithCognateDetector() { + val seedFile = File("data/seeds/cognate_rules_seed.json") + assertTrue( + "Expected seed file at ${seedFile.path}", + seedFile.exists(), + ) + + SeedDatabase.open().use { db -> + val insertedCount = + FileInputStream(seedFile).use { stream -> + CognateRulesProcessor.processRules(stream, db) + } + assertEquals(12, insertedCount) + + val rules = db.queryCognateRules() + assertEquals(12, rules.size) + + val ruleIds = rules.map { it.ruleId }.toSet() + val expectedIds = + setOf( + "tion_to_cion", + "sion_to_sion", + "ty_to_dad", + "ly_to_mente", + "ous_to_oso", + "ic_to_ico", + "al_to_al", + "ct_to_cto", + "nce_to_ncia", + "ary_to_ario", + "ive_to_ivo", + "ble_to_ble", + ) + assertEquals(expectedIds, ruleIds) + + val adverbRule = rules.first { it.ruleId == "ly_to_mente" } + assertNull(adverbRule.genderRule) + + val nounRule = rules.first { it.ruleId == "tion_to_cion" } + assertEquals("Always feminine (la)", nounRule.genderRule) + + for (rule in rules) { + assertTrue(rule.englishSuffix.startsWith("-")) + assertTrue(rule.spanishSuffix.startsWith("-")) + assertFalse(rule.spellingNote.isBlank()) + + val examples = CognateRulesProcessor.parseExamples(rule.examplesJson) + assertTrue( + "Rule ${rule.ruleId} should have 5..10 examples, found ${examples.size}", + examples.size in 5..10, + ) + + for (example in examples) { + assertTrue( + "Expected (${example.spanish}, ${example.english}) in ${rule.ruleId} to match CognateDetector", + CognateDetector.matchesCognate(example.spanish, example.english), + ) + } + } } } @@ -210,12 +284,15 @@ class PreprocessorTest { StandardCharsets.UTF_8, ) + val cognateRulesSeedFile = File("data/seeds/cognate_rules_seed.json") + val config = PipelineConfig( dbFile = dbFile, kaikkiFile = kaikkiFile, sublexFile = sublexFile, falseCognatesFile = fcFile, + cognateRulesFile = cognateRulesSeedFile, ) val pipeline = PreprocessorPipeline(config) @@ -224,6 +301,7 @@ class PreprocessorTest { assertTrue(dbFile.exists()) assertTrue(summary.dictionaryRows >= 2) assertEquals(2, summary.falseCognateRows) + assertEquals(12, summary.cognateRuleRows) tempDir.deleteRecursively() }