diff --git a/data/seeds/BUILD.bazel b/data/seeds/BUILD.bazel index 94db817..793b425 100644 --- a/data/seeds/BUILD.bazel +++ b/data/seeds/BUILD.bazel @@ -2,11 +2,13 @@ package(default_visibility = ["//visibility:public"]) exports_files([ "cognate_rules_seed.json", + "false_cognates_seed.json", ]) filegroup( name = "seed_files", srcs = [ "cognate_rules_seed.json", + "false_cognates_seed.json", ], ) diff --git a/data/seeds/false_cognates_seed.json b/data/seeds/false_cognates_seed.json new file mode 100644 index 0000000..43689f6 --- /dev/null +++ b/data/seeds/false_cognates_seed.json @@ -0,0 +1,252 @@ +[ + { + "lemma": "éxito", + "apparent_english": "exit", + "actual_english": "success / hit", + "real_spanish_for_apparent": "salida", + "severity": "HIGH_RISK", + "disambiguation_tip": "Think of a 'successful exit' from a startup: éxito means success, while salida is the physical door out.", + "example_es": "El proyecto fue un gran éxito para todo el equipo.", + "example_en": "The project was a great success for the whole team." + }, + { + "lemma": "carpeta", + "apparent_english": "carpet", + "actual_english": "folder / binder", + "real_spanish_for_apparent": "alfombra", + "severity": "HIGH_RISK", + "disambiguation_tip": "You file papers inside a carpeta on your desk and walk on an alfombra on the floor.", + "example_es": "Guarda los documentos importantes en la carpeta azul.", + "example_en": "Keep the important documents in the blue folder." + }, + { + "lemma": "embarazada", + "apparent_english": "embarrassed", + "actual_english": "pregnant", + "real_spanish_for_apparent": "avergonzado", + "severity": "HIGH_RISK", + "disambiguation_tip": "Embarazada means carrying a baby (pregnant); to say you feel embarrassed, use avergonzado or tener vergüenza.", + "example_es": "Mi hermana está embarazada de seis meses.", + "example_en": "My sister is six months pregnant." + }, + { + "lemma": "soportar", + "apparent_english": "to support", + "actual_english": "to tolerate / endure / stand", + "real_spanish_for_apparent": "apoyar", + "severity": "HIGH_RISK", + "disambiguation_tip": "Soportar is bearing a heavy burden ('I can't stand it'); to back or help someone is apoyar.", + "example_es": "Elena no soporta el ruido de la calle por la noche.", + "example_en": "Elena cannot stand the noise from the street at night." + }, + { + "lemma": "realizar", + "apparent_english": "to realize (notice)", + "actual_english": "to carry out / accomplish / fulfill", + "real_spanish_for_apparent": "darse cuenta de", + "severity": "HIGH_RISK", + "disambiguation_tip": "Realizar means to make something real by doing it; realizing a fact in your mind is darse cuenta de.", + "example_es": "Los investigadores realizaron un estudio muy completo.", + "example_en": "The researchers carried out a very thorough study." + }, + { + "lemma": "actualmente", + "apparent_english": "actually (in fact)", + "actual_english": "currently / nowadays", + "real_spanish_for_apparent": "en realidad", + "severity": "HIGH_RISK", + "disambiguation_tip": "Link actual and actualmente to 'at the current moment' (like French actuellement), not 'in fact'.", + "example_es": "Actualmente vivimos en el centro de Madrid.", + "example_en": "Currently we live in downtown Madrid." + }, + { + "lemma": "actual", + "apparent_english": "actual (real)", + "actual_english": "current / present-day", + "real_spanish_for_apparent": "real / verdadero", + "severity": "HIGH_RISK", + "disambiguation_tip": "El presidente actual is the current president, not the 'real' one.", + "example_es": "La situación actual requiere mucha paciencia.", + "example_en": "The current situation requires a lot of patience." + }, + { + "lemma": "asistir", + "apparent_english": "to assist (help)", + "actual_english": "to attend / be present at", + "real_spanish_for_apparent": "ayudar", + "severity": "HIGH_RISK", + "disambiguation_tip": "When you asistir a una reunión, you merely show up and attend; ayudar is to help.", + "example_es": "Más de cien personas asistieron a la conferencia.", + "example_en": "More than a hundred people attended the conference." + }, + { + "lemma": "largo", + "apparent_english": "large", + "actual_english": "long", + "real_spanish_for_apparent": "grande", + "severity": "HIGH_RISK", + "disambiguation_tip": "Largo measures length (long), whereas grande measures overall size (large/big).", + "example_es": "Fue un viaje muy largo en tren.", + "example_en": "It was a very long trip by train." + }, + { + "lemma": "librería", + "apparent_english": "library", + "actual_english": "bookstore", + "real_spanish_for_apparent": "biblioteca", + "severity": "HIGH_RISK", + "disambiguation_tip": "Ends in -ería like panadería (bakery)—a shop where you buy libros (books). Borrow books at a biblioteca.", + "example_es": "Compré esta novela en la librería del barrio.", + "example_en": "I bought this novel at the neighborhood bookstore." + }, + { + "lemma": "lectura", + "apparent_english": "lecture", + "actual_english": "reading", + "real_spanish_for_apparent": "conferencia / clase", + "severity": "HIGH_RISK", + "disambiguation_tip": "Connected to leer (to read) and lector (reader): lectura is the act or passage of reading.", + "example_es": "La lectura diaria mejora el vocabulario rápidamente.", + "example_en": "Daily reading improves vocabulary rapidly." + }, + { + "lemma": "sensible", + "apparent_english": "sensible (practical)", + "actual_english": "sensitive", + "real_spanish_for_apparent": "sensato", + "severity": "HIGH_RISK", + "disambiguation_tip": "Una persona sensible feels emotions deeply (sensitive); a level-headed, practical person is sensata.", + "example_es": "Tiene la piel muy sensible al sol.", + "example_en": "She has skin that is very sensitive to the sun." + }, + { + "lemma": "recordar", + "apparent_english": "to record", + "actual_english": "to remember / recall", + "real_spanish_for_apparent": "grabar", + "severity": "HIGH_RISK", + "disambiguation_tip": "From Latin cor (heart): recordar is bringing something back to heart (remembering); recording audio is grabar.", + "example_es": "No recuerdo el nombre de aquella calle.", + "example_en": "I do not remember the name of that street." + }, + { + "lemma": "delito", + "apparent_english": "delight", + "actual_english": "crime / offense", + "real_spanish_for_apparent": "deleite / placer", + "severity": "HIGH_RISK", + "disambiguation_tip": "Cousin of English 'delinquent' and 'delict': un delito is a legal offense or crime.", + "example_es": "El juez declaró que aquello era un delito grave.", + "example_en": "The judge declared that it was a serious crime." + }, + { + "lemma": "suceso", + "apparent_english": "success", + "actual_english": "event / incident / happening", + "real_spanish_for_apparent": "éxito", + "severity": "HIGH_RISK", + "disambiguation_tip": "Think of 'what succeeded/followed in time': un suceso is simply an event (or news incident), while éxito is success.", + "example_es": "El periódico publicó todos los detalles del suceso.", + "example_en": "The newspaper published all the details of the event." + }, + { + "lemma": "pretender", + "apparent_english": "to pretend (make believe)", + "actual_english": "to intend / aim / seek to", + "real_spanish_for_apparent": "fingir", + "severity": "HIGH_RISK", + "disambiguation_tip": "Pretender is serious ambition (what you aim or intend to do); faking or pretending is fingir.", + "example_es": "El autor pretende explicar la historia con claridad.", + "example_en": "The author intends to explain the history clearly." + }, + { + "lemma": "constipado", + "apparent_english": "constipated", + "actual_english": "having a common cold / congested", + "real_spanish_for_apparent": "estreñido", + "severity": "HIGH_RISK", + "disambiguation_tip": "Estar constipado refers to nasal congestion from a head cold, not digestive trouble.", + "example_es": "Estoy un poco constipado por el cambio de clima.", + "example_en": "I have a bit of a cold from the change in weather." + }, + { + "lemma": "fábrica", + "apparent_english": "fabric", + "actual_english": "factory / plant", + "real_spanish_for_apparent": "tela", + "severity": "HIGH_RISK", + "disambiguation_tip": "Where things are 'fabricated' (manufactured): fábrica is a factory, while cloth fabric is tela.", + "example_es": "La fábrica produce automóviles eléctricos.", + "example_en": "The factory produces electric cars." + }, + { + "lemma": "pariente", + "apparent_english": "parent", + "actual_english": "relative (uncle, cousin, etc.)", + "real_spanish_for_apparent": "padre / madre (padres)", + "severity": "HIGH_RISK", + "disambiguation_tip": "Parientes are extended family relatives; only your mother and father are tus padres.", + "example_es": "Todos mis parientes vinieron a la boda.", + "example_en": "All my relatives came to the wedding." + }, + { + "lemma": "sopa", + "apparent_english": "soap", + "actual_english": "soup", + "real_spanish_for_apparent": "jabón", + "severity": "HIGH_RISK", + "disambiguation_tip": "You eat sopa with a spoon and wash your hands with jabón.", + "example_es": "Pedimos una sopa caliente para la cena.", + "example_en": "We ordered a hot soup for dinner." + }, + { + "lemma": "compromiso", + "apparent_english": "compromise (mutual concession)", + "actual_english": "commitment / obligation / appointment", + "real_spanish_for_apparent": "acuerdo mutuo", + "severity": "MILD_NUANCE", + "disambiguation_tip": "Usually means a binding commitment, social obligation, or engagement rather than meeting halfway.", + "example_es": "No puedo ir porque tengo otro compromiso esta tarde.", + "example_en": "I cannot go because I have another commitment this afternoon." + }, + { + "lemma": "educado", + "apparent_english": "educated (formally schooled)", + "actual_english": "polite / well-mannered", + "real_spanish_for_apparent": "instruido / culto", + "severity": "MILD_NUANCE", + "disambiguation_tip": "Refers to upbringing and courtesy at home (polite); academic schooling is culto or con estudios.", + "example_es": "El camarero fue muy amable y educado con nosotros.", + "example_en": "The waiter was very kind and polite with us." + }, + { + "lemma": "molestar", + "apparent_english": "to molest", + "actual_english": "to bother / annoy / inconvenience", + "real_spanish_for_apparent": "acosar / agredir", + "severity": "MILD_NUANCE", + "disambiguation_tip": "An everyday, mild word meaning to bother or disturb ('Perdón por molestar' = 'Sorry to bother you').", + "example_es": "Disculpa, ¿te molesta si abro la ventana?", + "example_en": "Excuse me, does it bother you if I open the window?" + }, + { + "lemma": "discutir", + "apparent_english": "to discuss calmly", + "actual_english": "to argue / debate heatedly", + "real_spanish_for_apparent": "hablar de / conversar", + "severity": "MILD_NUANCE", + "disambiguation_tip": "In everyday Spanish, discutir carries much more heat (arguing or clashing) than a neutral English 'discussion'.", + "example_es": "Los vecinos discutieron durante horas por el ruido.", + "example_en": "The neighbors argued for hours over the noise." + }, + { + "lemma": "introducir", + "apparent_english": "to introduce (a person)", + "actual_english": "to insert / put in / bring in", + "real_spanish_for_apparent": "presentar", + "severity": "MILD_NUANCE", + "disambiguation_tip": "You introducir a key into a lock or a password into a form, but you presentar a friend to another person.", + "example_es": "Introduzca su contraseña para continuar.", + "example_en": "Enter your password to continue." + } +] diff --git a/tools/preprocessor/BUILD.bazel b/tools/preprocessor/BUILD.bazel index 5273509..721dfbd 100644 --- a/tools/preprocessor/BUILD.bazel +++ b/tools/preprocessor/BUILD.bazel @@ -28,6 +28,7 @@ kt_jvm_test( srcs = glob(["src/test/kotlin/**/*.kt"]), data = [ "//data/seeds:cognate_rules_seed.json", + "//data/seeds:false_cognates_seed.json", ], test_class = "com.anysoftkeyboard.calque.preprocessor.PreprocessorTest", deps = [ diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/FalseCognatesExpander.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/FalseCognatesExpander.kt index cee5375..4287bca 100644 --- a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/FalseCognatesExpander.kt +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/FalseCognatesExpander.kt @@ -5,6 +5,8 @@ import java.io.InputStreamReader import java.nio.charset.StandardCharsets import java.util.Locale import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.contentOrNull import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive @@ -55,32 +57,28 @@ object FalseCognatesExpander { fun parseSeeds(inputStream: InputStream): List { val jsonText = InputStreamReader(inputStream, StandardCharsets.UTF_8).use { it.readText() } val array = Json.parseToJsonElement(jsonText).jsonArray - val seeds = mutableListOf() + return array.mapNotNull { parseSeedElement(it.jsonObject) } + } - for (elem in array) { - val obj = elem.jsonObject - val lemma = obj["lemma"]?.jsonPrimitive?.content ?: continue - val apparentEnglish = obj["apparent_english"]?.jsonPrimitive?.content.orEmpty() - val actualEnglish = obj["actual_english"]?.jsonPrimitive?.content.orEmpty() - val realSpanish = obj["real_spanish_for_apparent"]?.jsonPrimitive?.content.orEmpty() - val severity = obj["severity"]?.jsonPrimitive?.content ?: "HIGH_RISK" - val tip = obj["disambiguation_tip"]?.jsonPrimitive?.content.orEmpty() - val exampleEs = obj["example_es"]?.jsonPrimitive?.content.orEmpty() - val exampleEn = obj["example_en"]?.jsonPrimitive?.content.orEmpty() + private fun parseSeedElement(obj: JsonObject): FalseCognateSeed? { + val lemma = obj["lemma"]?.jsonPrimitive?.contentOrNull ?: return null + val apparentEnglish = obj["apparent_english"]?.jsonPrimitive?.contentOrNull.orEmpty() + val actualEnglish = obj["actual_english"]?.jsonPrimitive?.contentOrNull.orEmpty() + val realSpanish = obj["real_spanish_for_apparent"]?.jsonPrimitive?.contentOrNull.orEmpty() + val severity = obj["severity"]?.jsonPrimitive?.contentOrNull ?: "HIGH_RISK" + val tip = obj["disambiguation_tip"]?.jsonPrimitive?.contentOrNull.orEmpty() + val exampleEs = obj["example_es"]?.jsonPrimitive?.contentOrNull.orEmpty() + val exampleEn = obj["example_en"]?.jsonPrimitive?.contentOrNull.orEmpty() - seeds.add( - FalseCognateSeed( - lemma = lemma, - apparentEnglish = apparentEnglish, - actualEnglish = actualEnglish, - realSpanishForApparent = realSpanish, - severity = severity, - disambiguationTip = tip, - exampleEs = exampleEs, - exampleEn = exampleEn, - ), - ) - } - return seeds + return FalseCognateSeed( + lemma = lemma, + apparentEnglish = apparentEnglish, + actualEnglish = actualEnglish, + realSpanishForApparent = realSpanish, + severity = severity, + disambiguationTip = tip, + exampleEs = exampleEs, + exampleEn = exampleEn, + ) } } diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt index 764708f..7540d72 100644 --- a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt @@ -16,65 +16,7 @@ import java.sql.Types class SeedDatabase(private val connection: Connection) : AutoCloseable { init { - createTables() - } - - private fun createTables() { - val statements = - listOf( - """ - CREATE TABLE IF NOT EXISTS dictionary ( - surface_form TEXT PRIMARY KEY, - lemma TEXT NOT NULL, - pos TEXT NOT NULL, - grammar_info TEXT, - english_gloss TEXT NOT NULL, - is_cognate INTEGER NOT NULL, - frequency_rank INTEGER - ) - """, - "CREATE INDEX IF NOT EXISTS idx_dictionary_lemma ON dictionary(lemma)", - """ - CREATE TABLE IF NOT EXISTS false_cognates ( - surface_form TEXT PRIMARY KEY, - lemma TEXT NOT NULL, - apparent_english TEXT NOT NULL, - actual_english TEXT NOT NULL, - real_spanish_for_apparent TEXT NOT NULL, - severity TEXT NOT NULL, - disambiguation_tip TEXT NOT NULL, - example_es TEXT NOT NULL, - example_en TEXT NOT NULL - ) - """, - "CREATE INDEX IF NOT EXISTS idx_false_cognates_lemma ON false_cognates(lemma)", - """ - CREATE TABLE IF NOT EXISTS curated_lessons ( - lesson_id TEXT PRIMARY KEY, - level INTEGER NOT NULL, - sequence_order INTEGER NOT NULL, - title TEXT NOT NULL, - target_concept TEXT NOT NULL, - content_json TEXT NOT NULL - ) - """, - "CREATE INDEX IF NOT EXISTS idx_curated_lessons_level ON curated_lessons(level)", - """ - CREATE TABLE IF NOT EXISTS cognate_rules ( - rule_id TEXT PRIMARY KEY, - english_suffix TEXT NOT NULL, - spanish_suffix TEXT NOT NULL, - gender_rule TEXT, - spelling_note TEXT NOT NULL, - examples_json TEXT NOT NULL - ) - """, - ) - connection.createStatement().use { stmt -> - for (sql in statements) { - stmt.execute(sql.trimIndent()) - } - } + DatabaseBinders.createTables(connection) } private inline fun executeTransaction(action: () -> Unit) { @@ -166,6 +108,27 @@ class SeedDatabase(private val connection: Connection) : AutoCloseable { } } + /** Queries all false cognate rows stored in the database. */ + fun queryFalseCognates(): List { + val sql = + """ + SELECT surface_form, lemma, apparent_english, actual_english, real_spanish_for_apparent, + severity, disambiguation_tip, example_es, example_en + FROM false_cognates + ORDER BY rowid ASC + """ + .trimIndent() + val rows = mutableListOf() + connection.createStatement().use { stmt -> + stmt.executeQuery(sql).use { rs -> + while (rs.next()) { + rows.add(DatabaseBinders.mapFalseCognateRow(rs)) + } + } + } + return rows + } + /** Inserts a batch of curated lessons. */ fun insertCuratedLessonsBatch(entries: Collection) { if (entries.isEmpty()) return @@ -281,6 +244,64 @@ internal object DatabaseBinders { private const val PARAM_RULE_SPELLING = 5 private const val PARAM_RULE_EXAMPLES = 6 + fun createTables(connection: Connection) { + val statements = + listOf( + """ + CREATE TABLE IF NOT EXISTS dictionary ( + surface_form TEXT PRIMARY KEY, + lemma TEXT NOT NULL, + pos TEXT NOT NULL, + grammar_info TEXT, + english_gloss TEXT NOT NULL, + is_cognate INTEGER NOT NULL, + frequency_rank INTEGER + ) + """, + "CREATE INDEX IF NOT EXISTS idx_dictionary_lemma ON dictionary(lemma)", + """ + CREATE TABLE IF NOT EXISTS false_cognates ( + surface_form TEXT PRIMARY KEY, + lemma TEXT NOT NULL, + apparent_english TEXT NOT NULL, + actual_english TEXT NOT NULL, + real_spanish_for_apparent TEXT NOT NULL, + severity TEXT NOT NULL, + disambiguation_tip TEXT NOT NULL, + example_es TEXT NOT NULL, + example_en TEXT NOT NULL + ) + """, + "CREATE INDEX IF NOT EXISTS idx_false_cognates_lemma ON false_cognates(lemma)", + """ + CREATE TABLE IF NOT EXISTS curated_lessons ( + lesson_id TEXT PRIMARY KEY, + level INTEGER NOT NULL, + sequence_order INTEGER NOT NULL, + title TEXT NOT NULL, + target_concept TEXT NOT NULL, + content_json TEXT NOT NULL + ) + """, + "CREATE INDEX IF NOT EXISTS idx_curated_lessons_level ON curated_lessons(level)", + """ + CREATE TABLE IF NOT EXISTS cognate_rules ( + rule_id TEXT PRIMARY KEY, + english_suffix TEXT NOT NULL, + spanish_suffix TEXT NOT NULL, + gender_rule TEXT, + spelling_note TEXT NOT NULL, + examples_json TEXT NOT NULL + ) + """, + ) + connection.createStatement().use { stmt -> + for (sql in statements) { + stmt.execute(sql.trimIndent()) + } + } + } + fun bindDictionaryRow(pstmt: PreparedStatement, entry: DictionaryRow) { pstmt.setString(PARAM_DICT_SURFACE, entry.surfaceForm) pstmt.setString(PARAM_DICT_LEMMA, entry.lemma) @@ -323,6 +344,20 @@ internal object DatabaseBinders { pstmt.setString(PARAM_FC_EX_EN, entry.exampleEn) } + fun mapFalseCognateRow(rs: ResultSet): FalseCognateRow { + return FalseCognateRow( + surfaceForm = rs.getString("surface_form"), + lemma = rs.getString("lemma"), + apparentEnglish = rs.getString("apparent_english"), + actualEnglish = rs.getString("actual_english"), + realSpanishForApparent = rs.getString("real_spanish_for_apparent"), + severity = rs.getString("severity"), + disambiguationTip = rs.getString("disambiguation_tip"), + exampleEs = rs.getString("example_es"), + exampleEn = rs.getString("example_en"), + ) + } + fun bindCuratedLessonRow(pstmt: PreparedStatement, entry: CuratedLessonRow) { pstmt.setString(PARAM_LESSON_ID, entry.lessonId) pstmt.setInt(PARAM_LESSON_LEVEL, entry.level) diff --git a/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt b/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt index df7e93c..f1cbdbe 100644 --- a/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt +++ b/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt @@ -121,6 +121,69 @@ class PreprocessorTest { } } + @Test + fun curatedFalseCognatesSeed_hasValidFourPartCardMetadata() { + val seedFile = File("data/seeds/false_cognates_seed.json") + assertTrue("Expected seed file at ${seedFile.path}", seedFile.exists()) + + val seeds = FileInputStream(seedFile).use { FalseCognatesExpander.parseSeeds(it) } + assertEquals(25, seeds.size) + + val lemmas = seeds.map { it.lemma }.toSet() + assertEquals(seeds.size, lemmas.size) + val coreLemmas = setOf("éxito", "carpeta", "embarazada", "soportar", "realizar") + assertTrue(lemmas.containsAll(coreLemmas)) + + val validSeverities = setOf("HIGH_RISK", "MILD_NUANCE") + assertTrue(seeds.any { it.severity == "HIGH_RISK" }) + assertTrue(seeds.any { it.severity == "MILD_NUANCE" }) + + for (seed in seeds) { + assertFalse(seed.lemma.isBlank()) + assertFalse(seed.apparentEnglish.isBlank()) + assertFalse(seed.actualEnglish.isBlank()) + assertFalse(seed.realSpanishForApparent.isBlank()) + assertTrue(seed.severity in validSeverities) + assertFalse(seed.disambiguationTip.isBlank()) + assertFalse(seed.exampleEs.isBlank()) + assertFalse(seed.exampleEn.isBlank()) + } + } + + @Test + fun curatedFalseCognatesSeed_expandsInflectionsIntoDatabase() { + val seedFile = File("data/seeds/false_cognates_seed.json") + SeedDatabase.open().use { db -> + db.insertDictionaryBatch( + listOf( + DictionaryRow( + "soporta", + "soportar", + "verb", + "3rd sg. present", + "tolerates", + false, + 400, + ), + DictionaryRow("carpetas", "carpeta", "noun", "plural", "folders", false, 500), + ), + ) + + val expandedCount = + FileInputStream(seedFile).use { stream -> FalseCognatesExpander.expand(stream, db) } + // 25 base lemmas + 2 inflected dictionary forms ("soporta", "carpetas") = 27 rows + assertEquals(27, expandedCount) + + val storedRows = db.queryFalseCognates() + assertEquals(27, storedRows.size) + val surfaceMap = storedRows.associateBy { it.surfaceForm } + assertEquals("soportar", surfaceMap["soporta"]?.lemma) + assertEquals("carpeta", surfaceMap["carpetas"]?.lemma) + assertEquals("HIGH_RISK", surfaceMap["éxito"]?.severity) + assertEquals("MILD_NUANCE", surfaceMap["compromiso"]?.severity) + } + } + @Test fun tatoebaProcessor_scoresAndBucketsSentences() { val inMemoryDb = SeedDatabase.open() @@ -264,26 +327,7 @@ class PreprocessorTest { val kaikkiFile = File(tempDir, "kaikki.jsonl") kaikkiFile.writeText("$sampleKaikki\n", StandardCharsets.UTF_8) - val fcFile = File(tempDir, "false_cognates.json") - fcFile.writeText( - """ - [ - { - "lemma": "carpeta", - "apparent_english": "carpet", - "actual_english": "folder", - "real_spanish_for_apparent": "alfombra", - "severity": "HIGH_RISK", - "disambiguation_tip": "A folder, not a carpet.", - "example_es": "Mi carpeta.", - "example_en": "My folder." - } - ] - """ - .trimIndent(), - StandardCharsets.UTF_8, - ) - + val falseCognatesSeedFile = File("data/seeds/false_cognates_seed.json") val cognateRulesSeedFile = File("data/seeds/cognate_rules_seed.json") val config = @@ -291,7 +335,7 @@ class PreprocessorTest { dbFile = dbFile, kaikkiFile = kaikkiFile, sublexFile = sublexFile, - falseCognatesFile = fcFile, + falseCognatesFile = falseCognatesSeedFile, cognateRulesFile = cognateRulesSeedFile, ) @@ -300,7 +344,8 @@ class PreprocessorTest { assertTrue(dbFile.exists()) assertTrue(summary.dictionaryRows >= 2) - assertEquals(2, summary.falseCognateRows) + // 25 base lemmas + "carpetas" plural from Kaikki = 26 rows + assertEquals(26, summary.falseCognateRows) assertEquals(12, summary.cognateRuleRows) tempDir.deleteRecursively()