diff --git a/app/BUILD.bazel b/app/BUILD.bazel index ffd0fb0..4dd547b 100644 --- a/app/BUILD.bazel +++ b/app/BUILD.bazel @@ -1 +1,10 @@ package(default_visibility = ["//visibility:public"]) + +exports_files([ + "src/main/assets/databases/seed.db", +]) + +filegroup( + name = "seed_db_asset", + srcs = ["src/main/assets/databases/seed.db"], +) diff --git a/app/src/main/assets/databases/seed.db b/app/src/main/assets/databases/seed.db new file mode 100644 index 0000000..1812cf8 Binary files /dev/null and b/app/src/main/assets/databases/seed.db differ diff --git a/core/data/BUILD.bazel b/core/data/BUILD.bazel index 6217a99..b8267d4 100644 --- a/core/data/BUILD.bazel +++ b/core/data/BUILD.bazel @@ -13,6 +13,7 @@ kt_jvm_library( kt_jvm_test( name = "seed_database_test", + size = "small", srcs = ["src/test/kotlin/com/anysoftkeyboard/calque/data/seed/SeedDatabaseTest.kt"], test_class = "com.anysoftkeyboard.calque.data.seed.SeedDatabaseTest", deps = [ @@ -23,6 +24,7 @@ kt_jvm_test( kt_jvm_test( name = "seed_dictionary_repository_test", + size = "small", srcs = ["src/test/kotlin/com/anysoftkeyboard/calque/data/seed/SeedDictionaryRepositoryTest.kt"], test_class = "com.anysoftkeyboard.calque.data.seed.SeedDictionaryRepositoryTest", deps = [ @@ -34,6 +36,7 @@ kt_jvm_test( kt_jvm_test( name = "seed_false_cognates_repository_test", + size = "small", srcs = ["src/test/kotlin/com/anysoftkeyboard/calque/data/seed/SeedFalseCognatesRepositoryTest.kt"], test_class = "com.anysoftkeyboard.calque.data.seed.SeedFalseCognatesRepositoryTest", deps = [ @@ -45,6 +48,7 @@ kt_jvm_test( kt_jvm_test( name = "seed_curated_lessons_repository_test", + size = "small", srcs = ["src/test/kotlin/com/anysoftkeyboard/calque/data/seed/SeedCuratedLessonsRepositoryTest.kt"], test_class = "com.anysoftkeyboard.calque.data.seed.SeedCuratedLessonsRepositoryTest", deps = [ @@ -55,6 +59,7 @@ kt_jvm_test( kt_jvm_test( name = "seed_cognate_rules_repository_test", + size = "small", srcs = ["src/test/kotlin/com/anysoftkeyboard/calque/data/seed/SeedCognateRulesRepositoryTest.kt"], test_class = "com.anysoftkeyboard.calque.data.seed.SeedCognateRulesRepositoryTest", deps = [ @@ -63,8 +68,23 @@ kt_jvm_test( ], ) +kt_jvm_test( + name = "packaged_seed_asset_test", + size = "small", + srcs = ["src/test/kotlin/com/anysoftkeyboard/calque/data/seed/PackagedSeedAssetTest.kt"], + data = [ + "//app:seed_db_asset", + ], + test_class = "com.anysoftkeyboard.calque.data.seed.PackagedSeedAssetTest", + deps = [ + ":data", + "@maven//:junit_junit", + ], +) + kt_jvm_test( name = "user_database_test", + size = "small", srcs = ["src/test/kotlin/com/anysoftkeyboard/calque/data/user/UserDatabaseTest.kt"], test_class = "com.anysoftkeyboard.calque.data.user.UserDatabaseTest", deps = [ @@ -75,6 +95,7 @@ kt_jvm_test( kt_jvm_test( name = "user_word_state_repository_test", + size = "small", srcs = ["src/test/kotlin/com/anysoftkeyboard/calque/data/user/UserWordStateRepositoryTest.kt"], test_class = "com.anysoftkeyboard.calque.data.user.UserWordStateRepositoryTest", deps = [ @@ -86,6 +107,7 @@ kt_jvm_test( kt_jvm_test( name = "user_custom_text_repository_test", + size = "small", srcs = ["src/test/kotlin/com/anysoftkeyboard/calque/data/user/UserCustomTextRepositoryTest.kt"], test_class = "com.anysoftkeyboard.calque.data.user.UserCustomTextRepositoryTest", deps = [ @@ -96,6 +118,7 @@ kt_jvm_test( kt_jvm_test( name = "calque_word_lookup_provider_test", + size = "small", srcs = ["src/test/kotlin/com/anysoftkeyboard/calque/data/CalqueWordLookupProviderTest.kt"], test_class = "com.anysoftkeyboard.calque.data.CalqueWordLookupProviderTest", deps = [ diff --git a/core/data/src/test/kotlin/com/anysoftkeyboard/calque/data/seed/PackagedSeedAssetTest.kt b/core/data/src/test/kotlin/com/anysoftkeyboard/calque/data/seed/PackagedSeedAssetTest.kt new file mode 100644 index 0000000..dd0d2da --- /dev/null +++ b/core/data/src/test/kotlin/com/anysoftkeyboard/calque/data/seed/PackagedSeedAssetTest.kt @@ -0,0 +1,72 @@ +package com.anysoftkeyboard.calque.data.seed + +import java.io.File +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class PackagedSeedAssetTest { + + @Test + fun packagedSeedAsset_existsAndHasAllSeedTablesPopulated() { + val assetFile = File("app/src/main/assets/databases/seed.db") + assertTrue("Expected packaged seed database at ${assetFile.path}", assetFile.exists()) + assertTrue( + "Expected database size > 10KB, got ${assetFile.length()}", + assetFile.length() > 10_000 + ) + + val database = SeedDatabase.open("jdbc:sqlite:${assetFile.path}") + database.use { db -> + val dictRepo = SqliteDictionaryRepository(db.connection) + val fcRepo = SqliteFalseCognatesRepository(db.connection) + val lessonRepo = SqliteCuratedLessonsRepository(db.connection) + val rulesRepo = SqliteCognateRulesRepository(db.connection) + + // 1. Verify dictionary entries + val hotelWord = dictRepo.getBySurfaceForm("hotel") + assertNotNull("Dictionary should contain 'hotel'", hotelWord) + assertEquals("hotel", hotelWord?.lemma) + assertTrue("hotel should be marked as cognate", hotelWord?.isCognate == true) + + val esWord = dictRepo.getBySurfaceForm("es") + assertNotNull("Dictionary should contain 'es'", esWord) + assertEquals("ser", esWord?.lemma) + + // 2. Verify false cognates + val exitoCard = fcRepo.getBySurfaceForm("éxito") + assertNotNull("False cognates should contain 'éxito'", exitoCard) + assertEquals("exit", exitoCard?.apparentEnglish) + assertEquals("salida", exitoCard?.realSpanishForApparent) + + val carpetasPlural = fcRepo.getBySurfaceForm("carpetas") + assertNotNull( + "False cognates should expand inflections like 'carpetas'", + carpetasPlural + ) + assertEquals("carpeta", carpetasPlural?.lemma) + + // 3. Verify curated lessons + val lessons = lessonRepo.getAll() + assertEquals("Should contain 8 curated lessons", 8, lessons.size) + val levels = lessons.map { it.level }.toSet() + assertEquals(setOf(0, 1, 2), levels) + + val firstLesson = lessonRepo.getById("lvl0_00_cognate_unlock") + assertNotNull(firstLesson) + assertEquals("The Cognate Unlock & Vowel Soundboard", firstLesson?.title) + assertEquals( + "El hotel moderno es excelente.", + firstLesson?.parseSentences()?.first()?.sentenceEs, + ) + + // 4. Verify cognate rules + val rules = rulesRepo.getAll() + assertEquals("Should contain 12 cognate rules", 12, rules.size) + val matchingRule = rulesRepo.findRuleForSpanishWord("situación") + assertNotNull(matchingRule) + assertEquals("tion_to_cion", matchingRule?.ruleId) + } + } +} diff --git a/data/seeds/BUILD.bazel b/data/seeds/BUILD.bazel index aaf04a5..565689d 100644 --- a/data/seeds/BUILD.bazel +++ b/data/seeds/BUILD.bazel @@ -3,6 +3,7 @@ package(default_visibility = ["//visibility:public"]) exports_files([ "cognate_rules_seed.json", "curated_lessons_seed.json", + "dictionary_seed.json", "false_cognates_seed.json", ]) @@ -11,6 +12,7 @@ filegroup( srcs = [ "cognate_rules_seed.json", "curated_lessons_seed.json", + "dictionary_seed.json", "false_cognates_seed.json", ], ) diff --git a/data/seeds/dictionary_seed.json b/data/seeds/dictionary_seed.json new file mode 100644 index 0000000..b204852 --- /dev/null +++ b/data/seeds/dictionary_seed.json @@ -0,0 +1,1568 @@ +[ + { + "surface_form": "el", + "lemma": "el", + "pos": "article", + "grammar_info": "masc. sing.", + "english_gloss": "the", + "is_cognate": false, + "frequency_rank": 1 + }, + { + "surface_form": "la", + "lemma": "el", + "pos": "article", + "grammar_info": "fem. sing.", + "english_gloss": "the", + "is_cognate": false, + "frequency_rank": 2 + }, + { + "surface_form": "de", + "lemma": "de", + "pos": "prep", + "grammar_info": null, + "english_gloss": "of / from", + "is_cognate": false, + "frequency_rank": 3 + }, + { + "surface_form": "que", + "lemma": "que", + "pos": "conj", + "grammar_info": null, + "english_gloss": "that / which", + "is_cognate": false, + "frequency_rank": 4 + }, + { + "surface_form": "y", + "lemma": "y", + "pos": "conj", + "grammar_info": null, + "english_gloss": "and", + "is_cognate": false, + "frequency_rank": 5 + }, + { + "surface_form": "a", + "lemma": "a", + "pos": "prep", + "grammar_info": null, + "english_gloss": "to / at", + "is_cognate": false, + "frequency_rank": 6 + }, + { + "surface_form": "en", + "lemma": "en", + "pos": "prep", + "grammar_info": null, + "english_gloss": "in / on / at", + "is_cognate": false, + "frequency_rank": 7 + }, + { + "surface_form": "un", + "lemma": "uno", + "pos": "article", + "grammar_info": "masc. sing.", + "english_gloss": "a / an", + "is_cognate": false, + "frequency_rank": 8 + }, + { + "surface_form": "una", + "lemma": "uno", + "pos": "article", + "grammar_info": "fem. sing.", + "english_gloss": "a / an", + "is_cognate": false, + "frequency_rank": 9 + }, + { + "surface_form": "ser", + "lemma": "ser", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to be", + "is_cognate": false, + "frequency_rank": 10 + }, + { + "surface_form": "es", + "lemma": "ser", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "is", + "is_cognate": false, + "frequency_rank": 11 + }, + { + "surface_form": "son", + "lemma": "ser", + "pos": "verb", + "grammar_info": "3rd pl. present", + "english_gloss": "are", + "is_cognate": false, + "frequency_rank": 12 + }, + { + "surface_form": "era", + "lemma": "ser", + "pos": "verb", + "grammar_info": "1st/3rd sing. imperfect", + "english_gloss": "was", + "is_cognate": false, + "frequency_rank": 13 + }, + { + "surface_form": "fue", + "lemma": "ser", + "pos": "verb", + "grammar_info": "3rd sing. preterite", + "english_gloss": "was", + "is_cognate": false, + "frequency_rank": 14 + }, + { + "surface_form": "se", + "lemma": "se", + "pos": "pron", + "grammar_info": "reflexive / indirect", + "english_gloss": "himself / herself / to him / to her", + "is_cognate": false, + "frequency_rank": 15 + }, + { + "surface_form": "no", + "lemma": "no", + "pos": "adv", + "grammar_info": null, + "english_gloss": "no / not", + "is_cognate": true, + "frequency_rank": 16 + }, + { + "surface_form": "haber", + "lemma": "haber", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to have (auxiliary) / to exist", + "is_cognate": false, + "frequency_rank": 17 + }, + { + "surface_form": "hay", + "lemma": "haber", + "pos": "verb", + "grammar_info": "present impersonal", + "english_gloss": "there is / there are", + "is_cognate": false, + "frequency_rank": 18 + }, + { + "surface_form": "por", + "lemma": "por", + "pos": "prep", + "grammar_info": null, + "english_gloss": "for / by / through", + "is_cognate": false, + "frequency_rank": 19 + }, + { + "surface_form": "con", + "lemma": "con", + "pos": "prep", + "grammar_info": null, + "english_gloss": "with", + "is_cognate": false, + "frequency_rank": 20 + }, + { + "surface_form": "para", + "lemma": "para", + "pos": "prep", + "grammar_info": null, + "english_gloss": "for / in order to", + "is_cognate": false, + "frequency_rank": 21 + }, + { + "surface_form": "como", + "lemma": "como", + "pos": "conj", + "grammar_info": null, + "english_gloss": "like / as", + "is_cognate": false, + "frequency_rank": 22 + }, + { + "surface_form": "estar", + "lemma": "estar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to be (state/location)", + "is_cognate": false, + "frequency_rank": 23 + }, + { + "surface_form": "está", + "lemma": "estar", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "is", + "is_cognate": false, + "frequency_rank": 24 + }, + { + "surface_form": "están", + "lemma": "estar", + "pos": "verb", + "grammar_info": "3rd pl. present", + "english_gloss": "are", + "is_cognate": false, + "frequency_rank": 25 + }, + { + "surface_form": "tener", + "lemma": "tener", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to have", + "is_cognate": false, + "frequency_rank": 26 + }, + { + "surface_form": "tiene", + "lemma": "tener", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "has", + "is_cognate": false, + "frequency_rank": 27 + }, + { + "surface_form": "tienen", + "lemma": "tener", + "pos": "verb", + "grammar_info": "3rd pl. present", + "english_gloss": "have", + "is_cognate": false, + "frequency_rank": 28 + }, + { + "surface_form": "le", + "lemma": "él", + "pos": "pron", + "grammar_info": "3rd sing. indirect object", + "english_gloss": "to him / to her", + "is_cognate": false, + "frequency_rank": 29 + }, + { + "surface_form": "lo", + "lemma": "él", + "pos": "pron", + "grammar_info": "3rd masc. sing. direct object", + "english_gloss": "it / him", + "is_cognate": false, + "frequency_rank": 30 + }, + { + "surface_form": "los", + "lemma": "el", + "pos": "article", + "grammar_info": "masc. pl.", + "english_gloss": "the", + "is_cognate": false, + "frequency_rank": 31 + }, + { + "surface_form": "las", + "lemma": "el", + "pos": "article", + "grammar_info": "fem. pl.", + "english_gloss": "the", + "is_cognate": false, + "frequency_rank": 32 + }, + { + "surface_form": "su", + "lemma": "su", + "pos": "adj", + "grammar_info": "possessive sing.", + "english_gloss": "his / her / their", + "is_cognate": false, + "frequency_rank": 33 + }, + { + "surface_form": "sus", + "lemma": "su", + "pos": "adj", + "grammar_info": "possessive pl.", + "english_gloss": "his / her / their", + "is_cognate": false, + "frequency_rank": 34 + }, + { + "surface_form": "pero", + "lemma": "pero", + "pos": "conj", + "grammar_info": null, + "english_gloss": "but", + "is_cognate": false, + "frequency_rank": 35 + }, + { + "surface_form": "más", + "lemma": "más", + "pos": "adv", + "grammar_info": null, + "english_gloss": "more", + "is_cognate": false, + "frequency_rank": 36 + }, + { + "surface_form": "hacer", + "lemma": "hacer", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to do / to make", + "is_cognate": false, + "frequency_rank": 37 + }, + { + "surface_form": "hace", + "lemma": "hacer", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "does / makes", + "is_cognate": false, + "frequency_rank": 38 + }, + { + "surface_form": "o", + "lemma": "o", + "pos": "conj", + "grammar_info": null, + "english_gloss": "or", + "is_cognate": false, + "frequency_rank": 39 + }, + { + "surface_form": "poder", + "lemma": "poder", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to be able to / can", + "is_cognate": false, + "frequency_rank": 40 + }, + { + "surface_form": "puede", + "lemma": "poder", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "can / is able to", + "is_cognate": false, + "frequency_rank": 41 + }, + { + "surface_form": "decir", + "lemma": "decir", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to say / to tell", + "is_cognate": false, + "frequency_rank": 42 + }, + { + "surface_form": "dice", + "lemma": "decir", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "says", + "is_cognate": false, + "frequency_rank": 43 + }, + { + "surface_form": "este", + "lemma": "este", + "pos": "adj", + "grammar_info": "demonstrative masc. sing.", + "english_gloss": "this", + "is_cognate": false, + "frequency_rank": 44 + }, + { + "surface_form": "esta", + "lemma": "este", + "pos": "adj", + "grammar_info": "demonstrative fem. sing.", + "english_gloss": "this", + "is_cognate": false, + "frequency_rank": 45 + }, + { + "surface_form": "ir", + "lemma": "ir", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to go", + "is_cognate": false, + "frequency_rank": 46 + }, + { + "surface_form": "va", + "lemma": "ir", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "goes / is going", + "is_cognate": false, + "frequency_rank": 47 + }, + { + "surface_form": "van", + "lemma": "ir", + "pos": "verb", + "grammar_info": "3rd pl. present", + "english_gloss": "go / are going", + "is_cognate": false, + "frequency_rank": 48 + }, + { + "surface_form": "otro", + "lemma": "otro", + "pos": "adj", + "grammar_info": "masc. sing.", + "english_gloss": "other / another", + "is_cognate": false, + "frequency_rank": 49 + }, + { + "surface_form": "otra", + "lemma": "otro", + "pos": "adj", + "grammar_info": "fem. sing.", + "english_gloss": "other / another", + "is_cognate": false, + "frequency_rank": 50 + }, + { + "surface_form": "al", + "lemma": "a el", + "pos": "prep", + "grammar_info": "contraction a + el", + "english_gloss": "to the", + "is_cognate": false, + "frequency_rank": 51 + }, + { + "surface_form": "del", + "lemma": "de el", + "pos": "prep", + "grammar_info": "contraction de + el", + "english_gloss": "of the", + "is_cognate": false, + "frequency_rank": 52 + }, + { + "surface_form": "cuando", + "lemma": "cuando", + "pos": "conj", + "grammar_info": null, + "english_gloss": "when", + "is_cognate": false, + "frequency_rank": 53 + }, + { + "surface_form": "muy", + "lemma": "muy", + "pos": "adv", + "grammar_info": null, + "english_gloss": "very", + "is_cognate": false, + "frequency_rank": 54 + }, + { + "surface_form": "sin", + "lemma": "sin", + "pos": "prep", + "grammar_info": null, + "english_gloss": "without", + "is_cognate": false, + "frequency_rank": 55 + }, + { + "surface_form": "sobre", + "lemma": "sobre", + "pos": "prep", + "grammar_info": null, + "english_gloss": "on / about", + "is_cognate": false, + "frequency_rank": 56 + }, + { + "surface_form": "entre", + "lemma": "entre", + "pos": "prep", + "grammar_info": null, + "english_gloss": "between / among", + "is_cognate": false, + "frequency_rank": 57 + }, + { + "surface_form": "también", + "lemma": "también", + "pos": "adv", + "grammar_info": null, + "english_gloss": "also / too", + "is_cognate": false, + "frequency_rank": 58 + }, + { + "surface_form": "me", + "lemma": "yo", + "pos": "pron", + "grammar_info": "1st sing. object", + "english_gloss": "me / to me", + "is_cognate": true, + "frequency_rank": 59 + }, + { + "surface_form": "te", + "lemma": "tú", + "pos": "pron", + "grammar_info": "2nd sing. object", + "english_gloss": "you / to you", + "is_cognate": false, + "frequency_rank": 60 + }, + { + "surface_form": "nos", + "lemma": "nosotros", + "pos": "pron", + "grammar_info": "1st pl. object", + "english_gloss": "us / to us", + "is_cognate": false, + "frequency_rank": 61 + }, + { + "surface_form": "hotel", + "lemma": "hotel", + "pos": "noun", + "grammar_info": "masc. sing.", + "english_gloss": "hotel", + "is_cognate": true, + "frequency_rank": 450 + }, + { + "surface_form": "moderno", + "lemma": "moderno", + "pos": "adj", + "grammar_info": "masc. sing.", + "english_gloss": "modern", + "is_cognate": true, + "frequency_rank": 600 + }, + { + "surface_form": "moderna", + "lemma": "moderno", + "pos": "adj", + "grammar_info": "fem. sing.", + "english_gloss": "modern", + "is_cognate": true, + "frequency_rank": 610 + }, + { + "surface_form": "excelente", + "lemma": "excelente", + "pos": "adj", + "grammar_info": "sing.", + "english_gloss": "excellent", + "is_cognate": true, + "frequency_rank": 700 + }, + { + "surface_form": "música", + "lemma": "música", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "music", + "is_cognate": true, + "frequency_rank": 250 + }, + { + "surface_form": "clásica", + "lemma": "clásico", + "pos": "adj", + "grammar_info": "fem. sing.", + "english_gloss": "classical", + "is_cognate": true, + "frequency_rank": 800 + }, + { + "surface_form": "clásico", + "lemma": "clásico", + "pos": "adj", + "grammar_info": "masc. sing.", + "english_gloss": "classic", + "is_cognate": true, + "frequency_rank": 790 + }, + { + "surface_form": "fantástica", + "lemma": "fantástico", + "pos": "adj", + "grammar_info": "fem. sing.", + "english_gloss": "fantastic", + "is_cognate": true, + "frequency_rank": 950 + }, + { + "surface_form": "fantástico", + "lemma": "fantástico", + "pos": "adj", + "grammar_info": "masc. sing.", + "english_gloss": "fantastic", + "is_cognate": true, + "frequency_rank": 940 + }, + { + "surface_form": "profesor", + "lemma": "profesor", + "pos": "noun", + "grammar_info": "masc. sing.", + "english_gloss": "professor / teacher", + "is_cognate": true, + "frequency_rank": 400 + }, + { + "surface_form": "profesora", + "lemma": "profesor", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "professor / teacher", + "is_cognate": true, + "frequency_rank": 420 + }, + { + "surface_form": "explicar", + "lemma": "explicar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to explain", + "is_cognate": true, + "frequency_rank": 320 + }, + { + "surface_form": "explica", + "lemma": "explicar", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "explains", + "is_cognate": true, + "frequency_rank": 325 + }, + { + "surface_form": "situación", + "lemma": "situación", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "situation", + "is_cognate": true, + "frequency_rank": 180 + }, + { + "surface_form": "universidad", + "lemma": "universidad", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "university", + "is_cognate": true, + "frequency_rank": 350 + }, + { + "surface_form": "organizar", + "lemma": "organizar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to organize", + "is_cognate": true, + "frequency_rank": 650 + }, + { + "surface_form": "organiza", + "lemma": "organizar", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "organizes", + "is_cognate": true, + "frequency_rank": 655 + }, + { + "surface_form": "conferencia", + "lemma": "conferencia", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "conference", + "is_cognate": true, + "frequency_rank": 820 + }, + { + "surface_form": "internacional", + "lemma": "internacional", + "pos": "adj", + "grammar_info": "sing.", + "english_gloss": "international", + "is_cognate": true, + "frequency_rank": 410 + }, + { + "surface_form": "director", + "lemma": "director", + "pos": "noun", + "grammar_info": "masc. sing.", + "english_gloss": "director", + "is_cognate": true, + "frequency_rank": 380 + }, + { + "surface_form": "directora", + "lemma": "director", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "director", + "is_cognate": true, + "frequency_rank": 390 + }, + { + "surface_form": "presentar", + "lemma": "presentar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to present", + "is_cognate": true, + "frequency_rank": 340 + }, + { + "surface_form": "presenta", + "lemma": "presentar", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "presents", + "is_cognate": true, + "frequency_rank": 345 + }, + { + "surface_form": "proyecto", + "lemma": "proyecto", + "pos": "noun", + "grammar_info": "masc. sing.", + "english_gloss": "project", + "is_cognate": true, + "frequency_rank": 310 + }, + { + "surface_form": "importante", + "lemma": "importante", + "pos": "adj", + "grammar_info": "sing.", + "english_gloss": "important", + "is_cognate": true, + "frequency_rank": 160 + }, + { + "surface_form": "importantes", + "lemma": "importante", + "pos": "adj", + "grammar_info": "plural", + "english_gloss": "important", + "is_cognate": true, + "frequency_rank": 165 + }, + { + "surface_form": "solución", + "lemma": "solución", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "solution", + "is_cognate": true, + "frequency_rank": 520 + }, + { + "surface_form": "soluciones", + "lemma": "solución", + "pos": "noun", + "grammar_info": "fem. pl.", + "english_gloss": "solutions", + "is_cognate": true, + "frequency_rank": 525 + }, + { + "surface_form": "posible", + "lemma": "posible", + "pos": "adj", + "grammar_info": "sing.", + "english_gloss": "possible", + "is_cognate": true, + "frequency_rank": 210 + }, + { + "surface_form": "posibles", + "lemma": "posible", + "pos": "adj", + "grammar_info": "plural", + "english_gloss": "possible", + "is_cognate": true, + "frequency_rank": 215 + }, + { + "surface_form": "requerir", + "lemma": "requerir", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to require", + "is_cognate": true, + "frequency_rank": 730 + }, + { + "surface_form": "requieren", + "lemma": "requerir", + "pos": "verb", + "grammar_info": "3rd pl. present", + "english_gloss": "require", + "is_cognate": true, + "frequency_rank": 735 + }, + { + "surface_form": "paciencia", + "lemma": "paciencia", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "patience", + "is_cognate": true, + "frequency_rank": 680 + }, + { + "surface_form": "sistema", + "lemma": "sistema", + "pos": "noun", + "grammar_info": "masc. sing.", + "english_gloss": "system", + "is_cognate": true, + "frequency_rank": 270 + }, + { + "surface_form": "funcionar", + "lemma": "funcionar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to function / to work", + "is_cognate": true, + "frequency_rank": 490 + }, + { + "surface_form": "funciona", + "lemma": "funcionar", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "functions / works", + "is_cognate": true, + "frequency_rank": 495 + }, + { + "surface_form": "correctamente", + "lemma": "correcto", + "pos": "adv", + "grammar_info": null, + "english_gloss": "correctly", + "is_cognate": true, + "frequency_rank": 780 + }, + { + "surface_form": "necesitar", + "lemma": "necesitar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to need", + "is_cognate": false, + "frequency_rank": 190 + }, + { + "surface_form": "necesita", + "lemma": "necesitar", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "needs", + "is_cognate": false, + "frequency_rank": 195 + }, + { + "surface_form": "equipo", + "lemma": "equipo", + "pos": "noun", + "grammar_info": "masc. sing.", + "english_gloss": "team / equipment", + "is_cognate": false, + "frequency_rank": 330 + }, + { + "surface_form": "información", + "lemma": "información", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "information", + "is_cognate": true, + "frequency_rank": 220 + }, + { + "surface_form": "buscar", + "lemma": "buscar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to look for / to search", + "is_cognate": false, + "frequency_rank": 240 + }, + { + "surface_form": "buscamos", + "lemma": "buscar", + "pos": "verb", + "grammar_info": "1st pl. present", + "english_gloss": "we look for", + "is_cognate": false, + "frequency_rank": 245 + }, + { + "surface_form": "buscaba", + "lemma": "buscar", + "pos": "verb", + "grammar_info": "1st/3rd sing. imperfect", + "english_gloss": "was looking for", + "is_cognate": false, + "frequency_rank": 248 + }, + { + "surface_form": "oportunidad", + "lemma": "oportunidad", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "opportunity", + "is_cognate": true, + "frequency_rank": 360 + }, + { + "surface_form": "diferente", + "lemma": "diferente", + "pos": "adj", + "grammar_info": "sing.", + "english_gloss": "different", + "is_cognate": true, + "frequency_rank": 280 + }, + { + "surface_form": "preferir", + "lemma": "preferir", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to prefer", + "is_cognate": true, + "frequency_rank": 580 + }, + { + "surface_form": "prefieren", + "lemma": "preferir", + "pos": "verb", + "grammar_info": "3rd pl. present", + "english_gloss": "they prefer", + "is_cognate": true, + "frequency_rank": 585 + }, + { + "surface_form": "estudiar", + "lemma": "estudiar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to study", + "is_cognate": true, + "frequency_rank": 470 + }, + { + "surface_form": "biblioteca", + "lemma": "biblioteca", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "library", + "is_cognate": false, + "frequency_rank": 850 + }, + { + "surface_form": "central", + "lemma": "central", + "pos": "adj", + "grammar_info": "sing.", + "english_gloss": "central", + "is_cognate": true, + "frequency_rank": 510 + }, + { + "surface_form": "médico", + "lemma": "médico", + "pos": "noun", + "grammar_info": "masc. sing.", + "english_gloss": "doctor / physician", + "is_cognate": false, + "frequency_rank": 440 + }, + { + "surface_form": "recomendar", + "lemma": "recomendar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to recommend", + "is_cognate": true, + "frequency_rank": 720 + }, + { + "surface_form": "recomienda", + "lemma": "recomendar", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "recommends", + "is_cognate": true, + "frequency_rank": 725 + }, + { + "surface_form": "totalmente", + "lemma": "total", + "pos": "adv", + "grammar_info": null, + "english_gloss": "totally", + "is_cognate": true, + "frequency_rank": 640 + }, + { + "surface_form": "querer", + "lemma": "querer", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to want", + "is_cognate": false, + "frequency_rank": 80 + }, + { + "surface_form": "queremos", + "lemma": "querer", + "pos": "verb", + "grammar_info": "1st pl. present", + "english_gloss": "we want", + "is_cognate": false, + "frequency_rank": 85 + }, + { + "surface_form": "calma", + "lemma": "calma", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "calm", + "is_cognate": true, + "frequency_rank": 760 + }, + { + "surface_form": "trabajar", + "lemma": "trabajar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to work", + "is_cognate": false, + "frequency_rank": 260 + }, + { + "surface_form": "trabaja", + "lemma": "trabajar", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "works", + "is_cognate": false, + "frequency_rank": 265 + }, + { + "surface_form": "oficina", + "lemma": "oficina", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "office", + "is_cognate": false, + "frequency_rank": 430 + }, + { + "surface_form": "activo", + "lemma": "activo", + "pos": "adj", + "grammar_info": "masc. sing.", + "english_gloss": "active", + "is_cognate": true, + "frequency_rank": 870 + }, + { + "surface_form": "activa", + "lemma": "activo", + "pos": "adj", + "grammar_info": "fem. sing.", + "english_gloss": "active", + "is_cognate": true, + "frequency_rank": 875 + }, + { + "surface_form": "guardar", + "lemma": "guardar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to keep / to save", + "is_cognate": false, + "frequency_rank": 540 + }, + { + "surface_form": "guarda", + "lemma": "guardar", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "keeps / saves", + "is_cognate": false, + "frequency_rank": 545 + }, + { + "surface_form": "todos", + "lemma": "todo", + "pos": "adj", + "grammar_info": "masc. pl.", + "english_gloss": "all", + "is_cognate": false, + "frequency_rank": 70 + }, + { + "surface_form": "contrato", + "lemma": "contrato", + "pos": "noun", + "grammar_info": "masc. sing.", + "english_gloss": "contract", + "is_cognate": true, + "frequency_rank": 630 + }, + { + "surface_form": "contratos", + "lemma": "contrato", + "pos": "noun", + "grammar_info": "masc. pl.", + "english_gloss": "contracts", + "is_cognate": true, + "frequency_rank": 635 + }, + { + "surface_form": "azul", + "lemma": "azul", + "pos": "adj", + "grammar_info": "sing.", + "english_gloss": "blue", + "is_cognate": false, + "frequency_rank": 810 + }, + { + "surface_form": "ruido", + "lemma": "ruido", + "pos": "noun", + "grammar_info": "masc. sing.", + "english_gloss": "noise", + "is_cognate": false, + "frequency_rank": 740 + }, + { + "surface_form": "decidir", + "lemma": "decidir", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to decide", + "is_cognate": true, + "frequency_rank": 370 + }, + { + "surface_form": "decide", + "lemma": "decidir", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "decides", + "is_cognate": true, + "frequency_rank": 375 + }, + { + "surface_form": "llover", + "lemma": "llover", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to rain", + "is_cognate": false, + "frequency_rank": 920 + }, + { + "surface_form": "llovía", + "lemma": "llover", + "pos": "verb", + "grammar_info": "3rd sing. imperfect", + "english_gloss": "it was raining", + "is_cognate": false, + "frequency_rank": 925 + }, + { + "surface_form": "intensamente", + "lemma": "intenso", + "pos": "adv", + "grammar_info": null, + "english_gloss": "intensely", + "is_cognate": true, + "frequency_rank": 960 + }, + { + "surface_form": "entrar", + "lemma": "entrar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to enter", + "is_cognate": true, + "frequency_rank": 290 + }, + { + "surface_form": "entró", + "lemma": "entrar", + "pos": "verb", + "grammar_info": "3rd sing. preterite", + "english_gloss": "entered", + "is_cognate": true, + "frequency_rank": 295 + }, + { + "surface_form": "pequeño", + "lemma": "pequeño", + "pos": "adj", + "grammar_info": "masc. sing.", + "english_gloss": "small", + "is_cognate": false, + "frequency_rank": 200 + }, + { + "surface_form": "pequeña", + "lemma": "pequeño", + "pos": "adj", + "grammar_info": "fem. sing.", + "english_gloss": "small", + "is_cognate": false, + "frequency_rank": 205 + }, + { + "surface_form": "barrio", + "lemma": "barrio", + "pos": "noun", + "grammar_info": "masc. sing.", + "english_gloss": "neighborhood", + "is_cognate": false, + "frequency_rank": 660 + }, + { + "surface_form": "novela", + "lemma": "novela", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "novel", + "is_cognate": true, + "frequency_rank": 770 + }, + { + "surface_form": "histórico", + "lemma": "histórico", + "pos": "adj", + "grammar_info": "masc. sing.", + "english_gloss": "historical", + "is_cognate": true, + "frequency_rank": 690 + }, + { + "surface_form": "histórica", + "lemma": "histórico", + "pos": "adj", + "grammar_info": "fem. sing.", + "english_gloss": "historical", + "is_cognate": true, + "frequency_rank": 695 + }, + { + "surface_form": "encontrar", + "lemma": "encontrar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to find", + "is_cognate": false, + "frequency_rank": 140 + }, + { + "surface_form": "encontró", + "lemma": "encontrar", + "pos": "verb", + "grammar_info": "3rd sing. preterite", + "english_gloss": "found", + "is_cognate": false, + "frequency_rank": 145 + }, + { + "surface_form": "carta", + "lemma": "carta", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "letter / card", + "is_cognate": false, + "frequency_rank": 480 + }, + { + "surface_form": "antiguo", + "lemma": "antiguo", + "pos": "adj", + "grammar_info": "masc. sing.", + "english_gloss": "ancient / antique", + "is_cognate": false, + "frequency_rank": 590 + }, + { + "surface_form": "antigua", + "lemma": "antiguo", + "pos": "adj", + "grammar_info": "fem. sing.", + "english_gloss": "ancient / antique", + "is_cognate": false, + "frequency_rank": 595 + }, + { + "surface_form": "página", + "lemma": "página", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "page", + "is_cognate": true, + "frequency_rank": 710 + }, + { + "surface_form": "páginas", + "lemma": "página", + "pos": "noun", + "grammar_info": "fem. pl.", + "english_gloss": "pages", + "is_cognate": true, + "frequency_rank": 715 + }, + { + "surface_form": "lectura", + "lemma": "lectura", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "reading", + "is_cognate": false, + "frequency_rank": 890 + }, + { + "surface_form": "documento", + "lemma": "documento", + "pos": "noun", + "grammar_info": "masc. sing.", + "english_gloss": "document", + "is_cognate": true, + "frequency_rank": 530 + }, + { + "surface_form": "revelar", + "lemma": "revelar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to reveal", + "is_cognate": true, + "frequency_rank": 840 + }, + { + "surface_form": "reveló", + "lemma": "revelar", + "pos": "verb", + "grammar_info": "3rd sing. preterite", + "english_gloss": "revealed", + "is_cognate": true, + "frequency_rank": 845 + }, + { + "surface_form": "secreto", + "lemma": "secreto", + "pos": "noun", + "grammar_info": "masc. sing.", + "english_gloss": "secret", + "is_cognate": true, + "frequency_rank": 620 + }, + { + "surface_form": "familiar", + "lemma": "familiar", + "pos": "adj", + "grammar_info": "sing.", + "english_gloss": "family / familiar", + "is_cognate": true, + "frequency_rank": 570 + }, + { + "surface_form": "extraordinario", + "lemma": "extraordinario", + "pos": "adj", + "grammar_info": "masc. sing.", + "english_gloss": "extraordinary", + "is_cognate": true, + "frequency_rank": 980 + }, + { + "surface_form": "éxito", + "lemma": "éxito", + "pos": "noun", + "grammar_info": "masc. sing.", + "english_gloss": "success", + "is_cognate": false, + "frequency_rank": 460 + }, + { + "surface_form": "éxitos", + "lemma": "éxito", + "pos": "noun", + "grammar_info": "masc. pl.", + "english_gloss": "successes", + "is_cognate": false, + "frequency_rank": 465 + }, + { + "surface_form": "carpeta", + "lemma": "carpeta", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "folder", + "is_cognate": false, + "frequency_rank": 910 + }, + { + "surface_form": "carpetas", + "lemma": "carpeta", + "pos": "noun", + "grammar_info": "fem. pl.", + "english_gloss": "folders", + "is_cognate": false, + "frequency_rank": 915 + }, + { + "surface_form": "soportar", + "lemma": "soportar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to tolerate / to endure", + "is_cognate": false, + "frequency_rank": 830 + }, + { + "surface_form": "soporta", + "lemma": "soportar", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "tolerates / stands", + "is_cognate": false, + "frequency_rank": 835 + }, + { + "surface_form": "embarazada", + "lemma": "embarazada", + "pos": "adj", + "grammar_info": "fem. sing.", + "english_gloss": "pregnant", + "is_cognate": false, + "frequency_rank": 930 + }, + { + "surface_form": "actualmente", + "lemma": "actual", + "pos": "adv", + "grammar_info": null, + "english_gloss": "currently", + "is_cognate": false, + "frequency_rank": 505 + }, + { + "surface_form": "realizar", + "lemma": "realizar", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to carry out / to fulfill", + "is_cognate": false, + "frequency_rank": 300 + }, + { + "surface_form": "realizan", + "lemma": "realizar", + "pos": "verb", + "grammar_info": "3rd pl. present", + "english_gloss": "carry out / fulfill", + "is_cognate": false, + "frequency_rank": 305 + }, + { + "surface_form": "sensible", + "lemma": "sensible", + "pos": "adj", + "grammar_info": "sing.", + "english_gloss": "sensitive", + "is_cognate": false, + "frequency_rank": 860 + }, + { + "surface_form": "asistir", + "lemma": "asistir", + "pos": "verb", + "grammar_info": "infinitive", + "english_gloss": "to attend", + "is_cognate": false, + "frequency_rank": 670 + }, + { + "surface_form": "asiste", + "lemma": "asistir", + "pos": "verb", + "grammar_info": "3rd sing. present", + "english_gloss": "attends", + "is_cognate": false, + "frequency_rank": 675 + }, + { + "surface_form": "librería", + "lemma": "librería", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "bookstore", + "is_cognate": false, + "frequency_rank": 970 + }, + { + "surface_form": "noticia", + "lemma": "noticia", + "pos": "noun", + "grammar_info": "fem. sing.", + "english_gloss": "news item", + "is_cognate": false, + "frequency_rank": 550 + }, + { + "surface_form": "noticias", + "lemma": "noticia", + "pos": "noun", + "grammar_info": "fem. pl.", + "english_gloss": "news", + "is_cognate": false, + "frequency_rank": 555 + } +] diff --git a/tools/preprocessor/BUILD.bazel b/tools/preprocessor/BUILD.bazel index c3c2d73..3423ed1 100644 --- a/tools/preprocessor/BUILD.bazel +++ b/tools/preprocessor/BUILD.bazel @@ -29,6 +29,7 @@ kt_jvm_test( data = [ "//data/seeds:cognate_rules_seed.json", "//data/seeds:curated_lessons_seed.json", + "//data/seeds:dictionary_seed.json", "//data/seeds:false_cognates_seed.json", ], test_class = "com.anysoftkeyboard.calque.preprocessor.PreprocessorTest", diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/DatabaseOptimizer.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/DatabaseOptimizer.kt new file mode 100644 index 0000000..84a2316 --- /dev/null +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/DatabaseOptimizer.kt @@ -0,0 +1,9 @@ +package com.anysoftkeyboard.calque.preprocessor + +/** Performs SQLite database optimization and compaction via ANALYZE and VACUUM. */ +fun SeedDatabase.optimize() { + connection.createStatement().use { stmt -> + stmt.execute("ANALYZE") + stmt.execute("VACUUM") + } +} diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/DictionarySeedProcessor.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/DictionarySeedProcessor.kt new file mode 100644 index 0000000..a5c6632 --- /dev/null +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/DictionarySeedProcessor.kt @@ -0,0 +1,58 @@ +package com.anysoftkeyboard.calque.preprocessor + +import java.io.InputStream +import java.io.InputStreamReader +import java.nio.charset.StandardCharsets +import java.util.Locale +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.booleanOrNull +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.intOrNull +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive + +/** Seeds the dictionary table from a structured JSON definition. */ +object DictionarySeedProcessor { + + /** + * Reads dictionary entries from [inputStream] and inserts them into [database]. + * + * @return Count of dictionary entries inserted. + */ + fun processDictionarySeed(inputStream: InputStream, database: SeedDatabase): Int { + val rows = parseDictionary(inputStream) + database.insertDictionaryBatch(rows) + return rows.size + } + + /** Parses the dictionary seed JSON stream into a list of [DictionaryRow] objects. */ + fun parseDictionary(inputStream: InputStream): List { + val jsonText = InputStreamReader(inputStream, StandardCharsets.UTF_8).use { it.readText() } + val array = Json.parseToJsonElement(jsonText).jsonArray + return array.mapNotNull { parseDictionaryElement(it.jsonObject) } + } + + private fun parseDictionaryElement(obj: JsonObject): DictionaryRow? { + val surfaceForm = + obj["surface_form"]?.jsonPrimitive?.contentOrNull?.lowercase(Locale.ROOT) ?: return null + val lemma = + obj["lemma"]?.jsonPrimitive?.contentOrNull?.lowercase(Locale.ROOT) ?: surfaceForm + val pos = obj["pos"]?.jsonPrimitive?.contentOrNull ?: "noun" + val grammarInfo = obj["grammar_info"]?.jsonPrimitive?.contentOrNull + val englishGloss = obj["english_gloss"]?.jsonPrimitive?.contentOrNull.orEmpty() + val isCognate = obj["is_cognate"]?.jsonPrimitive?.booleanOrNull ?: false + val frequencyRank = obj["frequency_rank"]?.jsonPrimitive?.intOrNull + + return DictionaryRow( + surfaceForm = surfaceForm, + lemma = lemma, + pos = pos, + grammarInfo = grammarInfo, + englishGloss = englishGloss, + isCognate = isCognate, + frequencyRank = frequencyRank, + ) + } +} diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Main.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Main.kt index 586fccb..7fcd465 100644 --- a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Main.kt +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Main.kt @@ -7,34 +7,23 @@ import kotlin.system.exitProcess fun main(args: Array) { val options = parseCommandLineArgs(args) - val dbPath = options["--db"] - if (dbPath == null) { - System.err.println( - """ - Usage: preprocessor --db [options] - - Options: - --kaikki Path to Kaikki Spanish JSONL dump - --sublex Path to SUBTLEX-ESP frequency file - --false-cognates Path to false_cognates_seed.json - --curated-lessons Path to curated_lessons_seed.json - --tatoeba Path to Tatoeba Spanish-English TSV file - --cognate-rules Path to cognate_rules_seed.json - """ - .trimIndent(), - ) - exitProcess(1) + val rawDbPath = options["--db"] + if (rawDbPath == null) { + printUsageAndExit() + return } + val dbFile = resolveFile(rawDbPath) ?: File(rawDbPath) val config = PipelineConfig( - dbFile = File(dbPath), - kaikkiFile = options["--kaikki"]?.let { File(it) }, - sublexFile = options["--sublex"]?.let { File(it) }, - falseCognatesFile = options["--false-cognates"]?.let { File(it) }, - curatedLessonsFile = options["--curated-lessons"]?.let { File(it) }, - tatoebaFile = options["--tatoeba"]?.let { File(it) }, - cognateRulesFile = options["--cognate-rules"]?.let { File(it) }, + dbFile = dbFile, + kaikkiFile = resolveFile(options["--kaikki"]), + sublexFile = resolveFile(options["--sublex"]), + dictionarySeedFile = resolveFile(options["--dictionary-seed"]), + falseCognatesFile = resolveFile(options["--false-cognates"]), + curatedLessonsFile = resolveFile(options["--curated-lessons"]), + tatoebaFile = resolveFile(options["--tatoeba"]), + cognateRulesFile = resolveFile(options["--cognate-rules"]), ) println("Starting Calque Data Ingestion Pipeline...") @@ -52,6 +41,33 @@ fun main(args: Array) { println(" - Cognate transformer rules seeded: ${summary.cognateRuleRows}") } +private fun printUsageAndExit() { + System.err.println( + """ + Usage: preprocessor --db [options] + + Options: + --kaikki Path to Kaikki Spanish JSONL dump + --sublex Path to SUBTLEX-ESP frequency file + --dictionary-seed Path to dictionary_seed.json + --false-cognates Path to false_cognates_seed.json + --curated-lessons Path to curated_lessons_seed.json + --tatoeba Path to Tatoeba Spanish-English TSV file + --cognate-rules Path to cognate_rules_seed.json + """ + .trimIndent(), + ) + exitProcess(1) +} + +private fun resolveFile(path: String?): File? { + if (path == null) return null + val file = File(path) + if (file.isAbsolute) return file + val workingDir = System.getenv("BUILD_WORKING_DIRECTORY") + return if (workingDir != null) File(workingDir, path) else file +} + private fun parseCommandLineArgs(args: Array): Map { val options = mutableMapOf() var i = 0 diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorPipeline.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorPipeline.kt index f57d3f2..209c87d 100644 --- a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorPipeline.kt +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorPipeline.kt @@ -8,6 +8,7 @@ data class PipelineConfig( val dbFile: File, val kaikkiFile: File? = null, val sublexFile: File? = null, + val dictionarySeedFile: File? = null, val falseCognatesFile: File? = null, val curatedLessonsFile: File? = null, val tatoebaFile: File? = null, @@ -35,10 +36,13 @@ class PreprocessorPipeline(private val config: PipelineConfig) { /** Executes the pipeline stages on an already opened [SeedDatabase]. */ fun executeWithDatabase(database: SeedDatabase): PipelineSummary { val sublexFrequencies = loadSublexFrequencies() - val dictCount = processKaikki(database, sublexFrequencies) + val dictSeedCount = processDictionarySeed(database) + val kaikkiCount = processKaikki(database, sublexFrequencies) + val dictCount = dictSeedCount + kaikkiCount val falseCognatesCount = processFalseCognates(database) val lessonsCount = processCuratedLessons(database) + processTatoeba(database) val rulesCount = processRules(database) + database.optimize() return PipelineSummary( dictionaryRows = dictCount, @@ -54,6 +58,14 @@ class PreprocessorPipeline(private val config: PipelineConfig) { return FileInputStream(file).use { SublexReader.readFrequencies(it) } } + private fun processDictionarySeed(database: SeedDatabase): Long { + val file = config.dictionarySeedFile + if (file == null || !file.exists()) return 0L + return FileInputStream(file).use { + DictionarySeedProcessor.processDictionarySeed(it, database).toLong() + } + } + private fun processKaikki(database: SeedDatabase, sublexFrequencies: Map): Long { val file = config.kaikkiFile if (file == null || !file.exists()) return 0L diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt index 4b9b142..e12d60a 100644 --- a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt @@ -13,7 +13,7 @@ import java.sql.Types * * Implements the table schemas defined in docs/DATA.md. */ -class SeedDatabase(private val connection: Connection) : AutoCloseable { +class SeedDatabase(internal val connection: Connection) : AutoCloseable { init { DatabaseBinders.createTables(connection) diff --git a/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt b/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt index 0088fa4..c14152e 100644 --- a/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt +++ b/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt @@ -200,7 +200,7 @@ class PreprocessorTest { "base", "fantastic", true, - 300 + 300, ), ), ) @@ -352,6 +352,59 @@ class PreprocessorTest { } } + @Test + fun dictionarySeedProcessor_parsesAndInsertsEntries() { + val inMemoryDb = SeedDatabase.open() + inMemoryDb.use { db -> + val sampleJson = + """ + [ + { + "surface_form": "hotel", + "lemma": "hotel", + "pos": "noun", + "grammar_info": "masc. sing.", + "english_gloss": "hotel", + "is_cognate": true, + "frequency_rank": 500 + } + ] + """ + .trimIndent() + + val count = + DictionarySeedProcessor.processDictionarySeed(sampleJson.byteInputStream(), db) + assertEquals(1, count) + + val hotel = db.lookupWord("hotel") + assertNotNull(hotel) + assertTrue(hotel?.isCognate == true) + assertEquals(500, hotel?.frequencyRank) + } + } + + @Test + fun curatedDictionarySeed_isValidAndPopulatesDatabase() { + val seedFile = File("data/seeds/dictionary_seed.json") + assertTrue("Expected seed file at ${seedFile.path}", seedFile.exists()) + + SeedDatabase.open().use { db -> + val insertedCount = + FileInputStream(seedFile).use { stream -> + DictionarySeedProcessor.processDictionarySeed(stream, db) + } + assertTrue("Expected >= 100 entries, found $insertedCount", insertedCount >= 100) + + val hotelInfo = db.lookupWord("hotel") + assertNotNull(hotelInfo) + assertTrue(hotelInfo?.isCognate == true) + + val esInfo = db.lookupWord("es") + assertNotNull(esInfo) + assertEquals("3rd sing. present", esInfo?.grammar) + } + } + @Test fun pipeline_executesEndToEndSuccessfully() { val tempDir = createTempDirectory("calque_pipeline_test").toFile() @@ -366,6 +419,7 @@ class PreprocessorTest { val kaikkiFile = File(tempDir, "kaikki.jsonl") kaikkiFile.writeText("$sampleKaikki\n", StandardCharsets.UTF_8) + val dictionarySeedFile = File("data/seeds/dictionary_seed.json") val falseCognatesSeedFile = File("data/seeds/false_cognates_seed.json") val curatedLessonsSeedFile = File("data/seeds/curated_lessons_seed.json") val cognateRulesSeedFile = File("data/seeds/cognate_rules_seed.json") @@ -375,6 +429,7 @@ class PreprocessorTest { dbFile = dbFile, kaikkiFile = kaikkiFile, sublexFile = sublexFile, + dictionarySeedFile = dictionarySeedFile, falseCognatesFile = falseCognatesSeedFile, curatedLessonsFile = curatedLessonsSeedFile, cognateRulesFile = cognateRulesSeedFile, @@ -384,9 +439,9 @@ class PreprocessorTest { val summary = pipeline.execute() assertTrue(dbFile.exists()) - assertTrue(summary.dictionaryRows >= 2) - // 25 base lemmas + "carpetas" plural from Kaikki = 26 rows - assertEquals(26, summary.falseCognateRows) + assertTrue(summary.dictionaryRows >= 100) + // 25 base lemmas + "carpetas" plural from Kaikki/dictionary = at least 26 rows + assertTrue(summary.falseCognateRows >= 26) assertEquals(8, summary.curatedLessonRows) assertEquals(12, summary.cognateRuleRows)