diff --git a/data/seeds/BUILD.bazel b/data/seeds/BUILD.bazel index 793b425..aaf04a5 100644 --- a/data/seeds/BUILD.bazel +++ b/data/seeds/BUILD.bazel @@ -2,6 +2,7 @@ package(default_visibility = ["//visibility:public"]) exports_files([ "cognate_rules_seed.json", + "curated_lessons_seed.json", "false_cognates_seed.json", ]) @@ -9,6 +10,7 @@ filegroup( name = "seed_files", srcs = [ "cognate_rules_seed.json", + "curated_lessons_seed.json", "false_cognates_seed.json", ], ) diff --git a/data/seeds/curated_lessons_seed.json b/data/seeds/curated_lessons_seed.json new file mode 100644 index 0000000..31fbc79 --- /dev/null +++ b/data/seeds/curated_lessons_seed.json @@ -0,0 +1,336 @@ +[ + { + "lesson_id": "lvl0_00_cognate_unlock", + "level": 0, + "sequence_order": 0, + "title": "The Cognate Unlock & Vowel Soundboard", + "target_concept": "Recognize shared Latin roots and steady five-vowel Spanish pronunciation in your very first sentences.", + "content": { + "sentences": [ + { + "sentence_es": "El hotel moderno es excelente.", + "translation_en": "The modern hotel is excellent.", + "interlinear_gloss": [ + { "es": "El", "en": "the" }, + { "es": "hotel", "en": "hotel" }, + { "es": "moderno", "en": "modern" }, + { "es": "es", "en": "is" }, + { "es": "excelente", "en": "excellent" } + ], + "grammar_callout": "Four of these five words are transparent cognates. Only 'el' (the) and 'es' (is) are structural glue." + }, + { + "sentence_es": "La música clásica es fantástica.", + "translation_en": "Classical music is fantastic.", + "interlinear_gloss": [ + { "es": "La", "en": "the" }, + { "es": "música", "en": "music" }, + { "es": "clásica", "en": "classical" }, + { "es": "es", "en": "is" }, + { "es": "fantástica", "en": "fantastic" } + ], + "grammar_callout": "Spanish uses 'la' before general feminine nouns like 'la música' even when English omits 'the'." + } + ] + } + }, + { + "lesson_id": "lvl0_01_svo_cognates", + "level": 0, + "sequence_order": 1, + "title": "Basic Subject-Verb-Object with Cognates", + "target_concept": "Decode standard Subject-Verb-Object clauses where suffix bridges (-ción, -dad) unlock the nouns.", + "content": { + "sentences": [ + { + "sentence_es": "El profesor explica la situación.", + "translation_en": "The professor explains the situation.", + "interlinear_gloss": [ + { "es": "El", "en": "the" }, + { "es": "profesor", "en": "professor" }, + { "es": "explica", "en": "explains" }, + { "es": "la", "en": "the" }, + { "es": "situación", "en": "situation" } + ], + "grammar_callout": "The suffix -ción always maps to English -tion and is always feminine (la situación)." + }, + { + "sentence_es": "La universidad organiza una conferencia internacional.", + "translation_en": "The university organizes an international conference.", + "interlinear_gloss": [ + { "es": "La", "en": "the" }, + { "es": "universidad", "en": "university" }, + { "es": "organiza", "en": "organizes" }, + { "es": "una", "en": "an" }, + { "es": "conferencia", "en": "conference" }, + { "es": "internacional", "en": "international" } + ], + "grammar_callout": "English -ty maps to -dad (universidad) and -nce maps to -ncia (conferencia)." + } + ] + } + }, + { + "lesson_id": "lvl0_02_adjectives_after_nouns", + "level": 0, + "sequence_order": 2, + "title": "Adjectives After Nouns", + "target_concept": "Spanish names the thing first and describes it second: Noun + Adjective.", + "content": { + "sentences": [ + { + "sentence_es": "El director presenta un proyecto importante.", + "translation_en": "The director presents an important project.", + "interlinear_gloss": [ + { "es": "El", "en": "the" }, + { "es": "director", "en": "director" }, + { "es": "presenta", "en": "presents" }, + { "es": "un", "en": "a" }, + { "es": "proyecto", "en": "project" }, + { "es": "importante", "en": "important" } + ], + "grammar_callout": "Read 'un proyecto importante' as a single unit: first see the noun (project), then apply the modifier (important)." + }, + { + "sentence_es": "Las soluciones posibles requieren paciencia.", + "translation_en": "The possible solutions require patience.", + "interlinear_gloss": [ + { "es": "Las", "en": "the (pl.)" }, + { "es": "soluciones", "en": "solutions" }, + { "es": "posibles", "en": "possible (pl.)" }, + { "es": "requieren", "en": "require" }, + { "es": "paciencia", "en": "patience" } + ], + "grammar_callout": "Adjectives also echo plural endings: 'soluciones' (plural) pulls 'posibles' into the plural." + } + ] + } + }, + { + "lesson_id": "lvl0_03_negation_and_questions", + "level": 0, + "sequence_order": 3, + "title": "Negation & Question Markers", + "target_concept": "Place 'no' directly before the verb to negate, and use inverted '¿' as an early heads-up for questions.", + "content": { + "sentences": [ + { + "sentence_es": "El sistema no funciona correctamente.", + "translation_en": "The system does not work properly.", + "interlinear_gloss": [ + { "es": "El", "en": "the" }, + { "es": "sistema", "en": "system" }, + { "es": "no", "en": "not" }, + { "es": "funciona", "en": "functions / works" }, + { "es": "correctamente", "en": "correctly" } + ], + "grammar_callout": "Spanish has no dummy auxiliary like 'does'. Simply put 'no' immediately before the action verb." + }, + { + "sentence_es": "¿Necesita el equipo más información?", + "translation_en": "Does the team need more information?", + "interlinear_gloss": [ + { "es": "¿Necesita", "en": "needs?" }, + { "es": "el", "en": "the" }, + { "es": "equipo", "en": "team" }, + { "es": "más", "en": "more" }, + { "es": "información?", "en": "information?" } + ], + "grammar_callout": "The opening '¿' tells your internal reading voice to adopt question intonation before you even reach the first word." + } + ] + } + }, + { + "lesson_id": "lvl0_04_omitted_subjects", + "level": 0, + "sequence_order": 4, + "title": "Omitted Subjects (Verb Endings as Pronouns)", + "target_concept": "Spanish drops subject pronouns ('I', 'we', 'they') because the verb ending already reveals who is acting.", + "content": { + "sentences": [ + { + "sentence_es": "Buscamos una oportunidad diferente.", + "translation_en": "We are looking for a different opportunity.", + "interlinear_gloss": [ + { "es": "Buscamos", "en": "we look for" }, + { "es": "una", "en": "a" }, + { "es": "oportunidad", "en": "opportunity" }, + { "es": "diferente", "en": "different" } + ], + "grammar_callout": "The ending '-mos' always means 'we' (nosotros). Also note that 'buscar' includes the 'for' inside the verb itself." + }, + { + "sentence_es": "Prefieren estudiar en la biblioteca central.", + "translation_en": "They prefer to study in the central library.", + "interlinear_gloss": [ + { "es": "Prefieren", "en": "they prefer" }, + { "es": "estudiar", "en": "to study" }, + { "es": "en", "en": "in" }, + { "es": "la", "en": "the" }, + { "es": "biblioteca", "en": "library" }, + { "es": "central", "en": "central" } + ], + "grammar_callout": "A verb ending in '-n' (-an / -en) signals a plural 'they' (or plural 'you')." + } + ] + } + }, + { + "lesson_id": "lvl0_05_clitic_object_pronouns", + "level": 0, + "sequence_order": 5, + "title": "Clitic Object Pronouns", + "target_concept": "Short object pronouns (lo, la, le, se, me) appear right before conjugated verbs or fuse onto the end of infinitives and gerunds.", + "content": { + "sentences": [ + { + "sentence_es": "El médico lo recomienda totalmente.", + "translation_en": "The doctor totally recommends it.", + "interlinear_gloss": [ + { "es": "El", "en": "the" }, + { "es": "médico", "en": "doctor" }, + { "es": "lo", "en": "it" }, + { "es": "recomienda", "en": "recommends" }, + { "es": "totalmente", "en": "totally" } + ], + "grammar_callout": "With conjugated verbs, 'lo' (it/him) jumps ahead of the verb: literally 'The doctor it recommends'." + }, + { + "sentence_es": "Queremos explicárselo con calma.", + "translation_en": "We want to explain it to him calmly.", + "interlinear_gloss": [ + { "es": "Queremos", "en": "we want" }, + { "es": "explicárselo", "en": "to explain + to him/her + it" }, + { "es": "con", "en": "with" }, + { "es": "calma", "en": "calm" } + ], + "grammar_callout": "Pronoun Surgery: 'explicárselo' unpacks into 'explicar' (to explain) + 'se' (to him/her/them) + 'lo' (it)." + } + ] + } + }, + { + "lesson_id": "lvl1_01_el_problema_en_la_oficina", + "level": 1, + "sequence_order": 1, + "title": "El Problema en la Oficina", + "target_concept": "Read a connected present-tense micro-story combining false cognates (soportar, carpeta) and clitic pronouns.", + "content": { + "sentences": [ + { + "sentence_es": "Actualmente Marco trabaja en una oficina muy activa.", + "translation_en": "Currently Marco works in a very busy office.", + "interlinear_gloss": [ + { "es": "Actualmente", "en": "currently" }, + { "es": "Marco", "en": "Marco" }, + { "es": "trabaja", "en": "works" }, + { "es": "en", "en": "in" }, + { "es": "una", "en": "an" }, + { "es": "oficina", "en": "office" }, + { "es": "muy", "en": "very" }, + { "es": "activa", "en": "active / busy" } + ], + "grammar_callout": "Trap alert: 'Actualmente' means 'currently', not 'actually'." + }, + { + "sentence_es": "Guarda todos los contratos importantes en una carpeta azul.", + "translation_en": "He keeps all the important contracts in a blue folder.", + "interlinear_gloss": [ + { "es": "Guarda", "en": "he keeps" }, + { "es": "todos", "en": "all" }, + { "es": "los", "en": "the" }, + { "es": "contratos", "en": "contracts" }, + { "es": "importantes", "en": "important" }, + { "es": "en", "en": "in" }, + { "es": "una", "en": "a" }, + { "es": "carpeta", "en": "folder" }, + { "es": "azul", "en": "blue" } + ], + "grammar_callout": "Trap alert: 'carpeta' is a desk folder or binder, never a floor carpet." + }, + { + "sentence_es": "Elena no soporta el ruido y decide explicárselo al director.", + "translation_en": "Elena cannot stand the noise and decides to explain it to the director.", + "interlinear_gloss": [ + { "es": "Elena", "en": "Elena" }, + { "es": "no", "en": "not" }, + { "es": "soporta", "en": "tolerates / stands" }, + { "es": "el", "en": "the" }, + { "es": "ruido", "en": "noise" }, + { "es": "y", "en": "and" }, + { "es": "decide", "en": "decides" }, + { "es": "explicárselo", "en": "to explain + to him + it" }, + { "es": "al", "en": "to the" }, + { "es": "director", "en": "director" } + ], + "grammar_callout": "Combines the false friend 'soporta' (tolerates) with the compound clitic verb 'explicárselo' (explicar + se + lo)." + } + ] + } + }, + { + "lesson_id": "lvl2_01_el_misterio_de_la_libreria", + "level": 2, + "sequence_order": 1, + "title": "El Misterio de la Librería", + "target_concept": "Navigate past-tense narrative contrasts (ongoing imperfecto vs. completed pretérito) in a short story.", + "content": { + "sentences": [ + { + "sentence_es": "Llovía intensamente cuando Clara entró en la pequeña librería del barrio.", + "translation_en": "It was raining heavily when Clara walked into the small neighborhood bookstore.", + "interlinear_gloss": [ + { "es": "Llovía", "en": "it was raining" }, + { "es": "intensamente", "en": "intensely / heavily" }, + { "es": "cuando", "en": "when" }, + { "es": "Clara", "en": "Clara" }, + { "es": "entró", "en": "entered" }, + { "es": "en", "en": "in / into" }, + { "es": "la", "en": "the" }, + { "es": "pequeña", "en": "small" }, + { "es": "librería", "en": "bookstore" }, + { "es": "del", "en": "of the" }, + { "es": "barrio", "en": "neighborhood" } + ], + "grammar_callout": "'Llovía' (-ía imperfecto) paints the ongoing background weather, while 'entró' (-ó pretérito) marks the single completed action." + }, + { + "sentence_es": "Buscaba una novela histórica pero encontró una carta antigua entre las páginas.", + "translation_en": "She was looking for a historical novel, but she found an old letter between the pages.", + "interlinear_gloss": [ + { "es": "Buscaba", "en": "she was looking for" }, + { "es": "una", "en": "a" }, + { "es": "novela", "en": "novel" }, + { "es": "histórica", "en": "historical" }, + { "es": "pero", "en": "but" }, + { "es": "encontró", "en": "she found" }, + { "es": "una", "en": "an" }, + { "es": "carta", "en": "letter" }, + { "es": "antigua", "en": "old / ancient" }, + { "es": "entre", "en": "between" }, + { "es": "las", "en": "the" }, + { "es": "páginas", "en": "pages" } + ], + "grammar_callout": "Again, '-aba' (buscaba = was looking for) sets the ongoing intention interrupted by '-ó' (encontró = found)." + }, + { + "sentence_es": "La lectura del documento reveló un secreto familiar extraordinario.", + "translation_en": "Reading the document revealed an extraordinary family secret.", + "interlinear_gloss": [ + { "es": "La", "en": "the" }, + { "es": "lectura", "en": "reading" }, + { "es": "del", "en": "of the" }, + { "es": "documento", "en": "document" }, + { "es": "reveló", "en": "revealed" }, + { "es": "un", "en": "an" }, + { "es": "secreto", "en": "secret" }, + { "es": "familiar", "en": "family" }, + { "es": "extraordinario", "en": "extraordinary" } + ], + "grammar_callout": "Trap alert: 'lectura' means 'reading' (not lecture), and 'familiar' as an adjective means 'related to family' (not just well-known)." + } + ] + } + } +] diff --git a/tools/preprocessor/BUILD.bazel b/tools/preprocessor/BUILD.bazel index 721dfbd..c3c2d73 100644 --- a/tools/preprocessor/BUILD.bazel +++ b/tools/preprocessor/BUILD.bazel @@ -28,6 +28,7 @@ kt_jvm_test( srcs = glob(["src/test/kotlin/**/*.kt"]), data = [ "//data/seeds:cognate_rules_seed.json", + "//data/seeds:curated_lessons_seed.json", "//data/seeds:false_cognates_seed.json", ], test_class = "com.anysoftkeyboard.calque.preprocessor.PreprocessorTest", diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/CuratedLessonsProcessor.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/CuratedLessonsProcessor.kt new file mode 100644 index 0000000..585b32b --- /dev/null +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/CuratedLessonsProcessor.kt @@ -0,0 +1,90 @@ +package com.anysoftkeyboard.calque.preprocessor + +import java.io.InputStream +import java.io.InputStreamReader +import java.nio.charset.StandardCharsets +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.intOrNull +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive + +/** Seeds the curated_lessons table from a handcrafted JSON definition. */ +object CuratedLessonsProcessor { + private const val DEFAULT_LEVEL = 0 + private const val DEFAULT_SEQUENCE = 1 + + /** + * Reads curated syntactic progression lessons from [inputStream] and inserts them into + * [database]. + * + * @return Count of curated lessons inserted. + */ + fun processLessons(inputStream: InputStream, database: SeedDatabase): Int { + val rows = parseLessons(inputStream) + database.insertCuratedLessonsBatch(rows) + return rows.size + } + + /** Parses the curated lessons seed JSON stream into a list of [CuratedLessonRow] objects. */ + fun parseLessons(inputStream: InputStream): List { + val jsonText = InputStreamReader(inputStream, StandardCharsets.UTF_8).use { it.readText() } + val array = Json.parseToJsonElement(jsonText).jsonArray + return array.mapNotNull { parseLessonElement(it.jsonObject) } + } + + private fun parseLessonElement(obj: JsonObject): CuratedLessonRow? { + val lessonId = obj["lesson_id"]?.jsonPrimitive?.contentOrNull ?: return null + val level = obj["level"]?.jsonPrimitive?.intOrNull ?: DEFAULT_LEVEL + val sequenceOrder = obj["sequence_order"]?.jsonPrimitive?.intOrNull ?: DEFAULT_SEQUENCE + val title = obj["title"]?.jsonPrimitive?.contentOrNull.orEmpty() + val targetConcept = obj["target_concept"]?.jsonPrimitive?.contentOrNull.orEmpty() + val contentJson = + obj["content"]?.toString() + ?: obj["content_json"]?.jsonPrimitive?.contentOrNull + ?: """{"sentences":[]}""" + + return CuratedLessonRow( + lessonId = lessonId, + level = level, + sequenceOrder = sequenceOrder, + title = title, + targetConcept = targetConcept, + contentJson = contentJson, + ) + } + + /** Parses the serialized `content_json` payload into a list of [LessonSentence] items. */ + fun parseLessonSentences(contentJson: String): List { + val root = Json.parseToJsonElement(contentJson).jsonObject + val sentencesArray = root["sentences"]?.jsonArray ?: return emptyList() + return sentencesArray.mapNotNull { parseSentenceElement(it.jsonObject) } + } + + private fun parseSentenceElement(obj: JsonObject): LessonSentence? { + val sentenceEs = obj["sentence_es"]?.jsonPrimitive?.contentOrNull ?: return null + val translationEn = obj["translation_en"]?.jsonPrimitive?.contentOrNull.orEmpty() + val glossArray = obj["interlinear_gloss"] as? JsonArray + val interlinearGloss = glossArray?.mapNotNull { parseGlossElement(it) }.orEmpty() + val grammarCallout = obj["grammar_callout"]?.jsonPrimitive?.contentOrNull + + return LessonSentence( + sentenceEs = sentenceEs, + translationEn = translationEn, + interlinearGloss = interlinearGloss, + grammarCallout = grammarCallout, + ) + } + + private fun parseGlossElement(elem: JsonElement): InterlinearGlossToken? { + val obj = elem as? JsonObject ?: return null + val es = obj["es"]?.jsonPrimitive?.contentOrNull?.trim().orEmpty() + val en = obj["en"]?.jsonPrimitive?.contentOrNull?.trim().orEmpty() + if (es.isEmpty() || en.isEmpty()) return null + return InterlinearGlossToken(es = es, en = en) + } +} diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Main.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Main.kt index 5df17db..586fccb 100644 --- a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Main.kt +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Main.kt @@ -14,11 +14,12 @@ fun main(args: Array) { Usage: preprocessor --db [options] Options: - --kaikki Path to Kaikki Spanish JSONL dump - --sublex Path to SUBTLEX-ESP frequency file - --false-cognates Path to false_cognates_seed.json - --tatoeba Path to Tatoeba Spanish-English TSV file - --cognate-rules Path to cognate_rules_seed.json + --kaikki Path to Kaikki Spanish JSONL dump + --sublex Path to SUBTLEX-ESP frequency file + --false-cognates Path to false_cognates_seed.json + --curated-lessons Path to curated_lessons_seed.json + --tatoeba Path to Tatoeba Spanish-English TSV file + --cognate-rules Path to cognate_rules_seed.json """ .trimIndent(), ) @@ -31,6 +32,7 @@ fun main(args: Array) { kaikkiFile = options["--kaikki"]?.let { File(it) }, sublexFile = options["--sublex"]?.let { File(it) }, falseCognatesFile = options["--false-cognates"]?.let { File(it) }, + curatedLessonsFile = options["--curated-lessons"]?.let { File(it) }, tatoebaFile = options["--tatoeba"]?.let { File(it) }, cognateRulesFile = options["--cognate-rules"]?.let { File(it) }, ) diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Models.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Models.kt index 850870c..26b8e0c 100644 --- a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Models.kt +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Models.kt @@ -46,6 +46,20 @@ data class CuratedLessonRow( val contentJson: String, ) +/** Represents a single token in an interlinear word-for-word literal gloss. */ +data class InterlinearGlossToken( + val es: String, + val en: String, +) + +/** Represents a structured sentence inside a curated lesson's `content_json`. */ +data class LessonSentence( + val sentenceEs: String, + val translationEn: String, + val interlinearGloss: List, + val grammarCallout: String?, +) + /** Model for the cognate_rules seed table. */ data class CognateRuleRow( val ruleId: String, diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorPipeline.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorPipeline.kt index e42d734..f57d3f2 100644 --- a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorPipeline.kt +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorPipeline.kt @@ -9,6 +9,7 @@ data class PipelineConfig( val kaikkiFile: File? = null, val sublexFile: File? = null, val falseCognatesFile: File? = null, + val curatedLessonsFile: File? = null, val tatoebaFile: File? = null, val cognateRulesFile: File? = null, ) @@ -36,7 +37,7 @@ class PreprocessorPipeline(private val config: PipelineConfig) { val sublexFrequencies = loadSublexFrequencies() val dictCount = processKaikki(database, sublexFrequencies) val falseCognatesCount = processFalseCognates(database) - val lessonsCount = processLessons(database) + val lessonsCount = processCuratedLessons(database) + processTatoeba(database) val rulesCount = processRules(database) return PipelineSummary( @@ -67,7 +68,13 @@ class PreprocessorPipeline(private val config: PipelineConfig) { return FileInputStream(file).use { FalseCognatesExpander.expand(it, database) } } - private fun processLessons(database: SeedDatabase): Int { + private fun processCuratedLessons(database: SeedDatabase): Int { + val file = config.curatedLessonsFile + if (file == null || !file.exists()) return 0 + return FileInputStream(file).use { CuratedLessonsProcessor.processLessons(it, database) } + } + + private fun processTatoeba(database: SeedDatabase): Int { val file = config.tatoebaFile if (file == null || !file.exists()) return 0 return FileInputStream(file).use { TatoebaProcessor.processSentences(it, database) } diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt index 7540d72..4b9b142 100644 --- a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt @@ -19,20 +19,6 @@ class SeedDatabase(private val connection: Connection) : AutoCloseable { DatabaseBinders.createTables(connection) } - private inline fun executeTransaction(action: () -> Unit) { - val prevAutoCommit = connection.autoCommit - connection.autoCommit = false - try { - action() - connection.commit() - } catch (e: SQLException) { - connection.rollback() - throw e - } finally { - connection.autoCommit = prevAutoCommit - } - } - /** Inserts a batch of dictionary rows inside a single transaction. */ fun insertDictionaryBatch(entries: Collection) { if (entries.isEmpty()) return @@ -44,7 +30,7 @@ class SeedDatabase(private val connection: Connection) : AutoCloseable { """ .trimIndent() - executeTransaction { + DatabaseBinders.executeTransaction(connection) { connection.prepareStatement(sql).use { pstmt -> for (entry in entries) { DatabaseBinders.bindDictionaryRow(pstmt, entry) @@ -97,7 +83,7 @@ class SeedDatabase(private val connection: Connection) : AutoCloseable { """ .trimIndent() - executeTransaction { + DatabaseBinders.executeTransaction(connection) { connection.prepareStatement(sql).use { pstmt -> for (entry in entries) { DatabaseBinders.bindFalseCognateRow(pstmt, entry) @@ -140,7 +126,7 @@ class SeedDatabase(private val connection: Connection) : AutoCloseable { """ .trimIndent() - executeTransaction { + DatabaseBinders.executeTransaction(connection) { connection.prepareStatement(sql).use { pstmt -> for (entry in entries) { DatabaseBinders.bindCuratedLessonRow(pstmt, entry) @@ -151,6 +137,26 @@ class SeedDatabase(private val connection: Connection) : AutoCloseable { } } + /** Queries all curated lessons ordered by level and sequence_order. */ + fun queryCuratedLessons(): List { + val sql = + """ + SELECT lesson_id, level, sequence_order, title, target_concept, content_json + FROM curated_lessons + ORDER BY level ASC, sequence_order ASC + """ + .trimIndent() + val lessons = mutableListOf() + connection.createStatement().use { stmt -> + stmt.executeQuery(sql).use { rs -> + while (rs.next()) { + lessons.add(DatabaseBinders.mapCuratedLessonRow(rs)) + } + } + } + return lessons + } + /** Inserts a batch of cognate rules. */ fun insertCognateRulesBatch(entries: Collection) { if (entries.isEmpty()) return @@ -162,7 +168,7 @@ class SeedDatabase(private val connection: Connection) : AutoCloseable { """ .trimIndent() - executeTransaction { + DatabaseBinders.executeTransaction(connection) { connection.prepareStatement(sql).use { pstmt -> for (entry in entries) { DatabaseBinders.bindCognateRuleRow(pstmt, entry) @@ -244,6 +250,20 @@ internal object DatabaseBinders { private const val PARAM_RULE_SPELLING = 5 private const val PARAM_RULE_EXAMPLES = 6 + inline fun executeTransaction(connection: Connection, action: () -> Unit) { + val prevAutoCommit = connection.autoCommit + connection.autoCommit = false + try { + action() + connection.commit() + } catch (e: SQLException) { + connection.rollback() + throw e + } finally { + connection.autoCommit = prevAutoCommit + } + } + fun createTables(connection: Connection) { val statements = listOf( @@ -367,6 +387,17 @@ internal object DatabaseBinders { pstmt.setString(PARAM_LESSON_CONTENT, entry.contentJson) } + fun mapCuratedLessonRow(rs: ResultSet): CuratedLessonRow { + return CuratedLessonRow( + lessonId = rs.getString("lesson_id"), + level = rs.getInt("level"), + sequenceOrder = rs.getInt("sequence_order"), + title = rs.getString("title"), + targetConcept = rs.getString("target_concept"), + contentJson = rs.getString("content_json"), + ) + } + fun bindCognateRuleRow(pstmt: PreparedStatement, entry: CognateRuleRow) { pstmt.setString(PARAM_RULE_ID, entry.ruleId) pstmt.setString(PARAM_RULE_EN_SUFFIX, entry.englishSuffix) diff --git a/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt b/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt index f1cbdbe..0088fa4 100644 --- a/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt +++ b/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt @@ -219,6 +219,45 @@ class PreprocessorTest { } } + @Test + fun curatedLessonsSeed_isValidAndPopulatesProgressionLevels() { + val seedFile = File("data/seeds/curated_lessons_seed.json") + assertTrue("Expected seed file at ${seedFile.path}", seedFile.exists()) + + SeedDatabase.open().use { db -> + val insertedCount = + FileInputStream(seedFile).use { stream -> + CuratedLessonsProcessor.processLessons(stream, db) + } + assertEquals(8, insertedCount) + + val lessons = db.queryCuratedLessons() + assertEquals(8, lessons.size) + assertEquals(setOf(0, 1, 2), lessons.map { it.level }.toSet()) + + val sentenceZeroLesson = lessons.first { it.lessonId == "lvl0_00_cognate_unlock" } + val sentenceZeroItems = + CuratedLessonsProcessor.parseLessonSentences(sentenceZeroLesson.contentJson) + assertEquals( + "El hotel moderno es excelente.", + sentenceZeroItems.first().sentenceEs, + ) + + for (lesson in lessons) { + assertFalse(lesson.title.isBlank()) + assertFalse(lesson.targetConcept.isBlank()) + val sentences = CuratedLessonsProcessor.parseLessonSentences(lesson.contentJson) + assertTrue(sentences.isNotEmpty()) + for (sentence in sentences) { + assertFalse(sentence.sentenceEs.isBlank()) + assertFalse(sentence.translationEn.isBlank()) + assertTrue(sentence.interlinearGloss.isNotEmpty()) + assertFalse(sentence.grammarCallout.isNullOrBlank()) + } + } + } + } + @Test fun cognateRulesProcessor_insertsRules() { val inMemoryDb = SeedDatabase.open() @@ -328,6 +367,7 @@ class PreprocessorTest { kaikkiFile.writeText("$sampleKaikki\n", StandardCharsets.UTF_8) val falseCognatesSeedFile = File("data/seeds/false_cognates_seed.json") + val curatedLessonsSeedFile = File("data/seeds/curated_lessons_seed.json") val cognateRulesSeedFile = File("data/seeds/cognate_rules_seed.json") val config = @@ -336,6 +376,7 @@ class PreprocessorTest { kaikkiFile = kaikkiFile, sublexFile = sublexFile, falseCognatesFile = falseCognatesSeedFile, + curatedLessonsFile = curatedLessonsSeedFile, cognateRulesFile = cognateRulesSeedFile, ) @@ -346,6 +387,7 @@ class PreprocessorTest { assertTrue(summary.dictionaryRows >= 2) // 25 base lemmas + "carpetas" plural from Kaikki = 26 rows assertEquals(26, summary.falseCognateRows) + assertEquals(8, summary.curatedLessonRows) assertEquals(12, summary.cognateRuleRows) tempDir.deleteRecursively()