diff --git a/tools/preprocessor/BUILD.bazel b/tools/preprocessor/BUILD.bazel index ffd0fb0..730850e 100644 --- a/tools/preprocessor/BUILD.bazel +++ b/tools/preprocessor/BUILD.bazel @@ -1 +1,37 @@ +load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_binary", "kt_jvm_library", "kt_jvm_test") + package(default_visibility = ["//visibility:public"]) + +kt_jvm_library( + name = "preprocessor_lib", + srcs = glob(["src/main/kotlin/**/*.kt"]), + visibility = ["//visibility:public"], + deps = [ + "//core/linguistics", + "@maven//:org_jetbrains_kotlinx_kotlinx_serialization_json", + "@maven//:org_xerial_sqlite_jdbc", + ], +) + +kt_jvm_binary( + name = "preprocessor", + main_class = "com.anysoftkeyboard.calque.preprocessor.MainKt", + visibility = ["//visibility:public"], + runtime_deps = [ + ":preprocessor_lib", + ], +) + +kt_jvm_test( + name = "preprocessor_test", + size = "small", + srcs = glob(["src/test/kotlin/**/*.kt"]), + test_class = "com.anysoftkeyboard.calque.preprocessor.PreprocessorTest", + deps = [ + ":preprocessor_lib", + "//core/linguistics", + "@maven//:junit_junit", + "@maven//:org_jetbrains_kotlinx_kotlinx_serialization_json", + "@maven//:org_xerial_sqlite_jdbc", + ], +) diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/CognateRulesProcessor.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/CognateRulesProcessor.kt new file mode 100644 index 0000000..96613a2 --- /dev/null +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/CognateRulesProcessor.kt @@ -0,0 +1,48 @@ +package com.anysoftkeyboard.calque.preprocessor + +import java.io.InputStream +import java.io.InputStreamReader +import java.nio.charset.StandardCharsets +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive + +/** Seeds the cognate_rules table from a JSON definition. */ +object CognateRulesProcessor { + + /** + * Reads cognate rules from [inputStream] and inserts them into [database]. + * + * @return Count of cognate rules inserted. + */ + fun processRules(inputStream: InputStream, database: SeedDatabase): Int { + val jsonText = InputStreamReader(inputStream, StandardCharsets.UTF_8).use { it.readText() } + val array = Json.parseToJsonElement(jsonText).jsonArray + val rows = mutableListOf() + + for (elem in array) { + val obj = elem.jsonObject + val ruleId = obj["rule_id"]?.jsonPrimitive?.content ?: continue + val englishSuffix = obj["english_suffix"]?.jsonPrimitive?.content.orEmpty() + val spanishSuffix = obj["spanish_suffix"]?.jsonPrimitive?.content.orEmpty() + val genderRule = obj["gender_rule"]?.jsonPrimitive?.content + val spellingNote = obj["spelling_note"]?.jsonPrimitive?.content.orEmpty() + val examplesJson = obj["examples"]?.toString() ?: "[]" + + rows.add( + CognateRuleRow( + ruleId = ruleId, + englishSuffix = englishSuffix, + spanishSuffix = spanishSuffix, + genderRule = genderRule, + spellingNote = spellingNote, + examplesJson = examplesJson, + ), + ) + } + + database.insertCognateRulesBatch(rows) + return rows.size + } +} diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/FalseCognatesExpander.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/FalseCognatesExpander.kt new file mode 100644 index 0000000..cee5375 --- /dev/null +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/FalseCognatesExpander.kt @@ -0,0 +1,86 @@ +package com.anysoftkeyboard.calque.preprocessor + +import java.io.InputStream +import java.io.InputStreamReader +import java.nio.charset.StandardCharsets +import java.util.Locale +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive + +/** + * Expands curated false cognate seeds across all conjugated and inflected forms in the dictionary. + * + * Implements Step 3 from INGESTION.md. + */ +object FalseCognatesExpander { + + /** + * Expands false cognates from an input stream and stores them into the database. + * + * @return Count of false cognate entries inserted. + */ + fun expand(inputStream: InputStream, database: SeedDatabase): Int { + val seeds = parseSeeds(inputStream) + val rows = mutableListOf() + + for (seed in seeds) { + val matchingSurfaces = database.querySurfacesForLemma(seed.lemma).toMutableSet() + // Ensure the lowercased base lemma is always included + matchingSurfaces.add(seed.lemma.lowercase(Locale.ROOT)) + + for (surface in matchingSurfaces) { + rows.add( + FalseCognateRow( + surfaceForm = surface, + lemma = seed.lemma, + apparentEnglish = seed.apparentEnglish, + actualEnglish = seed.actualEnglish, + realSpanishForApparent = seed.realSpanishForApparent, + severity = seed.severity, + disambiguationTip = seed.disambiguationTip, + exampleEs = seed.exampleEs, + exampleEn = seed.exampleEn, + ), + ) + } + } + + database.insertFalseCognatesBatch(rows) + return rows.size + } + + /** Parses the false cognates seed JSON stream into a list of [FalseCognateSeed] objects. */ + fun parseSeeds(inputStream: InputStream): List { + val jsonText = InputStreamReader(inputStream, StandardCharsets.UTF_8).use { it.readText() } + val array = Json.parseToJsonElement(jsonText).jsonArray + val seeds = mutableListOf() + + for (elem in array) { + val obj = elem.jsonObject + val lemma = obj["lemma"]?.jsonPrimitive?.content ?: continue + val apparentEnglish = obj["apparent_english"]?.jsonPrimitive?.content.orEmpty() + val actualEnglish = obj["actual_english"]?.jsonPrimitive?.content.orEmpty() + val realSpanish = obj["real_spanish_for_apparent"]?.jsonPrimitive?.content.orEmpty() + val severity = obj["severity"]?.jsonPrimitive?.content ?: "HIGH_RISK" + val tip = obj["disambiguation_tip"]?.jsonPrimitive?.content.orEmpty() + val exampleEs = obj["example_es"]?.jsonPrimitive?.content.orEmpty() + val exampleEn = obj["example_en"]?.jsonPrimitive?.content.orEmpty() + + seeds.add( + FalseCognateSeed( + lemma = lemma, + apparentEnglish = apparentEnglish, + actualEnglish = actualEnglish, + realSpanishForApparent = realSpanish, + severity = severity, + disambiguationTip = tip, + exampleEs = exampleEs, + exampleEn = exampleEn, + ), + ) + } + return seeds + } +} diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/KaikkiProcessor.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/KaikkiProcessor.kt new file mode 100644 index 0000000..2801f5b --- /dev/null +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/KaikkiProcessor.kt @@ -0,0 +1,131 @@ +package com.anysoftkeyboard.calque.preprocessor + +import com.anysoftkeyboard.calque.linguistics.CognateDetector +import java.io.BufferedReader +import java.io.InputStream +import java.io.InputStreamReader +import java.nio.charset.StandardCharsets +import java.util.Locale +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive + +/** + * Processor for streaming Kaikki JSONL dictionary dumps and populating the dictionary table. + * + * Implements Step 1 from INGESTION.md. + */ +object KaikkiProcessor { + private const val DEFAULT_BATCH_SIZE = 5000 + + /** + * Streams the Kaikki JSONL input, extracts lemmas and inflected forms, tags cognates, resolves + * SUBTLEX frequencies, and flattens them into the dictionary table. + * + * @return The count of total dictionary rows inserted. + */ + fun processStream( + inputStream: InputStream, + sublexFrequencies: Map, + database: SeedDatabase, + batchSize: Int = DEFAULT_BATCH_SIZE, + ): Long { + val reader = BufferedReader(InputStreamReader(inputStream, StandardCharsets.UTF_8)) + val batch = mutableListOf() + var insertedRows = 0L + + for (rawLine in reader.lineSequence()) { + val line = rawLine.trim() + if (line.isEmpty()) continue + + val rows = parseKaikkiLine(line, sublexFrequencies) + batch.addAll(rows) + + if (batch.size >= batchSize) { + database.insertDictionaryBatch(batch) + insertedRows += batch.size + batch.clear() + } + } + + if (batch.isNotEmpty()) { + database.insertDictionaryBatch(batch) + insertedRows += batch.size + batch.clear() + } + + return insertedRows + } + + /** Parses a single Kaikki JSON line into base and inflected [DictionaryRow]s. */ + fun parseKaikkiLine( + jsonLine: String, + sublexFrequencies: Map, + ): List { + val jsonElement = Json.parseToJsonElement(jsonLine).jsonObject + val lemma = jsonElement["word"]?.jsonPrimitive?.content ?: return emptyList() + val pos = jsonElement["pos"]?.jsonPrimitive?.content ?: "" + val primaryGloss = extractPrimaryGloss(jsonElement) + val normalizedLemma = lemma.lowercase(Locale.ROOT) + val freqRank = sublexFrequencies[normalizedLemma] + val isCognate = CognateDetector.detectCognate(lemma, primaryGloss) + + val results = mutableListOf() + + // 1. Base lemma row + results.add( + DictionaryRow( + surfaceForm = normalizedLemma, + lemma = lemma, + pos = pos, + grammarInfo = "base", + englishGloss = primaryGloss, + isCognate = isCognate, + frequencyRank = freqRank, + ), + ) + + // 2. Expand inflected forms + val formsArray = jsonElement["forms"]?.jsonArray + if (formsArray != null) { + for (formElement in formsArray) { + val formObj = formElement.jsonObject + val surface = + formObj["form"]?.jsonPrimitive?.content + ?: formObj["surface"]?.jsonPrimitive?.content + ?: continue + val tags = + formObj["tags"]?.jsonArray?.map { it.jsonPrimitive.content } ?: emptyList() + val grammar = if (tags.isNotEmpty()) tags.joinToString(", ") else null + + results.add( + DictionaryRow( + surfaceForm = surface.lowercase(Locale.ROOT), + lemma = lemma, + pos = pos, + grammarInfo = grammar, + englishGloss = primaryGloss, + isCognate = isCognate, + frequencyRank = freqRank, + ), + ) + } + } + + return results + } + + private fun extractPrimaryGloss(jsonObj: kotlinx.serialization.json.JsonObject): String { + val senses = jsonObj["senses"]?.jsonArray ?: return "" + for (senseElement in senses) { + val senseObj = senseElement.jsonObject + val glosses = senseObj["glosses"]?.jsonArray + val firstGloss = glosses?.firstOrNull()?.jsonPrimitive?.content + if (!firstGloss.isNullOrBlank()) { + return firstGloss + } + } + return "" + } +} diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Main.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Main.kt new file mode 100644 index 0000000..5df17db --- /dev/null +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Main.kt @@ -0,0 +1,66 @@ +package com.anysoftkeyboard.calque.preprocessor + +import java.io.File +import kotlin.system.exitProcess + +/** CLI binary entry point for the Calque seed.db build-time preprocessor. */ +fun main(args: Array) { + val options = parseCommandLineArgs(args) + + val dbPath = options["--db"] + if (dbPath == null) { + System.err.println( + """ + Usage: preprocessor --db [options] + + Options: + --kaikki Path to Kaikki Spanish JSONL dump + --sublex Path to SUBTLEX-ESP frequency file + --false-cognates Path to false_cognates_seed.json + --tatoeba Path to Tatoeba Spanish-English TSV file + --cognate-rules Path to cognate_rules_seed.json + """ + .trimIndent(), + ) + exitProcess(1) + } + + val config = + PipelineConfig( + dbFile = File(dbPath), + kaikkiFile = options["--kaikki"]?.let { File(it) }, + sublexFile = options["--sublex"]?.let { File(it) }, + falseCognatesFile = options["--false-cognates"]?.let { File(it) }, + tatoebaFile = options["--tatoeba"]?.let { File(it) }, + cognateRulesFile = options["--cognate-rules"]?.let { File(it) }, + ) + + println("Starting Calque Data Ingestion Pipeline...") + println("Target database: ${config.dbFile.absolutePath}") + + val startTime = System.currentTimeMillis() + val pipeline = PreprocessorPipeline(config) + val summary = pipeline.execute() + val elapsed = System.currentTimeMillis() - startTime + + println("Pipeline completed successfully in ${elapsed}ms:") + println(" - Dictionary entries inserted: ${summary.dictionaryRows}") + println(" - False cognate forms expanded: ${summary.falseCognateRows}") + println(" - Curated lesson sentences accepted: ${summary.curatedLessonRows}") + println(" - Cognate transformer rules seeded: ${summary.cognateRuleRows}") +} + +private fun parseCommandLineArgs(args: Array): Map { + val options = mutableMapOf() + var i = 0 + while (i < args.size) { + val arg = args[i] + if (arg.startsWith("--") && i + 1 < args.size && !args[i + 1].startsWith("--")) { + options[arg] = args[i + 1] + i += 2 + } else { + i++ + } + } + return options +} diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Models.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Models.kt new file mode 100644 index 0000000..8b15790 --- /dev/null +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Models.kt @@ -0,0 +1,57 @@ +package com.anysoftkeyboard.calque.preprocessor + +/** Model for the dictionary seed table. */ +data class DictionaryRow( + val surfaceForm: String, + val lemma: String, + val pos: String, + val grammarInfo: String?, + val englishGloss: String, + val isCognate: Boolean, + val frequencyRank: Int?, +) + +/** Model for the false_cognates seed table. */ +data class FalseCognateRow( + val surfaceForm: String, + val lemma: String, + val apparentEnglish: String, + val actualEnglish: String, + val realSpanishForApparent: String, + val severity: String, + val disambiguationTip: String, + val exampleEs: String, + val exampleEn: String, +) + +/** Model representing a raw curated seed entry before expansion. */ +data class FalseCognateSeed( + val lemma: String, + val apparentEnglish: String, + val actualEnglish: String, + val realSpanishForApparent: String, + val severity: String, + val disambiguationTip: String, + val exampleEs: String, + val exampleEn: String, +) + +/** Model for the curated_lessons seed table. */ +data class CuratedLessonRow( + val lessonId: String, + val level: Int, + val sequenceOrder: Int, + val title: String, + val targetConcept: String, + val contentJson: String, +) + +/** Model for the cognate_rules seed table. */ +data class CognateRuleRow( + val ruleId: String, + val englishSuffix: String, + val spanishSuffix: String, + val genderRule: String?, + val spellingNote: String, + val examplesJson: String, +) diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorPipeline.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorPipeline.kt new file mode 100644 index 0000000..e42d734 --- /dev/null +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorPipeline.kt @@ -0,0 +1,81 @@ +package com.anysoftkeyboard.calque.preprocessor + +import java.io.File +import java.io.FileInputStream + +/** Configuration for executing the build-time ingestion pipeline. */ +data class PipelineConfig( + val dbFile: File, + val kaikkiFile: File? = null, + val sublexFile: File? = null, + val falseCognatesFile: File? = null, + val tatoebaFile: File? = null, + val cognateRulesFile: File? = null, +) + +/** Summary of records populated during the pipeline execution. */ +data class PipelineSummary( + val dictionaryRows: Long, + val falseCognateRows: Int, + val curatedLessonRows: Int, + val cognateRuleRows: Int, +) + +/** Orchestrates Pipeline 1 build-time generation of seed.db as specified in docs/INGESTION.md. */ +class PreprocessorPipeline(private val config: PipelineConfig) { + + /** Executes the pipeline to generate or update the SQLite seed database. */ + fun execute(): PipelineSummary { + config.dbFile.parentFile?.mkdirs() + val jdbcUrl = "jdbc:sqlite:${config.dbFile.absolutePath}" + return SeedDatabase.open(jdbcUrl).use { database -> executeWithDatabase(database) } + } + + /** Executes the pipeline stages on an already opened [SeedDatabase]. */ + fun executeWithDatabase(database: SeedDatabase): PipelineSummary { + val sublexFrequencies = loadSublexFrequencies() + val dictCount = processKaikki(database, sublexFrequencies) + val falseCognatesCount = processFalseCognates(database) + val lessonsCount = processLessons(database) + val rulesCount = processRules(database) + + return PipelineSummary( + dictionaryRows = dictCount, + falseCognateRows = falseCognatesCount, + curatedLessonRows = lessonsCount, + cognateRuleRows = rulesCount, + ) + } + + private fun loadSublexFrequencies(): Map { + val file = config.sublexFile + if (file == null || !file.exists()) return emptyMap() + return FileInputStream(file).use { SublexReader.readFrequencies(it) } + } + + private fun processKaikki(database: SeedDatabase, sublexFrequencies: Map): Long { + val file = config.kaikkiFile + if (file == null || !file.exists()) return 0L + return FileInputStream(file).use { + KaikkiProcessor.processStream(it, sublexFrequencies, database) + } + } + + private fun processFalseCognates(database: SeedDatabase): Int { + val file = config.falseCognatesFile + if (file == null || !file.exists()) return 0 + return FileInputStream(file).use { FalseCognatesExpander.expand(it, database) } + } + + private fun processLessons(database: SeedDatabase): Int { + val file = config.tatoebaFile + if (file == null || !file.exists()) return 0 + return FileInputStream(file).use { TatoebaProcessor.processSentences(it, database) } + } + + private fun processRules(database: SeedDatabase): Int { + val file = config.cognateRulesFile + if (file == null || !file.exists()) return 0 + return FileInputStream(file).use { CognateRulesProcessor.processRules(it, database) } + } +} diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt new file mode 100644 index 0000000..301a2d4 --- /dev/null +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SeedDatabase.kt @@ -0,0 +1,327 @@ +package com.anysoftkeyboard.calque.preprocessor + +import com.anysoftkeyboard.calque.linguistics.ScoredWordInfo +import java.sql.Connection +import java.sql.DriverManager +import java.sql.PreparedStatement +import java.sql.ResultSet +import java.sql.SQLException +import java.sql.Types + +/** + * Manages the SQLite database operations for building the Calque seed.db database. + * + * Implements the table schemas defined in docs/DATA.md. + */ +class SeedDatabase(private val connection: Connection) : AutoCloseable { + + init { + createTables() + } + + private fun createTables() { + val statements = + listOf( + """ + CREATE TABLE IF NOT EXISTS dictionary ( + surface_form TEXT PRIMARY KEY, + lemma TEXT NOT NULL, + pos TEXT NOT NULL, + grammar_info TEXT, + english_gloss TEXT NOT NULL, + is_cognate INTEGER NOT NULL, + frequency_rank INTEGER + ) + """, + "CREATE INDEX IF NOT EXISTS idx_dictionary_lemma ON dictionary(lemma)", + """ + CREATE TABLE IF NOT EXISTS false_cognates ( + surface_form TEXT PRIMARY KEY, + lemma TEXT NOT NULL, + apparent_english TEXT NOT NULL, + actual_english TEXT NOT NULL, + real_spanish_for_apparent TEXT NOT NULL, + severity TEXT NOT NULL, + disambiguation_tip TEXT NOT NULL, + example_es TEXT NOT NULL, + example_en TEXT NOT NULL + ) + """, + "CREATE INDEX IF NOT EXISTS idx_false_cognates_lemma ON false_cognates(lemma)", + """ + CREATE TABLE IF NOT EXISTS curated_lessons ( + lesson_id TEXT PRIMARY KEY, + level INTEGER NOT NULL, + sequence_order INTEGER NOT NULL, + title TEXT NOT NULL, + target_concept TEXT NOT NULL, + content_json TEXT NOT NULL + ) + """, + "CREATE INDEX IF NOT EXISTS idx_curated_lessons_level ON curated_lessons(level)", + """ + CREATE TABLE IF NOT EXISTS cognate_rules ( + rule_id TEXT PRIMARY KEY, + english_suffix TEXT NOT NULL, + spanish_suffix TEXT NOT NULL, + gender_rule TEXT, + spelling_note TEXT NOT NULL, + examples_json TEXT NOT NULL + ) + """, + ) + connection.createStatement().use { stmt -> + for (sql in statements) { + stmt.execute(sql.trimIndent()) + } + } + } + + private inline fun executeTransaction(action: () -> Unit) { + val prevAutoCommit = connection.autoCommit + connection.autoCommit = false + try { + action() + connection.commit() + } catch (e: SQLException) { + connection.rollback() + throw e + } finally { + connection.autoCommit = prevAutoCommit + } + } + + /** Inserts a batch of dictionary rows inside a single transaction. */ + fun insertDictionaryBatch(entries: Collection) { + if (entries.isEmpty()) return + val sql = + """ + INSERT OR REPLACE INTO dictionary ( + surface_form, lemma, pos, grammar_info, english_gloss, is_cognate, frequency_rank + ) VALUES (?, ?, ?, ?, ?, ?, ?) + """ + .trimIndent() + + executeTransaction { + connection.prepareStatement(sql).use { pstmt -> + for (entry in entries) { + DatabaseBinders.bindDictionaryRow(pstmt, entry) + pstmt.addBatch() + } + pstmt.executeBatch() + } + } + } + + /** Queries all surface forms registered for a given lemma. */ + fun querySurfacesForLemma(lemma: String): List { + val sql = "SELECT surface_form FROM dictionary WHERE lemma = ?" + val surfaces = mutableListOf() + connection.prepareStatement(sql).use { pstmt -> + pstmt.setString(PARAM_QUERY_LEMMA, lemma) + pstmt.executeQuery().use { rs -> + while (rs.next()) { + surfaces.add(rs.getString("surface_form")) + } + } + } + return surfaces + } + + /** Looks up word properties from the dictionary table for sentence scoring. */ + fun lookupWord(surfaceForm: String): ScoredWordInfo? { + val sql = + "SELECT is_cognate, frequency_rank, grammar_info FROM dictionary WHERE surface_form = ? LIMIT 1" + connection.prepareStatement(sql).use { pstmt -> + pstmt.setString(PARAM_LOOKUP_SURFACE, surfaceForm) + pstmt.executeQuery().use { rs -> + if (rs.next()) { + return DatabaseBinders.mapWordInfo(rs) + } + } + } + return null + } + + /** Inserts a batch of false cognate rows inside a transaction. */ + fun insertFalseCognatesBatch(entries: Collection) { + if (entries.isEmpty()) return + val sql = + """ + INSERT OR REPLACE INTO false_cognates ( + surface_form, lemma, apparent_english, actual_english, real_spanish_for_apparent, + severity, disambiguation_tip, example_es, example_en + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """ + .trimIndent() + + executeTransaction { + connection.prepareStatement(sql).use { pstmt -> + for (entry in entries) { + DatabaseBinders.bindFalseCognateRow(pstmt, entry) + pstmt.addBatch() + } + pstmt.executeBatch() + } + } + } + + /** Inserts a batch of curated lessons. */ + fun insertCuratedLessonsBatch(entries: Collection) { + if (entries.isEmpty()) return + val sql = + """ + INSERT OR REPLACE INTO curated_lessons ( + lesson_id, level, sequence_order, title, target_concept, content_json + ) VALUES (?, ?, ?, ?, ?, ?) + """ + .trimIndent() + + executeTransaction { + connection.prepareStatement(sql).use { pstmt -> + for (entry in entries) { + DatabaseBinders.bindCuratedLessonRow(pstmt, entry) + pstmt.addBatch() + } + pstmt.executeBatch() + } + } + } + + /** Inserts a batch of cognate rules. */ + fun insertCognateRulesBatch(entries: Collection) { + if (entries.isEmpty()) return + val sql = + """ + INSERT OR REPLACE INTO cognate_rules ( + rule_id, english_suffix, spanish_suffix, gender_rule, spelling_note, examples_json + ) VALUES (?, ?, ?, ?, ?, ?) + """ + .trimIndent() + + executeTransaction { + connection.prepareStatement(sql).use { pstmt -> + for (entry in entries) { + DatabaseBinders.bindCognateRuleRow(pstmt, entry) + pstmt.addBatch() + } + pstmt.executeBatch() + } + } + } + + override fun close() { + connection.close() + } + + companion object { + const val COGNATE_FLAG_TRUE = 1 + + private const val PARAM_QUERY_LEMMA = 1 + private const val PARAM_LOOKUP_SURFACE = 1 + + /** Connects to an SQLite database at [jdbcUrl] or creates an in-memory database. */ + fun open(jdbcUrl: String = "jdbc:sqlite::memory:"): SeedDatabase { + val conn = DriverManager.getConnection(jdbcUrl) + return SeedDatabase(conn) + } + } +} + +internal object DatabaseBinders { + private const val PARAM_DICT_SURFACE = 1 + private const val PARAM_DICT_LEMMA = 2 + private const val PARAM_DICT_POS = 3 + private const val PARAM_DICT_GRAMMAR = 4 + private const val PARAM_DICT_GLOSS = 5 + private const val PARAM_DICT_COGNATE = 6 + private const val PARAM_DICT_FREQ = 7 + + private const val PARAM_FC_SURFACE = 1 + private const val PARAM_FC_LEMMA = 2 + private const val PARAM_FC_APPARENT = 3 + private const val PARAM_FC_ACTUAL = 4 + private const val PARAM_FC_REAL_SPANISH = 5 + private const val PARAM_FC_SEVERITY = 6 + private const val PARAM_FC_TIP = 7 + private const val PARAM_FC_EX_ES = 8 + private const val PARAM_FC_EX_EN = 9 + + private const val PARAM_LESSON_ID = 1 + private const val PARAM_LESSON_LEVEL = 2 + private const val PARAM_LESSON_SEQ = 3 + private const val PARAM_LESSON_TITLE = 4 + private const val PARAM_LESSON_CONCEPT = 5 + private const val PARAM_LESSON_CONTENT = 6 + + private const val PARAM_RULE_ID = 1 + private const val PARAM_RULE_EN_SUFFIX = 2 + private const val PARAM_RULE_ES_SUFFIX = 3 + private const val PARAM_RULE_GENDER = 4 + private const val PARAM_RULE_SPELLING = 5 + private const val PARAM_RULE_EXAMPLES = 6 + + fun bindDictionaryRow(pstmt: PreparedStatement, entry: DictionaryRow) { + pstmt.setString(PARAM_DICT_SURFACE, entry.surfaceForm) + pstmt.setString(PARAM_DICT_LEMMA, entry.lemma) + pstmt.setString(PARAM_DICT_POS, entry.pos) + if (entry.grammarInfo != null) { + pstmt.setString(PARAM_DICT_GRAMMAR, entry.grammarInfo) + } else { + pstmt.setNull(PARAM_DICT_GRAMMAR, Types.VARCHAR) + } + pstmt.setString(PARAM_DICT_GLOSS, entry.englishGloss) + pstmt.setInt(PARAM_DICT_COGNATE, if (entry.isCognate) SeedDatabase.COGNATE_FLAG_TRUE else 0) + if (entry.frequencyRank != null) { + pstmt.setInt(PARAM_DICT_FREQ, entry.frequencyRank) + } else { + pstmt.setNull(PARAM_DICT_FREQ, Types.INTEGER) + } + } + + fun mapWordInfo(rs: ResultSet): ScoredWordInfo { + val isCognate = rs.getInt("is_cognate") == SeedDatabase.COGNATE_FLAG_TRUE + val freqObj = rs.getObject("frequency_rank") + val freqRank = if (freqObj != null) rs.getInt("frequency_rank") else null + val grammar = rs.getString("grammar_info") + return ScoredWordInfo( + frequencyRank = freqRank, + isCognate = isCognate, + grammar = grammar, + ) + } + + fun bindFalseCognateRow(pstmt: PreparedStatement, entry: FalseCognateRow) { + pstmt.setString(PARAM_FC_SURFACE, entry.surfaceForm) + pstmt.setString(PARAM_FC_LEMMA, entry.lemma) + pstmt.setString(PARAM_FC_APPARENT, entry.apparentEnglish) + pstmt.setString(PARAM_FC_ACTUAL, entry.actualEnglish) + pstmt.setString(PARAM_FC_REAL_SPANISH, entry.realSpanishForApparent) + pstmt.setString(PARAM_FC_SEVERITY, entry.severity) + pstmt.setString(PARAM_FC_TIP, entry.disambiguationTip) + pstmt.setString(PARAM_FC_EX_ES, entry.exampleEs) + pstmt.setString(PARAM_FC_EX_EN, entry.exampleEn) + } + + fun bindCuratedLessonRow(pstmt: PreparedStatement, entry: CuratedLessonRow) { + pstmt.setString(PARAM_LESSON_ID, entry.lessonId) + pstmt.setInt(PARAM_LESSON_LEVEL, entry.level) + pstmt.setInt(PARAM_LESSON_SEQ, entry.sequenceOrder) + pstmt.setString(PARAM_LESSON_TITLE, entry.title) + pstmt.setString(PARAM_LESSON_CONCEPT, entry.targetConcept) + pstmt.setString(PARAM_LESSON_CONTENT, entry.contentJson) + } + + fun bindCognateRuleRow(pstmt: PreparedStatement, entry: CognateRuleRow) { + pstmt.setString(PARAM_RULE_ID, entry.ruleId) + pstmt.setString(PARAM_RULE_EN_SUFFIX, entry.englishSuffix) + pstmt.setString(PARAM_RULE_ES_SUFFIX, entry.spanishSuffix) + if (entry.genderRule != null) { + pstmt.setString(PARAM_RULE_GENDER, entry.genderRule) + } else { + pstmt.setNull(PARAM_RULE_GENDER, Types.VARCHAR) + } + pstmt.setString(PARAM_RULE_SPELLING, entry.spellingNote) + pstmt.setString(PARAM_RULE_EXAMPLES, entry.examplesJson) + } +} diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SublexReader.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SublexReader.kt new file mode 100644 index 0000000..4fdf591 --- /dev/null +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/SublexReader.kt @@ -0,0 +1,65 @@ +package com.anysoftkeyboard.calque.preprocessor + +import java.io.BufferedReader +import java.io.InputStream +import java.io.InputStreamReader +import java.nio.charset.StandardCharsets +import java.util.Locale + +/** + * Reader for SUBTLEX-ESP word frequency rankings. + * + * Maps lowercased Spanish words to their frequency rank (1 = most frequent). + */ +object SublexReader { + private val DELIMITER_REGEX = Regex("""\t+|\s{2,}""") + private const val TWO_COLUMN_COUNT = 2 + + /** + * Reads SUBTLEX-ESP data from an input stream and produces a map of words to frequency ranks. + */ + fun readFrequencies(inputStream: InputStream): Map { + val reader = BufferedReader(InputStreamReader(inputStream, StandardCharsets.UTF_8)) + val frequencyMap = mutableMapOf() + var lineRank = 0 + var rankColumnIndex = -1 + + val validLines = + reader + .lineSequence() + .map { it.trim() } + .filter { it.isNotEmpty() && !it.startsWith("#") } + + for (line in validLines) { + val parts = line.split(DELIMITER_REGEX) + val firstCol = parts.firstOrNull().orEmpty().trim().lowercase(Locale.ROOT) + + if (isHeader(firstCol)) { + rankColumnIndex = parts.indexOfFirst { it.trim().equals("rank", ignoreCase = true) } + } else if (firstCol.isNotEmpty()) { + lineRank++ + val parsedRank = extractRank(parts, rankColumnIndex) + val finalRank = parsedRank ?: lineRank + + if (!frequencyMap.containsKey(firstCol)) { + frequencyMap[firstCol] = finalRank + } + } + } + return frequencyMap + } + + private fun extractRank(parts: List, rankColumnIndex: Int): Int? { + return if (rankColumnIndex >= 0 && rankColumnIndex < parts.size) { + parts[rankColumnIndex].trim().toIntOrNull() + } else if (parts.size == TWO_COLUMN_COUNT) { + parts[1].trim().toIntOrNull() + } else { + null + } + } + + private fun isHeader(candidate: String): Boolean { + return candidate == "word" || candidate == "spelling" || candidate == "palabra" + } +} diff --git a/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/TatoebaProcessor.kt b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/TatoebaProcessor.kt new file mode 100644 index 0000000..9120fa7 --- /dev/null +++ b/tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/TatoebaProcessor.kt @@ -0,0 +1,133 @@ +package com.anysoftkeyboard.calque.preprocessor + +import com.anysoftkeyboard.calque.linguistics.SentenceLevel +import com.anysoftkeyboard.calque.linguistics.SentenceScorer +import java.io.BufferedReader +import java.io.InputStream +import java.io.InputStreamReader +import java.nio.charset.StandardCharsets +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.put + +/** + * Processes Tatoeba bilingual sentence pairs, scores them using [SentenceScorer], and saves + * accepted sentences into the curated_lessons table. + * + * Implements Step 4 from INGESTION.md. + */ +object TatoebaProcessor { + private const val DEFAULT_BATCH_SIZE = 1000 + private const val TSV_PARTS_MIN = 2 + private const val INDEX_SPANISH = 0 + private const val INDEX_ENGLISH = 1 + private const val INDEX_TITLE = 2 + private const val INDEX_CONCEPT = 3 + private const val LESSON_ID_PAD = 4 + + /** + * Reads Tatoeba bilingual sentence pairs from [inputStream], scores and buckets them, and + * inserts accepted sentences into the database. + * + * @return Count of curated lessons generated. + */ + fun processSentences( + inputStream: InputStream, + database: SeedDatabase, + batchSize: Int = DEFAULT_BATCH_SIZE, + ): Int { + val reader = BufferedReader(InputStreamReader(inputStream, StandardCharsets.UTF_8)) + val sequenceByLevel = mutableMapOf() + val batch = mutableListOf() + var totalInserted = 0 + + val lessonRows = + reader + .lineSequence() + .map { it.trim() } + .filter { it.isNotEmpty() && !it.startsWith("#") } + .mapNotNull { parseLine(it) } + .mapNotNull { evaluateSentence(it, database, sequenceByLevel) } + + for (lessonRow in lessonRows) { + batch.add(lessonRow) + if (batch.size >= batchSize) { + database.insertCuratedLessonsBatch(batch) + totalInserted += batch.size + batch.clear() + } + } + + if (batch.isNotEmpty()) { + database.insertCuratedLessonsBatch(batch) + totalInserted += batch.size + batch.clear() + } + + return totalInserted + } + + private fun parseLine(line: String): TatoebaEntry? { + val parts = line.split("\t") + if (parts.size < TSV_PARTS_MIN) return null + + val spanish = parts[INDEX_SPANISH].trim() + val english = parts[INDEX_ENGLISH].trim() + if (spanish.isEmpty() || english.isEmpty()) return null + + val title = + if (parts.size > INDEX_TITLE && parts[INDEX_TITLE].isNotBlank()) + parts[INDEX_TITLE].trim() + else null + val concept = + if (parts.size > INDEX_CONCEPT && parts[INDEX_CONCEPT].isNotBlank()) + parts[INDEX_CONCEPT].trim() + else null + + return TatoebaEntry(spanish, english, title, concept) + } + + private fun evaluateSentence( + entry: TatoebaEntry, + database: SeedDatabase, + sequenceByLevel: MutableMap, + ): CuratedLessonRow? { + val scoreResult = SentenceScorer.scoreSentence(entry.spanish) { database.lookupWord(it) } + val level = scoreResult.level ?: return null + + val currentSeq = (sequenceByLevel[level] ?: 0) + 1 + sequenceByLevel[level] = currentSeq + + val title = entry.title ?: "Lesson $currentSeq" + val concept = entry.targetConcept ?: "Level ${level.ordinal} Syntactic Pattern" + + val contentJson = + buildJsonObject { + put("sentence_es", entry.spanish) + put("translation_en", entry.english) + put("score", scoreResult.score) + put("level", level.name) + put("word_count", scoreResult.wordCount) + put("rare_word_ratio", scoreResult.rareWordRatio) + put("cognate_ratio", scoreResult.cognateRatio) + put("has_complex_tense", scoreResult.hasComplexTense) + } + .toString() + + val lessonId = "lvl${level.ordinal}_${currentSeq.toString().padStart(LESSON_ID_PAD, '0')}" + return CuratedLessonRow( + lessonId = lessonId, + level = level.ordinal, + sequenceOrder = currentSeq, + title = title, + targetConcept = concept, + contentJson = contentJson, + ) + } + + private data class TatoebaEntry( + val spanish: String, + val english: String, + val title: String?, + val targetConcept: String?, + ) +} diff --git a/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt b/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt new file mode 100644 index 0000000..8ca1a7b --- /dev/null +++ b/tools/preprocessor/src/test/kotlin/com/anysoftkeyboard/calque/preprocessor/PreprocessorTest.kt @@ -0,0 +1,230 @@ +package com.anysoftkeyboard.calque.preprocessor + +import java.io.File +import java.nio.charset.StandardCharsets +import kotlin.io.path.createTempDirectory +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class PreprocessorTest { + + @Test + fun sublexReader_readsFrequenciesCorrectly() { + val content = + """ + Spelling FREQcount CD Rank + de 99999 100 1 + la 88888 95 2 + casa 5000 80 150 + """ + .trimIndent() + + val frequencies = SublexReader.readFrequencies(content.byteInputStream()) + + assertEquals(3, frequencies.size) + assertEquals(1, frequencies["de"]) + assertEquals(2, frequencies["la"]) + assertEquals(150, frequencies["casa"]) + } + + @Test + fun kaikkiProcessor_parsesLemmasAndFormsWithCognateAndFreq() { + val sublexFrequencies = mapOf("hablar" to 42, "conversación" to 110) + val line1 = + """{"word":"hablar","pos":"verb","senses":[{"glosses":["to speak"]}],""" + + """"forms":[{"form":"hablo","tags":["first-person"]}]}""" + val line2 = + """{"word":"conversación","pos":"noun","senses":[{"glosses":["conversation"]}],""" + + """"forms":[{"form":"conversaciones","tags":["plural"]}]}""" + val kaikkiJson = "$line1\n$line2" + + val inMemoryDb = SeedDatabase.open() + inMemoryDb.use { db -> + val count = + KaikkiProcessor.processStream(kaikkiJson.byteInputStream(), sublexFrequencies, db) + + // hablar: base + 1 form = 2 rows + // conversación: base + 1 form = 2 rows + assertEquals(4L, count) + + val hablarInfo = db.lookupWord("hablar") + assertNotNull(hablarInfo) + assertEquals(42, hablarInfo?.frequencyRank) + assertEquals(false, hablarInfo?.isCognate) + + val conversacionInfo = db.lookupWord("conversación") + assertNotNull(conversacionInfo) + assertEquals(110, conversacionInfo?.frequencyRank) + assertEquals(true, conversacionInfo?.isCognate) + + val habloInfo = db.lookupWord("hablo") + assertNotNull(habloInfo) + assertEquals(42, habloInfo?.frequencyRank) + assertEquals("first-person", habloInfo?.grammar) + } + } + + @Test + fun falseCognatesExpander_expandsAcrossDictionarySurfaces() { + val inMemoryDb = SeedDatabase.open() + inMemoryDb.use { db -> + db.insertDictionaryBatch( + listOf( + DictionaryRow( + surfaceForm = "carpeta", + lemma = "carpeta", + pos = "noun", + grammarInfo = "base", + englishGloss = "folder", + isCognate = false, + frequencyRank = 500, + ), + DictionaryRow( + surfaceForm = "carpetas", + lemma = "carpeta", + pos = "noun", + grammarInfo = "plural", + englishGloss = "folder", + isCognate = false, + frequencyRank = 500, + ), + ), + ) + + val falseCognatesJson = + """ + [ + { + "lemma": "carpeta", + "apparent_english": "carpet", + "actual_english": "folder / binder", + "real_spanish_for_apparent": "alfombra", + "severity": "HIGH_RISK", + "disambiguation_tip": "Keep documents in a carpeta, walk on an alfombra.", + "example_es": "Pon el papel en la carpeta.", + "example_en": "Put the paper in the folder." + } + ] + """ + .trimIndent() + + val expandedCount = + FalseCognatesExpander.expand(falseCognatesJson.byteInputStream(), db) + + assertEquals(2, expandedCount) + } + } + + @Test + fun tatoebaProcessor_scoresAndBucketsSentences() { + val inMemoryDb = SeedDatabase.open() + inMemoryDb.use { db -> + db.insertDictionaryBatch( + listOf( + DictionaryRow("la", "la", "article", "base", "the", false, 2), + DictionaryRow("música", "música", "noun", "base", "music", true, 200), + DictionaryRow("es", "ser", "verb", "present", "is", false, 3), + DictionaryRow( + "fantástica", + "fantástico", + "adj", + "base", + "fantastic", + true, + 300 + ), + ), + ) + + val tatoebaContent = + """ + La música es fantástica. Music is fantastic. Cognate Harmony Adjective Agreement + Hola. Hello. + """ + .trimIndent() + + val lessonsCount = + TatoebaProcessor.processSentences(tatoebaContent.byteInputStream(), db) + + assertEquals(1, lessonsCount) + } + } + + @Test + fun cognateRulesProcessor_insertsRules() { + val inMemoryDb = SeedDatabase.open() + inMemoryDb.use { db -> + val rulesJson = + """ + [ + { + "rule_id": "tion_to_cion", + "english_suffix": "-tion", + "spanish_suffix": "-ción", + "gender_rule": "Always feminine (la)", + "spelling_note": "Drop double consonants", + "examples": ["action -> acción", "nation -> nación"] + } + ] + """ + .trimIndent() + + val rulesCount = CognateRulesProcessor.processRules(rulesJson.byteInputStream(), db) + assertEquals(1, rulesCount) + } + } + + @Test + fun pipeline_executesEndToEndSuccessfully() { + val tempDir = createTempDirectory("calque_pipeline_test").toFile() + val dbFile = File(tempDir, "seed.db") + + val sublexFile = File(tempDir, "sublex.tsv") + sublexFile.writeText("hablar\t10\ncarpeta\t200\n", StandardCharsets.UTF_8) + + val sampleForm = """{"form":"carpetas","tags":["plural"]}""" + val sampleKaikki = + """{"word":"carpeta","pos":"noun","senses":[{"glosses":["folder"]}],"forms":[$sampleForm]}""" + val kaikkiFile = File(tempDir, "kaikki.jsonl") + kaikkiFile.writeText("$sampleKaikki\n", StandardCharsets.UTF_8) + + val fcFile = File(tempDir, "false_cognates.json") + fcFile.writeText( + """ + [ + { + "lemma": "carpeta", + "apparent_english": "carpet", + "actual_english": "folder", + "real_spanish_for_apparent": "alfombra", + "severity": "HIGH_RISK", + "disambiguation_tip": "A folder, not a carpet.", + "example_es": "Mi carpeta.", + "example_en": "My folder." + } + ] + """ + .trimIndent(), + StandardCharsets.UTF_8, + ) + + val config = + PipelineConfig( + dbFile = dbFile, + kaikkiFile = kaikkiFile, + sublexFile = sublexFile, + falseCognatesFile = fcFile, + ) + + val pipeline = PreprocessorPipeline(config) + val summary = pipeline.execute() + + assertTrue(dbFile.exists()) + assertTrue(summary.dictionaryRows >= 2) + assertEquals(2, summary.falseCognateRows) + + tempDir.deleteRecursively() + } +}