[LLM] Implement build-time seed.db preprocessor pipeline

tools/preprocessor/BUILD.bazel

@@ -1 +1,37 @@
+load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_binary", "kt_jvm_library", "kt_jvm_test")
+
 package(default_visibility = ["//visibility:public"])
+
+kt_jvm_library(
+    name = "preprocessor_lib",
+    srcs = glob(["src/main/kotlin/**/*.kt"]),
+    visibility = ["//visibility:public"],
+    deps = [
+        "//core/linguistics",
+        "@maven//:org_jetbrains_kotlinx_kotlinx_serialization_json",
+        "@maven//:org_xerial_sqlite_jdbc",
+    ],
+)
+
+kt_jvm_binary(
+    name = "preprocessor",
+    main_class = "com.anysoftkeyboard.calque.preprocessor.MainKt",
+    visibility = ["//visibility:public"],
+    runtime_deps = [
+        ":preprocessor_lib",
+    ],
+)
+
+kt_jvm_test(
+    name = "preprocessor_test",
+    size = "small",
+    srcs = glob(["src/test/kotlin/**/*.kt"]),
+    test_class = "com.anysoftkeyboard.calque.preprocessor.PreprocessorTest",
+    deps = [
+        ":preprocessor_lib",
+        "//core/linguistics",
+        "@maven//:junit_junit",
+        "@maven//:org_jetbrains_kotlinx_kotlinx_serialization_json",
+        "@maven//:org_xerial_sqlite_jdbc",
+    ],
+)

tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/CognateRulesProcessor.kt

@@ -0,0 +1,48 @@
+package com.anysoftkeyboard.calque.preprocessor
+
+import java.io.InputStream
+import java.io.InputStreamReader
+import java.nio.charset.StandardCharsets
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
+
+/** Seeds the cognate_rules table from a JSON definition. */
+object CognateRulesProcessor {
+
+    /**
+     * Reads cognate rules from [inputStream] and inserts them into [database].
+     *
+     * @return Count of cognate rules inserted.
+     */
+    fun processRules(inputStream: InputStream, database: SeedDatabase): Int {
+        val jsonText = InputStreamReader(inputStream, StandardCharsets.UTF_8).use { it.readText() }
+        val array = Json.parseToJsonElement(jsonText).jsonArray
+        val rows = mutableListOf<CognateRuleRow>()
+
+        for (elem in array) {
+            val obj = elem.jsonObject
+            val ruleId = obj["rule_id"]?.jsonPrimitive?.content ?: continue
+            val englishSuffix = obj["english_suffix"]?.jsonPrimitive?.content.orEmpty()
+            val spanishSuffix = obj["spanish_suffix"]?.jsonPrimitive?.content.orEmpty()
+            val genderRule = obj["gender_rule"]?.jsonPrimitive?.content
+            val spellingNote = obj["spelling_note"]?.jsonPrimitive?.content.orEmpty()
+            val examplesJson = obj["examples"]?.toString() ?: "[]"
+
+            rows.add(
+                CognateRuleRow(
+                    ruleId = ruleId,
+                    englishSuffix = englishSuffix,
+                    spanishSuffix = spanishSuffix,
+                    genderRule = genderRule,
+                    spellingNote = spellingNote,
+                    examplesJson = examplesJson,
+                ),
+            )
+        }
+
+        database.insertCognateRulesBatch(rows)
+        return rows.size
+    }
+}

tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/FalseCognatesExpander.kt

@@ -0,0 +1,86 @@
+package com.anysoftkeyboard.calque.preprocessor
+
+import java.io.InputStream
+import java.io.InputStreamReader
+import java.nio.charset.StandardCharsets
+import java.util.Locale
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
+
+/**
+ * Expands curated false cognate seeds across all conjugated and inflected forms in the dictionary.
+ *
+ * Implements Step 3 from INGESTION.md.
+ */
+object FalseCognatesExpander {
+
+    /**
+     * Expands false cognates from an input stream and stores them into the database.
+     *
+     * @return Count of false cognate entries inserted.
+     */
+    fun expand(inputStream: InputStream, database: SeedDatabase): Int {
+        val seeds = parseSeeds(inputStream)
+        val rows = mutableListOf<FalseCognateRow>()
+
+        for (seed in seeds) {
+            val matchingSurfaces = database.querySurfacesForLemma(seed.lemma).toMutableSet()
+            // Ensure the lowercased base lemma is always included
+            matchingSurfaces.add(seed.lemma.lowercase(Locale.ROOT))
+
+            for (surface in matchingSurfaces) {
+                rows.add(
+                    FalseCognateRow(
+                        surfaceForm = surface,
+                        lemma = seed.lemma,
+                        apparentEnglish = seed.apparentEnglish,
+                        actualEnglish = seed.actualEnglish,
+                        realSpanishForApparent = seed.realSpanishForApparent,
+                        severity = seed.severity,
+                        disambiguationTip = seed.disambiguationTip,
+                        exampleEs = seed.exampleEs,
+                        exampleEn = seed.exampleEn,
+                    ),
+                )
+            }
+        }
+
+        database.insertFalseCognatesBatch(rows)
+        return rows.size
+    }
+
+    /** Parses the false cognates seed JSON stream into a list of [FalseCognateSeed] objects. */
+    fun parseSeeds(inputStream: InputStream): List<FalseCognateSeed> {
+        val jsonText = InputStreamReader(inputStream, StandardCharsets.UTF_8).use { it.readText() }
+        val array = Json.parseToJsonElement(jsonText).jsonArray
+        val seeds = mutableListOf<FalseCognateSeed>()
+
+        for (elem in array) {
+            val obj = elem.jsonObject
+            val lemma = obj["lemma"]?.jsonPrimitive?.content ?: continue
+            val apparentEnglish = obj["apparent_english"]?.jsonPrimitive?.content.orEmpty()
+            val actualEnglish = obj["actual_english"]?.jsonPrimitive?.content.orEmpty()
+            val realSpanish = obj["real_spanish_for_apparent"]?.jsonPrimitive?.content.orEmpty()
+            val severity = obj["severity"]?.jsonPrimitive?.content ?: "HIGH_RISK"
+            val tip = obj["disambiguation_tip"]?.jsonPrimitive?.content.orEmpty()
+            val exampleEs = obj["example_es"]?.jsonPrimitive?.content.orEmpty()
+            val exampleEn = obj["example_en"]?.jsonPrimitive?.content.orEmpty()
+
+            seeds.add(
+                FalseCognateSeed(
+                    lemma = lemma,
+                    apparentEnglish = apparentEnglish,
+                    actualEnglish = actualEnglish,
+                    realSpanishForApparent = realSpanish,
+                    severity = severity,
+                    disambiguationTip = tip,
+                    exampleEs = exampleEs,
+                    exampleEn = exampleEn,
+                ),
+            )
+        }
+        return seeds
+    }
+}

tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/KaikkiProcessor.kt

@@ -0,0 +1,131 @@
+package com.anysoftkeyboard.calque.preprocessor
+
+import com.anysoftkeyboard.calque.linguistics.CognateDetector
+import java.io.BufferedReader
+import java.io.InputStream
+import java.io.InputStreamReader
+import java.nio.charset.StandardCharsets
+import java.util.Locale
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonObject
+import kotlinx.serialization.json.jsonPrimitive
+
+/**
+ * Processor for streaming Kaikki JSONL dictionary dumps and populating the dictionary table.
+ *
+ * Implements Step 1 from INGESTION.md.
+ */
+object KaikkiProcessor {
+    private const val DEFAULT_BATCH_SIZE = 5000
+
+    /**
+     * Streams the Kaikki JSONL input, extracts lemmas and inflected forms, tags cognates, resolves
+     * SUBTLEX frequencies, and flattens them into the dictionary table.
+     *
+     * @return The count of total dictionary rows inserted.
+     */
+    fun processStream(
+        inputStream: InputStream,
+        sublexFrequencies: Map<String, Int>,
+        database: SeedDatabase,
+        batchSize: Int = DEFAULT_BATCH_SIZE,
+    ): Long {
+        val reader = BufferedReader(InputStreamReader(inputStream, StandardCharsets.UTF_8))
+        val batch = mutableListOf<DictionaryRow>()
+        var insertedRows = 0L
+
+        for (rawLine in reader.lineSequence()) {
+            val line = rawLine.trim()
+            if (line.isEmpty()) continue
+
+            val rows = parseKaikkiLine(line, sublexFrequencies)
+            batch.addAll(rows)
+
+            if (batch.size >= batchSize) {
+                database.insertDictionaryBatch(batch)
+                insertedRows += batch.size
+                batch.clear()
+            }
+        }
+
+        if (batch.isNotEmpty()) {
+            database.insertDictionaryBatch(batch)
+            insertedRows += batch.size
+            batch.clear()
+        }
+
+        return insertedRows
+    }
+
+    /** Parses a single Kaikki JSON line into base and inflected [DictionaryRow]s. */
+    fun parseKaikkiLine(
+        jsonLine: String,
+        sublexFrequencies: Map<String, Int>,
+    ): List<DictionaryRow> {
+        val jsonElement = Json.parseToJsonElement(jsonLine).jsonObject
+        val lemma = jsonElement["word"]?.jsonPrimitive?.content ?: return emptyList()
+        val pos = jsonElement["pos"]?.jsonPrimitive?.content ?: ""
+        val primaryGloss = extractPrimaryGloss(jsonElement)
+        val normalizedLemma = lemma.lowercase(Locale.ROOT)
+        val freqRank = sublexFrequencies[normalizedLemma]
+        val isCognate = CognateDetector.detectCognate(lemma, primaryGloss)
+
+        val results = mutableListOf<DictionaryRow>()
+
+        // 1. Base lemma row
+        results.add(
+            DictionaryRow(
+                surfaceForm = normalizedLemma,
+                lemma = lemma,
+                pos = pos,
+                grammarInfo = "base",
+                englishGloss = primaryGloss,
+                isCognate = isCognate,
+                frequencyRank = freqRank,
+            ),
+        )
+
+        // 2. Expand inflected forms
+        val formsArray = jsonElement["forms"]?.jsonArray
+        if (formsArray != null) {
+            for (formElement in formsArray) {
+                val formObj = formElement.jsonObject
+                val surface =
+                    formObj["form"]?.jsonPrimitive?.content
+                        ?: formObj["surface"]?.jsonPrimitive?.content
+                        ?: continue
+                val tags =
+                    formObj["tags"]?.jsonArray?.map { it.jsonPrimitive.content } ?: emptyList()
+                val grammar = if (tags.isNotEmpty()) tags.joinToString(", ") else null
+
+                results.add(
+                    DictionaryRow(
+                        surfaceForm = surface.lowercase(Locale.ROOT),
+                        lemma = lemma,
+                        pos = pos,
+                        grammarInfo = grammar,
+                        englishGloss = primaryGloss,
+                        isCognate = isCognate,
+                        frequencyRank = freqRank,
+                    ),
+                )
+            }
+        }
+
+        return results
+    }
+
+    private fun extractPrimaryGloss(jsonObj: kotlinx.serialization.json.JsonObject): String {
+        val senses = jsonObj["senses"]?.jsonArray ?: return ""
+        for (senseElement in senses) {
+            val senseObj = senseElement.jsonObject
+            val glosses = senseObj["glosses"]?.jsonArray
+            val firstGloss = glosses?.firstOrNull()?.jsonPrimitive?.content
+            if (!firstGloss.isNullOrBlank()) {
+                return firstGloss
+            }
+        }
+        return ""
+    }
+}

tools/preprocessor/src/main/kotlin/com/anysoftkeyboard/calque/preprocessor/Main.kt

@@ -0,0 +1,66 @@
+package com.anysoftkeyboard.calque.preprocessor
+
+import java.io.File
+import kotlin.system.exitProcess
+
+/** CLI binary entry point for the Calque seed.db build-time preprocessor. */
+fun main(args: Array<String>) {
+    val options = parseCommandLineArgs(args)
+
+    val dbPath = options["--db"]
+    if (dbPath == null) {
+        System.err.println(
+            """
+            Usage: preprocessor --db <output-path-to-seed.db> [options]
+
+            Options:
+              --kaikki <path>          Path to Kaikki Spanish JSONL dump
+              --sublex <path>          Path to SUBTLEX-ESP frequency file
+              --false-cognates <path>  Path to false_cognates_seed.json
+              --tatoeba <path>         Path to Tatoeba Spanish-English TSV file
+              --cognate-rules <path>   Path to cognate_rules_seed.json
+            """
+                .trimIndent(),
+        )
+        exitProcess(1)
+    }
+
+    val config =
+        PipelineConfig(
+            dbFile = File(dbPath),
+            kaikkiFile = options["--kaikki"]?.let { File(it) },
+            sublexFile = options["--sublex"]?.let { File(it) },
+            falseCognatesFile = options["--false-cognates"]?.let { File(it) },
+            tatoebaFile = options["--tatoeba"]?.let { File(it) },
+            cognateRulesFile = options["--cognate-rules"]?.let { File(it) },
+        )
+
+    println("Starting Calque Data Ingestion Pipeline...")
+    println("Target database: ${config.dbFile.absolutePath}")
+
+    val startTime = System.currentTimeMillis()
+    val pipeline = PreprocessorPipeline(config)
+    val summary = pipeline.execute()
+    val elapsed = System.currentTimeMillis() - startTime
+
+    println("Pipeline completed successfully in ${elapsed}ms:")
+    println("  - Dictionary entries inserted: ${summary.dictionaryRows}")
+    println("  - False cognate forms expanded: ${summary.falseCognateRows}")
+    println("  - Curated lesson sentences accepted: ${summary.curatedLessonRows}")
+    println("  - Cognate transformer rules seeded: ${summary.cognateRuleRows}")
+}
+
+private fun parseCommandLineArgs(args: Array<String>): Map<String, String> {
+    val options = mutableMapOf<String, String>()
+    var i = 0
+    while (i < args.size) {
+        val arg = args[i]
+        if (arg.startsWith("--") && i + 1 < args.size && !args[i + 1].startsWith("--")) {
+            options[arg] = args[i + 1]
+            i += 2
+        } else {
+            i++
+        }
+    }
+    return options
+}