update dictionay

app/src/androidTest/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramAssetInstrumentedTest.kt

@@ -0,0 +1,44 @@
+package com.kazumaproject.markdownhelperkeyboard.converter.ngram
+
+import android.content.Context
+import androidx.test.core.app.ApplicationProvider
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import com.kazumaproject.graph.Node
+import org.junit.After
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertTrue
+import org.junit.Test
+import org.junit.runner.RunWith
+
+@RunWith(AndroidJUnit4::class)
+class SystemNgramAssetInstrumentedTest {
+    private val context: Context
+        get() = ApplicationProvider.getApplicationContext()
+
+    @After
+    fun tearDown() {
+        SystemNgramRuntime.resetForTesting()
+    }
+
+    @Test
+    fun version3AndVersion4AssetsLoadAndMatchOnPhysicalDevice() {
+        val dictionary = SystemNgramAssetLoader.load(context)
+
+        assertEquals(2_170, dictionary.ruleCount)
+        assertEquals(56_984 + 15_191, dictionary.storageBytes)
+        assertTrue(dictionary.matchesSingleNode(node("カワボ")))
+        assertFalse(dictionary.matchesSingleNode(node("存在しない表記")))
+    }
+
+    private fun node(word: String) = Node(
+        l = 1.toShort(),
+        r = 1.toShort(),
+        score = 0,
+        f = 0,
+        tango = word,
+        len = 1.toShort(),
+        yomiUsed = word,
+        sPos = 0,
+    )
+}

app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/CompositeSystemNgramDictionary.kt

@@ -0,0 +1,40 @@
+package com.kazumaproject.markdownhelperkeyboard.converter.ngram
+
+import com.kazumaproject.graph.Node
+
+/**
+ * Presents the version-3 n-gram and version-4 unigram assets as one scoreless dictionary.
+ *
+ * The two binary formats share the same packed storage layout but have different matching
+ * arities. Keeping the physical readers separate lets each reader retain its strict validation
+ * while the path search can query both through the existing runtime gate.
+ */
+class CompositeSystemNgramDictionary(
+    private val dictionaries: List<SystemNgramDictionary>,
+) : SystemNgramDictionary {
+    init {
+        require(dictionaries.isNotEmpty()) { "At least one system n-gram dictionary is required" }
+    }
+
+    override val ruleCount: Int = dictionaries.sumOf { it.ruleCount }
+    override val storageBytes: Int = dictionaries.sumOf { it.storageBytes }
+
+    override fun matchesSingleNode(node: Node): Boolean =
+        dictionaries.any { it.matchesSingleNode(node) }
+
+    override fun matches(
+        node0: Node,
+        node1: Node,
+        node2: Node?,
+        node3: Node?,
+        node4: Node?,
+    ): Boolean = dictionaries.any { dictionary ->
+        dictionary.matches(node0, node1, node2, node3, node4)
+    }
+
+    override fun mayMatchFirstPair(node0: Node, node1: Node): Boolean =
+        dictionaries.any { it.mayMatchFirstPair(node0, node1) }
+
+    override fun mayMatchFirstNode(node: Node): Boolean =
+        dictionaries.any { it.mayMatchFirstNode(node) }
+}

app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/PackedSystemNgramDictionary.kt

@@ -9,6 +9,7 @@ class PackedSystemNgramDictionary private constructor(
     private val bytes: ByteArray,
 ) : SystemNgramDictionary {
     private val buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN)
+    private val formatVersion = buffer.getInt(4)
     override val ruleCount: Int = buffer.getInt(8)
     override val storageBytes: Int = bytes.size
     private val contextCount = buffer.getInt(12)
@@ -34,7 +35,12 @@ class PackedSystemNgramDictionary private constructor(
     init {
         verify()
         firstNodeValuesByKind = arrayOfNulls(4)
-        firstPairHashesByKinds = buildFirstPairHashes(firstNodeValuesByKind)
+        firstPairHashesByKinds = if (formatVersion == VERSION) {
+            buildFirstPairHashes(firstNodeValuesByKind)
+        } else {
+            buildFirstNodeHashes(firstNodeValuesByKind)
+            arrayOfNulls(16)
+        }
         firstPairKindKeys = firstPairHashesByKinds.indices
             .filter { firstPairHashesByKinds[it] != null }
             .toIntArray()
@@ -43,13 +49,37 @@ class PackedSystemNgramDictionary private constructor(
             .toIntArray()
     }
 
+    override fun matchesSingleNode(node: Node): Boolean {
+        if (formatVersion != UNIGRAM_VERSION || node.tango == "BOS" || node.tango == "EOS") {
+            return false
+        }
+        val local = checkNotNull(scratch.get())
+        var signatureIndex = 0
+        while (signatureIndex < signatureCount) {
+            val signature = buffer.getInt(signaturesOffset + signatureIndex * 4)
+            val queryLength = encodeQuery(
+                target = local.query,
+                signature = signature,
+                node0 = node,
+                node1 = node,
+                node2 = null,
+                node3 = null,
+                node4 = null,
+            )
+            if (queryLength >= 0 && contains(local.query, queryLength, local.record)) return true
+            signatureIndex++
+        }
+        return false
+    }
+
     override fun matches(
         node0: Node,
         node1: Node,
         node2: Node?,
         node3: Node?,
         node4: Node?,
     ): Boolean {
+        if (formatVersion != VERSION) return false
         if (node0.tango == "BOS" || node1.tango == "EOS") return false
         val local = checkNotNull(scratch.get())
         var signatureIndex = 0
@@ -84,6 +114,7 @@ class PackedSystemNgramDictionary private constructor(
     }
 
     override fun mayMatchFirstPair(node0: Node, node1: Node): Boolean {
+        if (formatVersion != VERSION) return false
         if (node0.tango == "BOS" || node1.tango == "EOS") return false
         var node0WordHash = 0
         var node1WordHash = 0
@@ -170,6 +201,22 @@ class PackedSystemNgramDictionary private constructor(
         return result
     }
 
+    private fun buildFirstNodeHashes(
+        firstNodeValues: Array<LongHashSet?>,
+    ) {
+        val record = ByteArray(maxKeyBytes.coerceAtLeast(1))
+        repeat(ruleCount) { recordId ->
+            val recordLength = decodeRecord(recordId, record)
+            val signature = (record[0].toInt() and 0xff) or
+                ((record[1].toInt() and 0xff) shl 8)
+            val firstKind = (signature ushr 3) and 0x3
+            val first = readPrefixFeature(record, 2, recordLength, firstKind)
+            val firstNodeSet = firstNodeValues[firstKind]
+                ?: LongHashSet().also { firstNodeValues[firstKind] = it }
+            firstNodeSet.add(first.value.toLong())
+        }
+    }
+
     private fun readPrefixFeature(
         record: ByteArray,
         start: Int,
@@ -471,7 +518,9 @@ class PackedSystemNgramDictionary private constructor(
     private fun verify() {
         require(bytes.size >= HEADER_SIZE) { "Truncated n-gram header" }
         require(buffer.getInt(0) == MAGIC) { "Invalid n-gram magic" }
-        require(buffer.getInt(4) == VERSION) { "Unsupported n-gram version" }
+        require(formatVersion == VERSION || formatVersion == UNIGRAM_VERSION) {
+            "Unsupported n-gram version"
+        }
         require(buffer.getInt(56) == bytes.size) { "Invalid n-gram file size" }
         require(ruleCount > 0) { "Invalid n-gram rule count" }
         require(contextCount > 0) { "Invalid n-gram context count" }
@@ -504,7 +553,14 @@ class PackedSystemNgramDictionary private constructor(
         repeat(signatureCount) { index ->
             val signature = buffer.getInt(signaturesOffset + index * 4)
             val order = signature and 0x7
-            require(order in 2..5) { "Invalid n-gram signature order" }
+            if (formatVersion == VERSION) {
+                require(order in 2..5) { "Invalid n-gram signature order" }
+            } else {
+                require(order == 1) { "Invalid unigram signature order" }
+                require(((signature ushr 3) and 0x3) == KIND_WORD) {
+                    "Invalid unigram signature kind"
+                }
+            }
             repeat(order) { feature ->
                 require(((signature ushr (3 + feature * 2)) and 0x3) in KIND_WORD..KIND_ANY) {
                     "Invalid n-gram signature kind"
@@ -647,6 +703,7 @@ class PackedSystemNgramDictionary private constructor(
     companion object {
         private const val MAGIC = 0x4A4B4E47
         private const val VERSION = 3
+        private const val UNIGRAM_VERSION = 4
         private const val HEADER_SIZE = 80
         private const val BLOCK_SIZE = 16
         private const val MAX_BUCKET_COUNT = 65536

app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramAssetLoader.kt

@@ -1,21 +1,42 @@
 package com.kazumaproject.markdownhelperkeyboard.converter.ngram
 
 import android.content.Context
+import java.io.FileNotFoundException
 
 object SystemNgramAssetLoader {
-    private const val ASSET_PATH = "ngram/system_ngram.dat"
+    private const val NGRAM_ASSET_PATH = "ngram/system_ngram.dat"
+    private const val UNIGRAM_ASSET_PATH = "ngram/system_ngram_unigram.dat"
 
-    fun load(context: Context): SystemNgramDictionary = context.assets.open(ASSET_PATH).use { input ->
-        val expectedSize = input.available()
-        require(expectedSize > 0) { "Empty system n-gram asset" }
-        val bytes = ByteArray(expectedSize)
-        var offset = 0
-        while (offset < bytes.size) {
-            val count = input.read(bytes, offset, bytes.size - offset)
-            require(count > 0) { "Truncated system n-gram asset" }
-            offset += count
+    fun load(context: Context): SystemNgramDictionary {
+        val dictionaries = buildList {
+            add(loadAsset(context, NGRAM_ASSET_PATH))
+            try {
+                add(loadAsset(context, UNIGRAM_ASSET_PATH))
+            } catch (_: FileNotFoundException) {
+                // Keep compatibility with builds which predate the optional unigram asset.
+            }
+        }
+        return if (dictionaries.size == 1) {
+            dictionaries.single()
+        } else {
+            CompositeSystemNgramDictionary(dictionaries)
         }
-        require(input.read() == -1) { "System n-gram asset size changed while reading" }
-        PackedSystemNgramDictionary.read(bytes)
     }
+
+    private fun loadAsset(context: Context, assetPath: String): SystemNgramDictionary =
+        context.assets.open(assetPath).use { input ->
+            val expectedSize = input.available()
+            require(expectedSize > 0) { "Empty system n-gram asset: $assetPath" }
+            val bytes = ByteArray(expectedSize)
+            var offset = 0
+            while (offset < bytes.size) {
+                val count = input.read(bytes, offset, bytes.size - offset)
+                require(count > 0) { "Truncated system n-gram asset: $assetPath" }
+                offset += count
+            }
+            require(input.read() == -1) {
+                "System n-gram asset size changed while reading: $assetPath"
+            }
+            PackedSystemNgramDictionary.read(bytes)
+        }
 }

app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramDictionary.kt

@@ -6,6 +6,9 @@ interface SystemNgramDictionary {
     val ruleCount: Int
     val storageBytes: Int
 
+    /** Returns true when a single conversion node matches a unigram rule. */
+    fun matchesSingleNode(node: Node): Boolean = false
+
     fun matches(
         node0: Node,
         node1: Node,

app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/path_algorithm/FindPath.kt

@@ -2231,6 +2231,7 @@ class FindPath(
         if (dictionary.ruleCount == 0) return false
         var start = path.next
         while (start != null && start.node.tango != "EOS") {
+            if (dictionary.matchesSingleNode(start.node)) return true
             val second = start.next
             if (second == null || second.node.tango == "EOS") return false
             if (
@@ -2250,9 +2251,10 @@ class FindPath(
     /**
      * Proves that no packed system n-gram can occur in the pruned lattice before requesting the
      * old 32–64 candidate safety window.  Every candidate path consists of nodes whose reading
-     * ranges are adjacent, so every matching rule must have a first pair represented by one of
-     * these edges.  The packed dictionary's prefix index may return a conservative false positive
-     * but never a false negative; unknown dictionary implementations keep the old search path.
+     * ranges are adjacent, so every matching rule must have a first node or first pair represented
+     * by this lattice.  The packed dictionary's prefix index may return a conservative false
+     * positive but never a false negative; unknown dictionary implementations keep the old search
+     * path.
      */
     private fun latticeMayContainSystemNgram(
         graph: MutableMap<Int, MutableList<Node>>,
@@ -2263,7 +2265,7 @@ class FindPath(
             val leftNodes = graph[leftEnd] ?: continue
             for (leftNode in leftNodes) {
                 if (leftNode.sPos + leftNode.len.toInt() != leftEnd) continue
-                if (!dictionary.mayMatchFirstNode(leftNode)) continue
+                if (dictionary.mayMatchFirstNode(leftNode)) return true
                 for (rightEnd in leftEnd + 1..length) {
                     val rightNodes = graph[rightEnd] ?: continue
                     for (rightNode in rightNodes) {

app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/PackedSystemNgramDictionaryTest.kt

@@ -15,6 +15,11 @@ class PackedSystemNgramDictionaryTest {
         get() = checkNotNull(javaClass.classLoader?.getResourceAsStream("ngram/system_ngram_v3_test.dat"))
             .use { it.readBytes() }
 
+    private val unigramFixture: ByteArray
+        get() = checkNotNull(
+            javaClass.classLoader?.getResourceAsStream("ngram/system_ngram_unigram_v4_test.dat"),
+        ).use { it.readBytes() }
+
     @Test
     fun exactRuleMatchesAndOneCharacterDifferenceDoesNot() {
         val dictionary = dictionary()
@@ -78,6 +83,18 @@ class PackedSystemNgramDictionaryTest {
         assertTrue(dictionary.matches(node("一"), node("二"), node("三"), node("四"), node("五")))
     }
 
+    @Test
+    fun version4MatchesSingleWordRulesWithoutParticipatingInPairMatches() {
+        val dictionary = PackedSystemNgramDictionary.read(unigramFixture)
+
+        assertTrue(dictionary.matchesSingleNode(node("カワボ")))
+        assertFalse(dictionary.matchesSingleNode(node("存在しない表記")))
+        assertTrue(dictionary.mayMatchFirstNode(node("カワボ")))
+        assertFalse(dictionary.mayMatchFirstNode(node("存在しない表記")))
+        assertFalse(dictionary.matches(node("カワボ"), node("候補"), null, null, null))
+        assertFalse(dictionary.mayMatchFirstPair(node("カワボ"), node("候補")))
+    }
+
     @Test
     fun rejectsBadMagicVersionCrcOffsetsAndTruncation() {
         assertRejected(fixture.copyOf().also { it[0] = 0 })

app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramRuntimeTest.kt

@@ -2,6 +2,7 @@ package com.kazumaproject.markdownhelperkeyboard.converter.ngram
 
 import android.content.Context
 import androidx.test.core.app.ApplicationProvider
+import com.kazumaproject.graph.Node
 import org.junit.After
 import org.junit.Assert.assertEquals
 import org.junit.Assert.assertSame
@@ -46,4 +47,25 @@ class SystemNgramRuntimeTest {
         SystemNgramRuntime.setEnabled(context, true)
         assertSame(loaded, SystemNgramRuntime.current())
     }
+
+    @Test
+    fun loadsVersion3AndVersion4AssetsAsOneDictionary() {
+        SystemNgramRuntime.initialize(context, true)
+
+        val loaded = SystemNgramRuntime.loadedDictionary()
+        assertEquals(2_170, loaded.ruleCount)
+        assertEquals(56_984 + 15_191, loaded.storageBytes)
+        assertTrue(loaded.matchesSingleNode(node("カワボ")))
+    }
+
+    private fun node(word: String) = Node(
+        l = 1.toShort(),
+        r = 1.toShort(),
+        score = 0,
+        f = 0,
+        tango = word,
+        len = 1.toShort(),
+        yomiUsed = word,
+        sPos = 0,
+    )
 }