diff --git a/app/src/androidTest/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramAssetInstrumentedTest.kt b/app/src/androidTest/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramAssetInstrumentedTest.kt new file mode 100644 index 00000000..56f51a96 --- /dev/null +++ b/app/src/androidTest/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramAssetInstrumentedTest.kt @@ -0,0 +1,44 @@ +package com.kazumaproject.markdownhelperkeyboard.converter.ngram + +import android.content.Context +import androidx.test.core.app.ApplicationProvider +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.kazumaproject.graph.Node +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class SystemNgramAssetInstrumentedTest { + private val context: Context + get() = ApplicationProvider.getApplicationContext() + + @After + fun tearDown() { + SystemNgramRuntime.resetForTesting() + } + + @Test + fun version3AndVersion4AssetsLoadAndMatchOnPhysicalDevice() { + val dictionary = SystemNgramAssetLoader.load(context) + + assertEquals(2_170, dictionary.ruleCount) + assertEquals(56_984 + 15_191, dictionary.storageBytes) + assertTrue(dictionary.matchesSingleNode(node("カワボ"))) + assertFalse(dictionary.matchesSingleNode(node("存在しない表記"))) + } + + private fun node(word: String) = Node( + l = 1.toShort(), + r = 1.toShort(), + score = 0, + f = 0, + tango = word, + len = 1.toShort(), + yomiUsed = word, + sPos = 0, + ) +} diff --git a/app/src/main/assets/ngram/system_ngram.dat b/app/src/main/assets/ngram/system_ngram.dat index e21650bf..cb4f3e5c 100644 Binary files a/app/src/main/assets/ngram/system_ngram.dat and b/app/src/main/assets/ngram/system_ngram.dat differ diff --git a/app/src/main/assets/ngram/system_ngram_unigram.dat b/app/src/main/assets/ngram/system_ngram_unigram.dat new file mode 100644 index 00000000..5a52f597 Binary files /dev/null and b/app/src/main/assets/ngram/system_ngram_unigram.dat differ diff --git a/app/src/main/assets/system/tango.dat.zip b/app/src/main/assets/system/tango.dat.zip index f8c9e144..569529f5 100644 Binary files a/app/src/main/assets/system/tango.dat.zip and b/app/src/main/assets/system/tango.dat.zip differ diff --git a/app/src/main/assets/system/token.dat.zip b/app/src/main/assets/system/token.dat.zip index e1cf7b06..d40a51b6 100644 Binary files a/app/src/main/assets/system/token.dat.zip and b/app/src/main/assets/system/token.dat.zip differ diff --git a/app/src/main/assets/system/yomi.dat.zip b/app/src/main/assets/system/yomi.dat.zip index c491c387..22885385 100644 Binary files a/app/src/main/assets/system/yomi.dat.zip and b/app/src/main/assets/system/yomi.dat.zip differ diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/CompositeSystemNgramDictionary.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/CompositeSystemNgramDictionary.kt new file mode 100644 index 00000000..a7981bc6 --- /dev/null +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/CompositeSystemNgramDictionary.kt @@ -0,0 +1,40 @@ +package com.kazumaproject.markdownhelperkeyboard.converter.ngram + +import com.kazumaproject.graph.Node + +/** + * Presents the version-3 n-gram and version-4 unigram assets as one scoreless dictionary. + * + * The two binary formats share the same packed storage layout but have different matching + * arities. Keeping the physical readers separate lets each reader retain its strict validation + * while the path search can query both through the existing runtime gate. + */ +class CompositeSystemNgramDictionary( + private val dictionaries: List, +) : SystemNgramDictionary { + init { + require(dictionaries.isNotEmpty()) { "At least one system n-gram dictionary is required" } + } + + override val ruleCount: Int = dictionaries.sumOf { it.ruleCount } + override val storageBytes: Int = dictionaries.sumOf { it.storageBytes } + + override fun matchesSingleNode(node: Node): Boolean = + dictionaries.any { it.matchesSingleNode(node) } + + override fun matches( + node0: Node, + node1: Node, + node2: Node?, + node3: Node?, + node4: Node?, + ): Boolean = dictionaries.any { dictionary -> + dictionary.matches(node0, node1, node2, node3, node4) + } + + override fun mayMatchFirstPair(node0: Node, node1: Node): Boolean = + dictionaries.any { it.mayMatchFirstPair(node0, node1) } + + override fun mayMatchFirstNode(node: Node): Boolean = + dictionaries.any { it.mayMatchFirstNode(node) } +} diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/PackedSystemNgramDictionary.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/PackedSystemNgramDictionary.kt index 746752c5..25175430 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/PackedSystemNgramDictionary.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/PackedSystemNgramDictionary.kt @@ -9,6 +9,7 @@ class PackedSystemNgramDictionary private constructor( private val bytes: ByteArray, ) : SystemNgramDictionary { private val buffer = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN) + private val formatVersion = buffer.getInt(4) override val ruleCount: Int = buffer.getInt(8) override val storageBytes: Int = bytes.size private val contextCount = buffer.getInt(12) @@ -34,7 +35,12 @@ class PackedSystemNgramDictionary private constructor( init { verify() firstNodeValuesByKind = arrayOfNulls(4) - firstPairHashesByKinds = buildFirstPairHashes(firstNodeValuesByKind) + firstPairHashesByKinds = if (formatVersion == VERSION) { + buildFirstPairHashes(firstNodeValuesByKind) + } else { + buildFirstNodeHashes(firstNodeValuesByKind) + arrayOfNulls(16) + } firstPairKindKeys = firstPairHashesByKinds.indices .filter { firstPairHashesByKinds[it] != null } .toIntArray() @@ -43,6 +49,29 @@ class PackedSystemNgramDictionary private constructor( .toIntArray() } + override fun matchesSingleNode(node: Node): Boolean { + if (formatVersion != UNIGRAM_VERSION || node.tango == "BOS" || node.tango == "EOS") { + return false + } + val local = checkNotNull(scratch.get()) + var signatureIndex = 0 + while (signatureIndex < signatureCount) { + val signature = buffer.getInt(signaturesOffset + signatureIndex * 4) + val queryLength = encodeQuery( + target = local.query, + signature = signature, + node0 = node, + node1 = node, + node2 = null, + node3 = null, + node4 = null, + ) + if (queryLength >= 0 && contains(local.query, queryLength, local.record)) return true + signatureIndex++ + } + return false + } + override fun matches( node0: Node, node1: Node, @@ -50,6 +79,7 @@ class PackedSystemNgramDictionary private constructor( node3: Node?, node4: Node?, ): Boolean { + if (formatVersion != VERSION) return false if (node0.tango == "BOS" || node1.tango == "EOS") return false val local = checkNotNull(scratch.get()) var signatureIndex = 0 @@ -84,6 +114,7 @@ class PackedSystemNgramDictionary private constructor( } override fun mayMatchFirstPair(node0: Node, node1: Node): Boolean { + if (formatVersion != VERSION) return false if (node0.tango == "BOS" || node1.tango == "EOS") return false var node0WordHash = 0 var node1WordHash = 0 @@ -170,6 +201,22 @@ class PackedSystemNgramDictionary private constructor( return result } + private fun buildFirstNodeHashes( + firstNodeValues: Array, + ) { + val record = ByteArray(maxKeyBytes.coerceAtLeast(1)) + repeat(ruleCount) { recordId -> + val recordLength = decodeRecord(recordId, record) + val signature = (record[0].toInt() and 0xff) or + ((record[1].toInt() and 0xff) shl 8) + val firstKind = (signature ushr 3) and 0x3 + val first = readPrefixFeature(record, 2, recordLength, firstKind) + val firstNodeSet = firstNodeValues[firstKind] + ?: LongHashSet().also { firstNodeValues[firstKind] = it } + firstNodeSet.add(first.value.toLong()) + } + } + private fun readPrefixFeature( record: ByteArray, start: Int, @@ -471,7 +518,9 @@ class PackedSystemNgramDictionary private constructor( private fun verify() { require(bytes.size >= HEADER_SIZE) { "Truncated n-gram header" } require(buffer.getInt(0) == MAGIC) { "Invalid n-gram magic" } - require(buffer.getInt(4) == VERSION) { "Unsupported n-gram version" } + require(formatVersion == VERSION || formatVersion == UNIGRAM_VERSION) { + "Unsupported n-gram version" + } require(buffer.getInt(56) == bytes.size) { "Invalid n-gram file size" } require(ruleCount > 0) { "Invalid n-gram rule count" } require(contextCount > 0) { "Invalid n-gram context count" } @@ -504,7 +553,14 @@ class PackedSystemNgramDictionary private constructor( repeat(signatureCount) { index -> val signature = buffer.getInt(signaturesOffset + index * 4) val order = signature and 0x7 - require(order in 2..5) { "Invalid n-gram signature order" } + if (formatVersion == VERSION) { + require(order in 2..5) { "Invalid n-gram signature order" } + } else { + require(order == 1) { "Invalid unigram signature order" } + require(((signature ushr 3) and 0x3) == KIND_WORD) { + "Invalid unigram signature kind" + } + } repeat(order) { feature -> require(((signature ushr (3 + feature * 2)) and 0x3) in KIND_WORD..KIND_ANY) { "Invalid n-gram signature kind" @@ -647,6 +703,7 @@ class PackedSystemNgramDictionary private constructor( companion object { private const val MAGIC = 0x4A4B4E47 private const val VERSION = 3 + private const val UNIGRAM_VERSION = 4 private const val HEADER_SIZE = 80 private const val BLOCK_SIZE = 16 private const val MAX_BUCKET_COUNT = 65536 diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramAssetLoader.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramAssetLoader.kt index 23dc0e08..fa0c2a20 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramAssetLoader.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramAssetLoader.kt @@ -1,21 +1,42 @@ package com.kazumaproject.markdownhelperkeyboard.converter.ngram import android.content.Context +import java.io.FileNotFoundException object SystemNgramAssetLoader { - private const val ASSET_PATH = "ngram/system_ngram.dat" + private const val NGRAM_ASSET_PATH = "ngram/system_ngram.dat" + private const val UNIGRAM_ASSET_PATH = "ngram/system_ngram_unigram.dat" - fun load(context: Context): SystemNgramDictionary = context.assets.open(ASSET_PATH).use { input -> - val expectedSize = input.available() - require(expectedSize > 0) { "Empty system n-gram asset" } - val bytes = ByteArray(expectedSize) - var offset = 0 - while (offset < bytes.size) { - val count = input.read(bytes, offset, bytes.size - offset) - require(count > 0) { "Truncated system n-gram asset" } - offset += count + fun load(context: Context): SystemNgramDictionary { + val dictionaries = buildList { + add(loadAsset(context, NGRAM_ASSET_PATH)) + try { + add(loadAsset(context, UNIGRAM_ASSET_PATH)) + } catch (_: FileNotFoundException) { + // Keep compatibility with builds which predate the optional unigram asset. + } + } + return if (dictionaries.size == 1) { + dictionaries.single() + } else { + CompositeSystemNgramDictionary(dictionaries) } - require(input.read() == -1) { "System n-gram asset size changed while reading" } - PackedSystemNgramDictionary.read(bytes) } + + private fun loadAsset(context: Context, assetPath: String): SystemNgramDictionary = + context.assets.open(assetPath).use { input -> + val expectedSize = input.available() + require(expectedSize > 0) { "Empty system n-gram asset: $assetPath" } + val bytes = ByteArray(expectedSize) + var offset = 0 + while (offset < bytes.size) { + val count = input.read(bytes, offset, bytes.size - offset) + require(count > 0) { "Truncated system n-gram asset: $assetPath" } + offset += count + } + require(input.read() == -1) { + "System n-gram asset size changed while reading: $assetPath" + } + PackedSystemNgramDictionary.read(bytes) + } } diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramDictionary.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramDictionary.kt index e453cfbd..20f84c8c 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramDictionary.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramDictionary.kt @@ -6,6 +6,9 @@ interface SystemNgramDictionary { val ruleCount: Int val storageBytes: Int + /** Returns true when a single conversion node matches a unigram rule. */ + fun matchesSingleNode(node: Node): Boolean = false + fun matches( node0: Node, node1: Node, diff --git a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/path_algorithm/FindPath.kt b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/path_algorithm/FindPath.kt index d947b871..b9b04245 100644 --- a/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/path_algorithm/FindPath.kt +++ b/app/src/main/java/com/kazumaproject/markdownhelperkeyboard/converter/path_algorithm/FindPath.kt @@ -2231,6 +2231,7 @@ class FindPath( if (dictionary.ruleCount == 0) return false var start = path.next while (start != null && start.node.tango != "EOS") { + if (dictionary.matchesSingleNode(start.node)) return true val second = start.next if (second == null || second.node.tango == "EOS") return false if ( @@ -2250,9 +2251,10 @@ class FindPath( /** * Proves that no packed system n-gram can occur in the pruned lattice before requesting the * old 32–64 candidate safety window. Every candidate path consists of nodes whose reading - * ranges are adjacent, so every matching rule must have a first pair represented by one of - * these edges. The packed dictionary's prefix index may return a conservative false positive - * but never a false negative; unknown dictionary implementations keep the old search path. + * ranges are adjacent, so every matching rule must have a first node or first pair represented + * by this lattice. The packed dictionary's prefix index may return a conservative false + * positive but never a false negative; unknown dictionary implementations keep the old search + * path. */ private fun latticeMayContainSystemNgram( graph: MutableMap>, @@ -2263,7 +2265,7 @@ class FindPath( val leftNodes = graph[leftEnd] ?: continue for (leftNode in leftNodes) { if (leftNode.sPos + leftNode.len.toInt() != leftEnd) continue - if (!dictionary.mayMatchFirstNode(leftNode)) continue + if (dictionary.mayMatchFirstNode(leftNode)) return true for (rightEnd in leftEnd + 1..length) { val rightNodes = graph[rightEnd] ?: continue for (rightNode in rightNodes) { diff --git a/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/PackedSystemNgramDictionaryTest.kt b/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/PackedSystemNgramDictionaryTest.kt index 0ee2133b..cd270fed 100644 --- a/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/PackedSystemNgramDictionaryTest.kt +++ b/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/PackedSystemNgramDictionaryTest.kt @@ -15,6 +15,11 @@ class PackedSystemNgramDictionaryTest { get() = checkNotNull(javaClass.classLoader?.getResourceAsStream("ngram/system_ngram_v3_test.dat")) .use { it.readBytes() } + private val unigramFixture: ByteArray + get() = checkNotNull( + javaClass.classLoader?.getResourceAsStream("ngram/system_ngram_unigram_v4_test.dat"), + ).use { it.readBytes() } + @Test fun exactRuleMatchesAndOneCharacterDifferenceDoesNot() { val dictionary = dictionary() @@ -78,6 +83,18 @@ class PackedSystemNgramDictionaryTest { assertTrue(dictionary.matches(node("一"), node("二"), node("三"), node("四"), node("五"))) } + @Test + fun version4MatchesSingleWordRulesWithoutParticipatingInPairMatches() { + val dictionary = PackedSystemNgramDictionary.read(unigramFixture) + + assertTrue(dictionary.matchesSingleNode(node("カワボ"))) + assertFalse(dictionary.matchesSingleNode(node("存在しない表記"))) + assertTrue(dictionary.mayMatchFirstNode(node("カワボ"))) + assertFalse(dictionary.mayMatchFirstNode(node("存在しない表記"))) + assertFalse(dictionary.matches(node("カワボ"), node("候補"), null, null, null)) + assertFalse(dictionary.mayMatchFirstPair(node("カワボ"), node("候補"))) + } + @Test fun rejectsBadMagicVersionCrcOffsetsAndTruncation() { assertRejected(fixture.copyOf().also { it[0] = 0 }) diff --git a/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramRuntimeTest.kt b/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramRuntimeTest.kt index f94411f2..fe7b8a70 100644 --- a/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramRuntimeTest.kt +++ b/app/src/test/java/com/kazumaproject/markdownhelperkeyboard/converter/ngram/SystemNgramRuntimeTest.kt @@ -2,6 +2,7 @@ package com.kazumaproject.markdownhelperkeyboard.converter.ngram import android.content.Context import androidx.test.core.app.ApplicationProvider +import com.kazumaproject.graph.Node import org.junit.After import org.junit.Assert.assertEquals import org.junit.Assert.assertSame @@ -46,4 +47,25 @@ class SystemNgramRuntimeTest { SystemNgramRuntime.setEnabled(context, true) assertSame(loaded, SystemNgramRuntime.current()) } + + @Test + fun loadsVersion3AndVersion4AssetsAsOneDictionary() { + SystemNgramRuntime.initialize(context, true) + + val loaded = SystemNgramRuntime.loadedDictionary() + assertEquals(2_170, loaded.ruleCount) + assertEquals(56_984 + 15_191, loaded.storageBytes) + assertTrue(loaded.matchesSingleNode(node("カワボ"))) + } + + private fun node(word: String) = Node( + l = 1.toShort(), + r = 1.toShort(), + score = 0, + f = 0, + tango = word, + len = 1.toShort(), + yomiUsed = word, + sPos = 0, + ) } diff --git a/app/src/test/resources/ngram/system_ngram_unigram_v4_test.dat b/app/src/test/resources/ngram/system_ngram_unigram_v4_test.dat new file mode 100644 index 00000000..5a52f597 Binary files /dev/null and b/app/src/test/resources/ngram/system_ngram_unigram_v4_test.dat differ