diff --git a/.github/workflows/ground-truth.yml b/.github/workflows/ground-truth.yml new file mode 100644 index 000000000..a75300c78 --- /dev/null +++ b/.github/workflows/ground-truth.yml @@ -0,0 +1,94 @@ +name: Ground Truth Validation + +on: + push: + branches: [ main, develop ] + paths: + - "skainet-test/skainet-test-groundtruth/**" + - ".github/workflows/ground-truth.yml" + pull_request: + branches: [ main, develop ] + paths: + - "skainet-test/skainet-test-groundtruth/**" + - ".github/workflows/ground-truth.yml" + # skainet-ground-truth's PyTorch fixtures can change independently of any commit here — + # a weekly run catches that drift even when nothing in this repo touched the module. + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# Set default permission for all jobs to none +permissions: {} + +jobs: + ground-truth: + runs-on: ubuntu-latest + timeout-minutes: 30 + + permissions: + contents: read + + env: + # actions/checkout's `path` can't escape the primary repo's own workspace with `..` + # (hard-rejected: "Repository path ... is not under ...") — so this lands as a plain + # subdirectory of SKaiNET's checkout instead of a true sibling, and every Gradle + # invocation below points at it explicitly via -PgroundTruthSourceDir (the override + # skainet-test-groundtruth/build.gradle.kts's groundTruthProjectDir already supports — + # local dev still uses the sibling-checkout default, only CI needs this). + GROUND_TRUTH_SOURCE_DIR: ${{ github.workspace }}/skainet-ground-truth/pytorch + + steps: + - name: Checkout SKaiNET + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Checkout skainet-ground-truth + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: SKaiNET-developers/skainet-ground-truth + path: skainet-ground-truth + + - name: Copy CI gradle.properties + run: mkdir -p ~/.gradle ; cp .github/ci-gradle.properties ~/.gradle/gradle.properties + + - name: Set up JDK 21 + uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961 # v5.7.0 + with: + distribution: 'zulu' + java-version: 21 + + - name: Build ground truth Docker image + run: | + ./gradlew --no-daemon --stacktrace --no-configuration-cache \ + -PgroundTruthSourceDir="$GROUND_TRUTH_SOURCE_DIR" \ + :skainet-test:skainet-test-groundtruth:buildGroundTruthDocker + + - name: Generate ground truth GGUF files + run: | + ./gradlew --no-daemon --stacktrace --no-configuration-cache \ + -PgroundTruthSourceDir="$GROUND_TRUTH_SOURCE_DIR" \ + :skainet-test:skainet-test-groundtruth:generateGroundTruth + + # -PrequireGroundTruth=true turns a missing/broken pipeline into a hard failure instead + # of the tests silently skipping (see GroundTruthConfig.requireAvailable) — the whole + # point of this workflow is proving the pipeline above actually produced something. + - name: Run validation tests + env: + GRADLE_OPTS: -Dorg.gradle.jvmargs="-Xmx4g -Dfile.encoding=UTF-8" + run: | + ./gradlew --no-daemon --stacktrace --no-configuration-cache \ + -PgroundTruthSourceDir="$GROUND_TRUTH_SOURCE_DIR" \ + :skainet-test:skainet-test-groundtruth:jvmTest -PrequireGroundTruth=true + + - name: Upload test reports + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ground-truth-reports + path: | + skainet-test/skainet-test-groundtruth/build/reports/tests/** + skainet-test/skainet-test-groundtruth/build/test-results/** + retention-days: 14 diff --git a/skainet-test/skainet-test-groundtruth/build.gradle.kts b/skainet-test/skainet-test-groundtruth/build.gradle.kts index 4c9e0e8c7..334268a37 100644 --- a/skainet-test/skainet-test-groundtruth/build.gradle.kts +++ b/skainet-test/skainet-test-groundtruth/build.gradle.kts @@ -9,8 +9,15 @@ plugins { // Ground Truth Generation via Docker // ============================================================================= -// Path to the skainet-ground-truth project (relative to SKaiNET root) -val groundTruthProjectDir = rootProject.projectDir.parentFile.resolve("skainet-ground-truth/pytorch") +// Path to the skainet-ground-truth project's `pytorch` subdirectory. Override with +// -PgroundTruthSourceDir=/path/to/pytorch — e.g. once fixture generation moves to a +// numcrux-hosted corpus (https://github.com/numcrux) instead of this SKaiNET-org sibling +// checkout. Defaults to today's convenience: a `skainet-ground-truth` checkout next to +// this repo (../skainet-ground-truth relative to SKaiNET's root), matching the layout +// GroundTruthConfig.findDefaultResultsDir() also assumes on the Kotlin test side. +val groundTruthProjectDir = (findProperty("groundTruthSourceDir") as String?) + ?.let(::File) + ?: rootProject.projectDir.parentFile.resolve("skainet-ground-truth/pytorch") val groundTruthResultsDir = groundTruthProjectDir.resolve("results") // Docker image name for ground truth generation @@ -97,6 +104,9 @@ tasks.register("listGroundTruth") { // Make ground truth results available as a system property for tests tasks.withType { systemProperty("groundtruth.results.dir", groundTruthResultsDir.absolutePath) + // -PrequireGroundTruth=true (CI) turns missing-ground-truth from a silent skip into a + // hard failure — see GroundTruthConfig.requireAvailable / TestAssumptions.kt. + systemProperty("groundtruth.require", (findProperty("requireGroundTruth") as String?) ?: "false") } kotlin { diff --git a/skainet-test/skainet-test-groundtruth/src/commonMain/kotlin/sk/ainet/test/groundtruth/GroundTruthTestCase.kt b/skainet-test/skainet-test-groundtruth/src/commonMain/kotlin/sk/ainet/test/groundtruth/GroundTruthTestCase.kt index c6a79a3b7..a9267e508 100644 --- a/skainet-test/skainet-test-groundtruth/src/commonMain/kotlin/sk/ainet/test/groundtruth/GroundTruthTestCase.kt +++ b/skainet-test/skainet-test-groundtruth/src/commonMain/kotlin/sk/ainet/test/groundtruth/GroundTruthTestCase.kt @@ -29,9 +29,55 @@ public data class GroundTruthTestCase( val useCase: String? = null, /** Path to the source GGUF file */ - val sourcePath: String? = null + val sourcePath: String? = null, + + /** + * Operation parameters as written by the Python side's `op.*` GGUF metadata keys + * (e.g. `op.padding` -> `"padding" to 1`), decoded to Int/Float/List/List + * per value. See [resolvedParams] to turn this into a typed [OperationParams]. + */ + val rawOpParams: Map = emptyMap() ) +/** + * Maps [GroundTruthTestCase.rawOpParams] onto [OperationParams]'s named, typed fields. + * A scalar value (`op.padding = 1`) becomes a symmetric pair for the height/width-style + * params, matching how [OperationParamsBuilder.padding] already turns a single Int into + * `(value, value)` — the Python side writes symmetric params as a plain scalar, not a + * 2-element array (see `skainet-ground-truth`'s `op_params={"padding": 1, ...}` usage). + */ +public fun GroundTruthTestCase.resolvedParams(): OperationParams { + fun pair(name: String): Pair? = when (val v = rawOpParams[name]) { + is Int -> v to v + is Float -> v.toInt() to v.toInt() + is List<*> -> when (v.size) { + 1 -> (v[0] as Number).toInt().let { it to it } + else -> (v.getOrNull(0) as? Number)?.toInt()?.let { h -> + (v.getOrNull(1) as? Number)?.toInt()?.let { w -> h to w } + } + } + else -> null + } + + fun int(name: String): Int? = (rawOpParams[name] as? Number)?.toInt() + ?: (rawOpParams[name] as? List<*>)?.firstOrNull().let { (it as? Number)?.toInt() } + + fun float(name: String): Float? = (rawOpParams[name] as? Number)?.toFloat() + + return OperationParams( + stride = pair("stride"), + padding = pair("padding"), + dilation = pair("dilation"), + groups = int("groups"), + kernelSize = pair("kernel_size") ?: pair("kernelSize"), + dim = int("dim"), + startDim = int("start_dim") ?: int("startDim"), + endDim = int("end_dim") ?: int("endDim"), + negativeSlope = float("negative_slope") ?: float("negativeSlope"), + alpha = float("alpha") + ) +} + /** * Represents a tensor loaded from ground truth GGUF file. * Stores the raw float data and shape information. diff --git a/skainet-test/skainet-test-groundtruth/src/commonMain/kotlin/sk/ainet/test/groundtruth/GroundTruthValidator.kt b/skainet-test/skainet-test-groundtruth/src/commonMain/kotlin/sk/ainet/test/groundtruth/GroundTruthValidator.kt index 6d3bb31ac..914c72e5f 100644 --- a/skainet-test/skainet-test-groundtruth/src/commonMain/kotlin/sk/ainet/test/groundtruth/GroundTruthValidator.kt +++ b/skainet-test/skainet-test-groundtruth/src/commonMain/kotlin/sk/ainet/test/groundtruth/GroundTruthValidator.kt @@ -106,14 +106,16 @@ public class GroundTruthValidator( * Validate a ground truth test case. * * @param testCase The test case to validate - * @param params Operation parameters (stride, padding, etc.) + * @param params Operation parameters (stride, padding, etc.) — defaults to + * [testCase]'s own `op.*` GGUF metadata ([GroundTruthTestCase.resolvedParams]), + * not an empty [OperationParams]; pass an explicit value to override. * @param tolerance Absolute tolerance for comparison (auto-selected if null) * @param rtol Relative tolerance for comparison * @param validateGradients Whether to also validate gradient computation */ public fun validate( testCase: GroundTruthTestCase, - params: OperationParams = OperationParams(), + params: OperationParams = testCase.resolvedParams(), tolerance: Float? = null, rtol: Float = 1e-5f, validateGradients: Boolean = false @@ -180,7 +182,7 @@ public class GroundTruthValidator( */ public fun validateAll( testCases: List, - paramsProvider: (GroundTruthTestCase) -> OperationParams = { OperationParams() }, + paramsProvider: (GroundTruthTestCase) -> OperationParams = { it.resolvedParams() }, tolerance: Float? = null, rtol: Float = 1e-5f ): List { @@ -194,7 +196,7 @@ public class GroundTruthValidator( */ public fun assertValid( testCase: GroundTruthTestCase, - params: OperationParams = OperationParams(), + params: OperationParams = testCase.resolvedParams(), tolerance: Float? = null, rtol: Float = 1e-5f ) { @@ -275,7 +277,7 @@ public class GroundTruthValidator( */ public fun GroundTruthTestCase.validateWith( ops: TensorOps, - params: OperationParams = OperationParams(), + params: OperationParams = resolvedParams(), tolerance: Float? = null ): GroundTruthValidator.ValidationResult { return GroundTruthValidator(ops).validate(this, params, tolerance) @@ -286,7 +288,7 @@ public fun GroundTruthTestCase.validateWith( */ public fun GroundTruthTestCase.assertValidWith( ops: TensorOps, - params: OperationParams = OperationParams(), + params: OperationParams = resolvedParams(), tolerance: Float? = null ) { GroundTruthValidator(ops).assertValid(this, params, tolerance) diff --git a/skainet-test/skainet-test-groundtruth/src/jvmMain/kotlin/sk/ainet/test/groundtruth/GroundTruthConfig.kt b/skainet-test/skainet-test-groundtruth/src/jvmMain/kotlin/sk/ainet/test/groundtruth/GroundTruthConfig.kt index d12998c3d..129d08351 100644 --- a/skainet-test/skainet-test-groundtruth/src/jvmMain/kotlin/sk/ainet/test/groundtruth/GroundTruthConfig.kt +++ b/skainet-test/skainet-test-groundtruth/src/jvmMain/kotlin/sk/ainet/test/groundtruth/GroundTruthConfig.kt @@ -33,6 +33,16 @@ public object GroundTruthConfig { false } + /** + * When true, missing ground truth is a hard test failure instead of a skip — set via + * `-PrequireGroundTruth=true` (see build.gradle.kts), used in CI so a broken/missing + * pipeline shows up red instead of silently skipping every ground-truth test. + * Local dev runs default to false: skip gracefully when the sibling + * `../skainet-ground-truth` checkout or generated GGUF files aren't present. + */ + public val requireAvailable: Boolean + get() = System.getProperty("groundtruth.require")?.toBoolean() ?: false + /** * Get all available test suites. */ diff --git a/skainet-test/skainet-test-groundtruth/src/jvmMain/kotlin/sk/ainet/test/groundtruth/GroundTruthLoader.kt b/skainet-test/skainet-test-groundtruth/src/jvmMain/kotlin/sk/ainet/test/groundtruth/GroundTruthLoader.kt index f28268ab9..1e38ae829 100644 --- a/skainet-test/skainet-test-groundtruth/src/jvmMain/kotlin/sk/ainet/test/groundtruth/GroundTruthLoader.kt +++ b/skainet-test/skainet-test-groundtruth/src/jvmMain/kotlin/sk/ainet/test/groundtruth/GroundTruthLoader.kt @@ -5,6 +5,7 @@ import kotlinx.io.buffered import sk.ainet.io.gguf.GGMLQuantizationType import sk.ainet.io.gguf.GGUFReader import sk.ainet.io.gguf.GGUFValueType +import sk.ainet.io.gguf.ReaderField import sk.ainet.io.gguf.ReaderTensor import sk.ainet.lang.tensor.Shape import java.io.File @@ -94,10 +95,49 @@ public object GroundTruthLoader { expectedGradients = gradients.ifEmpty { null }, testSuite = testSuite, useCase = useCase, - sourcePath = sourcePath + sourcePath = sourcePath, + rawOpParams = extractOpParams(reader) ) } + /** + * Extract every `op.*` GGUF metadata field into a name -> value map (name with the + * `op.` prefix stripped). Written by `store_experiment_as_gguf`'s `op_params` handling + * (`gt/pytorch/io/writer.py`): `add_int32`/`add_uint32` for a scalar int, `add_float32` + * for a scalar float, `add_array` for a list/tuple of ints (e.g. an asymmetric + * `(stride_h, stride_w)`) — see [decodeFieldValue] for the corresponding GGUF-side shape. + */ + private fun extractOpParams(reader: GGUFReader): Map { + val result = mutableMapOf() + for ((name, field) in reader.fields) { + if (!name.startsWith("op.")) continue + decodeFieldValue(field)?.let { result[name.removePrefix("op.")] = it } + } + return result + } + + /** + * Decode a [sk.ainet.io.gguf.ReaderField] into a plain Int/Float/List/List, + * for the scalar-or-array-of-numbers shape `op.*` fields always use. `field.data` holds + * indices into `field.parts` for the field's actual value(s) — one index for a scalar, + * N indices for an N-element array (see GGUFReader.getFieldParts's ARRAY branch); each + * `field.parts[idx]` is itself a single-element list holding the raw decoded number. + */ + private fun decodeFieldValue(field: ReaderField): Any? { + val raw = field.data.mapNotNull { idx -> field.parts.getOrNull(idx)?.firstOrNull() } + if (raw.isEmpty()) return null + val numbers = raw.map { toNumber(it) ?: return null } + val isArray = field.types.firstOrNull() == GGUFValueType.ARRAY + return if (isArray) numbers else numbers.first() + } + + private fun toNumber(value: Any): Number? = when (value) { + is UInt -> value.toInt() + is ULong -> value.toLong() + is Number -> value + else -> null + } + /** * Loads all test cases from a directory recursively. * Finds all .gguf files and loads them as test cases. @@ -225,35 +265,41 @@ public operator fun List.get(testSuite: String, useCase: St /** * JVM-specific validator extensions with file system support. + * + * `params: OperationParams? = null` (not a bare [OperationParams] default) throughout this + * file so "not specified" can fall through to each test case's own `op.*`-derived + * [GroundTruthTestCase.resolvedParams] instead of silently overriding it with an empty one. */ public fun GroundTruthValidator.validate( ggufPath: String, - params: OperationParams = OperationParams(), + params: OperationParams? = null, tolerance: Float? = null, rtol: Float = 1e-5f, validateGradients: Boolean = false ): GroundTruthValidator.ValidationResult { - return validate(GroundTruthLoader.load(ggufPath), params, tolerance, rtol, validateGradients) + val testCase = GroundTruthLoader.load(ggufPath) + return validate(testCase, params ?: testCase.resolvedParams(), tolerance, rtol, validateGradients) } public fun GroundTruthValidator.validate( file: File, - params: OperationParams = OperationParams(), + params: OperationParams? = null, tolerance: Float? = null, rtol: Float = 1e-5f, validateGradients: Boolean = false ): GroundTruthValidator.ValidationResult { - return validate(GroundTruthLoader.load(file), params, tolerance, rtol, validateGradients) + val testCase = GroundTruthLoader.load(file) + return validate(testCase, params ?: testCase.resolvedParams(), tolerance, rtol, validateGradients) } public fun GroundTruthValidator.validateDirectory( directory: File, - params: OperationParams = OperationParams(), + params: OperationParams? = null, tolerance: Float? = null, rtol: Float = 1e-5f ): List { val testCases = GroundTruthLoader.loadFromDirectory(directory) - return testCases.map { validate(it, params, tolerance, rtol) } + return testCases.map { validate(it, params ?: it.resolvedParams(), tolerance, rtol) } } public fun GroundTruthValidator.validateTestSuite( @@ -268,7 +314,7 @@ public fun GroundTruthValidator.validateTestSuite( public fun GroundTruthValidator.assertValid( ggufPath: String, - params: OperationParams = OperationParams(), + params: OperationParams? = null, tolerance: Float? = null, rtol: Float = 1e-5f ) { diff --git a/skainet-test/skainet-test-groundtruth/src/jvmTest/kotlin/sk/ainet/test/groundtruth/GroundTruthIntegrationTest.kt b/skainet-test/skainet-test-groundtruth/src/jvmTest/kotlin/sk/ainet/test/groundtruth/GroundTruthIntegrationTest.kt index 7f5a08d29..590831216 100644 --- a/skainet-test/skainet-test-groundtruth/src/jvmTest/kotlin/sk/ainet/test/groundtruth/GroundTruthIntegrationTest.kt +++ b/skainet-test/skainet-test-groundtruth/src/jvmTest/kotlin/sk/ainet/test/groundtruth/GroundTruthIntegrationTest.kt @@ -38,9 +38,9 @@ class GroundTruthIntegrationTest { val results = testCases.map { testCase -> println(" Validating: ${testCase.description}") - // Infer parameters from test case description - val params = inferConv2dParams(testCase) - validator.validate(testCase, params) + // Params come from the GGUF's own op.* metadata (GroundTruthTestCase.resolvedParams, + // the default validator.validate already uses) — no per-test-case guessing needed. + validator.validate(testCase) } // Print summary @@ -120,8 +120,7 @@ class GroundTruthIntegrationTest { println(" Found ${testCases.size} test cases") val results = testCases.map { testCase -> - val params = inferOperationParams(testCase) - validator.validate(testCase, params, tolerance = ToleranceConfig.RELAXED) + validator.validate(testCase, tolerance = ToleranceConfig.RELAXED) } allResults.addAll(results) @@ -145,48 +144,4 @@ class GroundTruthIntegrationTest { } } - // ========================================================================= - // Helper Functions - // ========================================================================= - - private fun inferConv2dParams(testCase: GroundTruthTestCase): OperationParams { - val desc = testCase.description.lowercase() - return operationParams { - when { - desc.contains("stride") && desc.contains("2") -> stride(2) - desc.contains("strided") -> stride(2) - } - when { - desc.contains("padding") && desc.contains("1") -> padding(1) - desc.contains("padded") -> padding(1) - } - when { - desc.contains("dilation") && desc.contains("2") -> dilation(2) - desc.contains("dilated") -> dilation(2) - } - when { - desc.contains("depthwise") -> groups(3) // Assume 3 channels - desc.contains("grouped") -> groups(2) - } - } - } - - private fun inferOperationParams(testCase: GroundTruthTestCase): OperationParams { - val opName = testCase.operationName.lowercase() - val desc = testCase.description.lowercase() - - return when { - opName.contains("conv") -> inferConv2dParams(testCase) - opName.contains("flatten") -> operationParams { - // Default flatten params - startDim(1) - endDim(-1) - } - opName.contains("pool") -> operationParams { - kernelSize(2) - stride(2) - } - else -> OperationParams() - } - } } diff --git a/skainet-test/skainet-test-groundtruth/src/jvmTest/kotlin/sk/ainet/test/groundtruth/TestAssumptions.kt b/skainet-test/skainet-test-groundtruth/src/jvmTest/kotlin/sk/ainet/test/groundtruth/TestAssumptions.kt index a1f04227f..61114bbbf 100644 --- a/skainet-test/skainet-test-groundtruth/src/jvmTest/kotlin/sk/ainet/test/groundtruth/TestAssumptions.kt +++ b/skainet-test/skainet-test-groundtruth/src/jvmTest/kotlin/sk/ainet/test/groundtruth/TestAssumptions.kt @@ -4,13 +4,25 @@ import org.junit.Assume /** * JUnit assumption helpers for ground truth tests. - * These allow tests to be skipped when ground truth is not available. + * + * By default these skip the test when ground truth isn't available — a normal state for + * local dev, since generating it needs Docker + a sibling `../skainet-ground-truth` + * checkout most contributors won't have. With [GroundTruthConfig.requireAvailable] set + * (`-PrequireGroundTruth=true`, used in CI), the same conditions fail the test instead — + * so a broken or unwired pipeline shows up red, not as an invisible skip. */ /** - * Skip test if ground truth is not available. + * Skip (or, if required, fail) the test if ground truth is not available. */ fun assumeGroundTruthAvailable() { + if (GroundTruthConfig.requireAvailable) { + check(GroundTruthConfig.isAvailable) { + "Ground truth required (-PrequireGroundTruth=true) but not available. " + + "Run './gradlew buildGroundTruthDocker generateGroundTruth' first." + } + return + } Assume.assumeTrue( "Ground truth not available. Run './gradlew generateGroundTruth' first.", GroundTruthConfig.isAvailable @@ -18,10 +30,17 @@ fun assumeGroundTruthAvailable() { } /** - * Skip test if a specific test suite is not available. + * Skip (or, if required, fail) the test if a specific test suite is not available. */ fun assumeTestSuiteAvailable(testSuite: String) { assumeGroundTruthAvailable() + if (GroundTruthConfig.requireAvailable) { + check(GroundTruthConfig.testSuiteDir(testSuite).exists()) { + "Test suite '$testSuite' required (-PrequireGroundTruth=true) but not available " + + "in ground truth results." + } + return + } Assume.assumeTrue( "Test suite '$testSuite' not available in ground truth results.", GroundTruthConfig.testSuiteDir(testSuite).exists()