fix: Quartz mis-reports pytorch/jax matrix-job test results

scripts/receive_therock/tests/therock_update_status_json_test.py

@@ -1772,6 +1772,12 @@ def test_completed_fanout_build_refreshes_same_run_test_leaves() -> None:
             _job("Build | py 3.12 | torch release/2.10 / Test | gfx110X-all"),
         ],
     )
+    # Deliberately mismatched vs. the linux leaf above: this proves the match
+    # is keyed on (run_id, run_attempt) alone, not platform. That's safe in
+    # practice -- GitHub's run_id is unique per repository across every
+    # workflow/platform, so a real linux and windows run can never collide on
+    # one -- but it's an isolation technique, not a model of real data; don't
+    # read it as "a windows run can update a linux leaf" in production.
     completed_build.classification.platform = "windows"
     tusj._merge_run_into_document(
         doc, completed_build, tusj._create_leaf(completed_build)
@@ -1784,6 +1790,138 @@ def test_completed_fanout_build_refreshes_same_run_test_leaves() -> None:
     assert leaf.variants[0].status is Status.success
 
 
+def test_fanout_projection_uses_variant_rollup_not_raw_run_conclusion() -> None:
+    # The build run's own top-level GitHub conclusion is not necessarily the
+    # worst-of its matrix cells (e.g. a cell whose nested test job failed does
+    # not always flip the run's own conclusion). A projected test leaf must
+    # take the worst-of its own variants, not the raw run conclusion.
+    doc = StatusDocument()
+    stale_test = _variant_run(
+        pipeline_type="pytorch",
+        pipeline_phase="test",
+        architectures=["gfx110X-all"],
+        run_id=902,
+        conclusion=None,
+        jobs=[
+            _job("Build | py 3.12 | torch release/2.10 / Build"),
+            _job(
+                "Build | py 3.12 | torch release/2.10 / Test | gfx110X-all",
+                conclusion=None,
+                completed=None,
+            ),
+        ],
+    )
+    doc.upsert_leaf(
+        "linux", "gfx110X-all", "pytorch", "test", tusj._create_leaf(stale_test)
+    )
+
+    # The run's own conclusion reports success even though its nested test
+    # job for this cell failed.
+    completed_build = _variant_run(
+        pipeline_type="pytorch",
+        pipeline_phase="build",
+        run_id=902,
+        conclusion="success",
+        jobs=[
+            _job("Build | py 3.12 | torch release/2.10 / Build"),
+            _job(
+                "Build | py 3.12 | torch release/2.10 / Test | gfx110X-all",
+                conclusion="failure",
+            ),
+        ],
+    )
+    completed_build.classification.platform = "windows"
+    tusj._merge_run_into_document(
+        doc, completed_build, tusj._create_leaf(completed_build)
+    )
+
+    leaf = doc.pipelines.pytorch.test["linux"]["gfx110X-all"]
+    assert leaf.status is Status.failure
+    assert leaf.variants is not None
+    assert leaf.variants[0].status is Status.failure
+
+
+def test_fanout_projection_folds_raw_run_conclusion_into_rollup() -> None:
+    # The inverse of the case above: every reported cell looks clean, but the
+    # run itself was cancelled (e.g. a cell whose job never even started, so
+    # it never shows up in `variants` at all). The projected test leaf must
+    # still surface that cancellation rather than reporting the variants'
+    # all-success rollup verbatim.
+    doc = StatusDocument()
+    stale_test = _variant_run(
+        pipeline_type="pytorch",
+        pipeline_phase="test",
+        architectures=["gfx110X-all"],
+        run_id=903,
+        conclusion=None,
+        jobs=[
+            _job(
+                "Build | py 3.12 | torch release/2.10 / Test | gfx110X-all",
+                conclusion=None,
+                completed=None,
+            ),
+        ],
+    )
+    doc.upsert_leaf(
+        "linux", "gfx110X-all", "pytorch", "test", tusj._create_leaf(stale_test)
+    )
+
+    # Same platform as the stale leaf above: this test is about the
+    # cancellation-folding logic, not about the (run_id, run_attempt)-only
+    # matching (already covered by
+    # test_completed_fanout_build_refreshes_same_run_test_leaves), so it
+    # doesn't need a platform mismatch to make its point.
+    cancelled_build = _variant_run(
+        pipeline_type="pytorch",
+        pipeline_phase="build",
+        run_id=903,
+        conclusion="cancelled",
+        jobs=[
+            _job(
+                "Build | py 3.12 | torch release/2.10 / Test | gfx110X-all",
+                conclusion="success",
+            ),
+        ],
+    )
+    tusj._merge_run_into_document(
+        doc, cancelled_build, tusj._create_leaf(cancelled_build)
+    )
+
+    leaf = doc.pipelines.pytorch.test["linux"]["gfx110X-all"]
+    assert leaf.status is Status.cancelled
+
+
+def test_variant_job_name_matches_uppercase_ancestor_segment() -> None:
+    # A calling orchestrator (e.g. rockrel) can wrap TheRock's own build job
+    # in a differently-cased ancestor segment, e.g.
+    #   "Release | py 3.12 | JAX 0.11.0 / Build | py 3.12 | jax rocm-jaxlib-v0.11.0"
+    # The build job's own tail (lowercase, full ref) must still win over that
+    # ancestor. A nested test sub-job has no (py, ref) segment of its own and
+    # must fall back to the uppercase ancestor instead of being dropped.
+    run = _variant_run(
+        pipeline_type="jax",
+        pipeline_phase="build",
+        jobs=[
+            _job(
+                "build_jax_wheels / Release | py 3.12 | JAX 0.11.0 / "
+                "Build | py 3.12 | jax rocm-jaxlib-v0.11.0"
+            ),
+            _job(
+                "build_jax_wheels / Release | py 3.13 | JAX 0.11.0 / "
+                "Test | gfx94X-dcgpu | linux-gfx942-1gpu-ccs-csp-ossci-rocm / "
+                "Test JAX | gfx94X-dcgpu",
+                conclusion="cancelled",
+            ),
+        ],
+    )
+    variants = tusj._derive_variants(run)
+    by_py = {v.matrix["py"]: v for v in variants}
+    assert by_py["3.12"].matrix["jax_ref"] == "rocm-jaxlib-v0.11.0"
+    assert by_py["3.12"].status is Status.success
+    assert by_py["3.13"].matrix["jax_ref"] == "0.11.0"
+    assert by_py["3.13"].status is Status.cancelled
+
+
 def test_skip_workflow_names_are_all_disregarded(tmp_path: Path) -> None:
     # Guards the generic `_SKIP_WORKFLOW_NAMES` mechanism itself, not just the
     # one workflow it was introduced for: whatever is in the set must be

scripts/receive_therock/therock_update_status_json.py

@@ -326,8 +326,14 @@ def _update_document_metadata(
 # Matrix-cell job name for fan-out builds, e.g. TheRock's
 #   "Build | py 3.12 | torch release/2.10"   (pytorch)
 #   "Build | py 3.12 | jax rocm-jaxlib-v0.9" (jax)
-# `.search` (not fullmatch) so a reusable-workflow prefix/suffix still matches.
-_MATRIX_JOB_RE = re.compile(r"py\s+(?P<py>\S+)\s*\|\s*(?:torch|jax)\s+(?P<ref>\S+)")
+# Case-insensitive: a calling orchestrator's own composite job name can wrap
+# this in a differently-cased ancestor segment, e.g. rockrel's
+# "Release | py 3.12 | JAX 0.11.0 / Build | py 3.12 | jax rocm-jaxlib-v0.11.0"
+# -- the nested Test sub-job under that same cell has no (py, ref) of its own
+# and relies entirely on that ancestor segment to be recognized.
+_MATRIX_JOB_RE = re.compile(
+    r"py\s+(?P<py>\S+)\s*\|\s*(?:torch|jax)\s+(?P<ref>\S+)", re.IGNORECASE
+)
 
 # pipeline_type -> the matrix axis key used in the variant (reference schema:
 # pytorch cells key the ref as "torch", jax cells as "jax_ref").
@@ -382,9 +388,17 @@ def _variants_from_jobs(
     cells: dict[tuple[str, str], list[WorkflowJobRecord]] = {}
     order: list[tuple[str, str]] = []
     for j in jobs:
-        match = _MATRIX_JOB_RE.search(j.name)
-        if not match:
+        # Take the *last* match, not the first: a nested job's composite name
+        # is "ancestor segment(s) / ... / own segment", and the own segment
+        # (closest to the actual job) is the authoritative (py, ref) -- e.g.
+        # a build job's own tail carries the full ref, while an orchestrator
+        # ancestor segment upstream of it may carry a shorter/looser one. A
+        # job with no segment of its own (a nested test sub-job) falls back
+        # to whichever ancestor segment matched.
+        matches = list(_MATRIX_JOB_RE.finditer(j.name))
+        if not matches:
             continue
+        match = matches[-1]
         key = (match.group("py"), match.group("ref"))
         if key not in cells:
             cells[key] = []
@@ -492,11 +506,30 @@ def _refresh_same_run_fanout_tests(
 ) -> bool:
     """Refresh same-run test leaves from a completed fan-out workflow snapshot.
 
-    Delegated PyTorch/JAX release workflows report the shared entry run id.
-    Early notifications can project the run's job list into per-arch test
-    leaves while some matrix cells are still in progress; the final top-level
-    completion is classified as the build phase, so it would otherwise leave
-    those same-run test leaves stale.
+    PyTorch/JAX test coverage (`test_pytorch_wheels.yml` / `test_linux_jax_wheels.yml`)
+    is invoked as a reusable `workflow_call` nested inside the delegated release
+    workflow -- not dispatched as its own top-level run -- so its jobs land in
+    the *same* run id, job list, and webhook notifications as the entry build.
+    There is no job-name-level split between "build" and "test" jobs: the
+    registry classifies the whole run as `pipeline_type`/`pipeline_phase="build"`
+    (see `WORKFLOW_SPECS`), and `_variants_from_jobs` already groups every job
+    sharing a (py, ref) cell -- build and nested test alike -- into one
+    `Variant` per cell (see `test_reusable_matrix_nested_jobs_collapse_to_one_variant_per_cell`).
+    Early notifications can project that job-list snapshot into per-arch test
+    leaves (keyed by the same run id) while some cells are still in progress;
+    the final notification is still classified as the build phase, so without
+    this function those same-run test leaves would go stale once the build
+    itself is done.
+
+    `leaf.status` is this run's own top-level GitHub conclusion. It is not
+    necessarily the worst-of its `variants` (e.g. a matrix cell whose nested
+    test job failed/cancelled does not always flip the run's own conclusion,
+    or a cell can be entirely missing from `variants` if its job never
+    started). Fold it into the rollup rather than only using it as an
+    empty-variants fallback, so a terminal failure/cancellation at the run
+    level cannot be masked by whatever the individual cells happened to report
+    -- mirroring what `_merge_matrix_build_leaf` does for the build leaf
+    itself.
     """
     cls = workflow_run.classification
     if (
@@ -508,6 +541,9 @@ def _refresh_same_run_fanout_tests(
     ):
         return False
 
+    projected_status = rollup_statuses(
+        (*(v.status for v in leaf.variants), leaf.status), leaf.status
+    )
     pipeline = getattr(doc.pipelines, cls.pipeline_type)
     wrote = False
     for phase_map in (pipeline.test, pipeline.test_full):
@@ -519,7 +555,7 @@ def _refresh_same_run_fanout_tests(
                     continue
                 if not existing.should_replace(leaf):
                     continue
-                existing.status = leaf.status
+                existing.status = projected_status
                 existing.completed_at = leaf.completed_at
                 existing.variants = leaf.variants
                 wrote = True