diff --git a/scripts/receive_therock/tests/therock_update_status_json_test.py b/scripts/receive_therock/tests/therock_update_status_json_test.py index 60c59961..831c29c2 100644 --- a/scripts/receive_therock/tests/therock_update_status_json_test.py +++ b/scripts/receive_therock/tests/therock_update_status_json_test.py @@ -1772,6 +1772,12 @@ def test_completed_fanout_build_refreshes_same_run_test_leaves() -> None: _job("Build | py 3.12 | torch release/2.10 / Test | gfx110X-all"), ], ) + # Deliberately mismatched vs. the linux leaf above: this proves the match + # is keyed on (run_id, run_attempt) alone, not platform. That's safe in + # practice -- GitHub's run_id is unique per repository across every + # workflow/platform, so a real linux and windows run can never collide on + # one -- but it's an isolation technique, not a model of real data; don't + # read it as "a windows run can update a linux leaf" in production. completed_build.classification.platform = "windows" tusj._merge_run_into_document( doc, completed_build, tusj._create_leaf(completed_build) @@ -1784,6 +1790,138 @@ def test_completed_fanout_build_refreshes_same_run_test_leaves() -> None: assert leaf.variants[0].status is Status.success +def test_fanout_projection_uses_variant_rollup_not_raw_run_conclusion() -> None: + # The build run's own top-level GitHub conclusion is not necessarily the + # worst-of its matrix cells (e.g. a cell whose nested test job failed does + # not always flip the run's own conclusion). A projected test leaf must + # take the worst-of its own variants, not the raw run conclusion. + doc = StatusDocument() + stale_test = _variant_run( + pipeline_type="pytorch", + pipeline_phase="test", + architectures=["gfx110X-all"], + run_id=902, + conclusion=None, + jobs=[ + _job("Build | py 3.12 | torch release/2.10 / Build"), + _job( + "Build | py 3.12 | torch release/2.10 / Test | gfx110X-all", + conclusion=None, + completed=None, + ), + ], + ) + doc.upsert_leaf( + "linux", "gfx110X-all", "pytorch", "test", tusj._create_leaf(stale_test) + ) + + # The run's own conclusion reports success even though its nested test + # job for this cell failed. + completed_build = _variant_run( + pipeline_type="pytorch", + pipeline_phase="build", + run_id=902, + conclusion="success", + jobs=[ + _job("Build | py 3.12 | torch release/2.10 / Build"), + _job( + "Build | py 3.12 | torch release/2.10 / Test | gfx110X-all", + conclusion="failure", + ), + ], + ) + completed_build.classification.platform = "windows" + tusj._merge_run_into_document( + doc, completed_build, tusj._create_leaf(completed_build) + ) + + leaf = doc.pipelines.pytorch.test["linux"]["gfx110X-all"] + assert leaf.status is Status.failure + assert leaf.variants is not None + assert leaf.variants[0].status is Status.failure + + +def test_fanout_projection_folds_raw_run_conclusion_into_rollup() -> None: + # The inverse of the case above: every reported cell looks clean, but the + # run itself was cancelled (e.g. a cell whose job never even started, so + # it never shows up in `variants` at all). The projected test leaf must + # still surface that cancellation rather than reporting the variants' + # all-success rollup verbatim. + doc = StatusDocument() + stale_test = _variant_run( + pipeline_type="pytorch", + pipeline_phase="test", + architectures=["gfx110X-all"], + run_id=903, + conclusion=None, + jobs=[ + _job( + "Build | py 3.12 | torch release/2.10 / Test | gfx110X-all", + conclusion=None, + completed=None, + ), + ], + ) + doc.upsert_leaf( + "linux", "gfx110X-all", "pytorch", "test", tusj._create_leaf(stale_test) + ) + + # Same platform as the stale leaf above: this test is about the + # cancellation-folding logic, not about the (run_id, run_attempt)-only + # matching (already covered by + # test_completed_fanout_build_refreshes_same_run_test_leaves), so it + # doesn't need a platform mismatch to make its point. + cancelled_build = _variant_run( + pipeline_type="pytorch", + pipeline_phase="build", + run_id=903, + conclusion="cancelled", + jobs=[ + _job( + "Build | py 3.12 | torch release/2.10 / Test | gfx110X-all", + conclusion="success", + ), + ], + ) + tusj._merge_run_into_document( + doc, cancelled_build, tusj._create_leaf(cancelled_build) + ) + + leaf = doc.pipelines.pytorch.test["linux"]["gfx110X-all"] + assert leaf.status is Status.cancelled + + +def test_variant_job_name_matches_uppercase_ancestor_segment() -> None: + # A calling orchestrator (e.g. rockrel) can wrap TheRock's own build job + # in a differently-cased ancestor segment, e.g. + # "Release | py 3.12 | JAX 0.11.0 / Build | py 3.12 | jax rocm-jaxlib-v0.11.0" + # The build job's own tail (lowercase, full ref) must still win over that + # ancestor. A nested test sub-job has no (py, ref) segment of its own and + # must fall back to the uppercase ancestor instead of being dropped. + run = _variant_run( + pipeline_type="jax", + pipeline_phase="build", + jobs=[ + _job( + "build_jax_wheels / Release | py 3.12 | JAX 0.11.0 / " + "Build | py 3.12 | jax rocm-jaxlib-v0.11.0" + ), + _job( + "build_jax_wheels / Release | py 3.13 | JAX 0.11.0 / " + "Test | gfx94X-dcgpu | linux-gfx942-1gpu-ccs-csp-ossci-rocm / " + "Test JAX | gfx94X-dcgpu", + conclusion="cancelled", + ), + ], + ) + variants = tusj._derive_variants(run) + by_py = {v.matrix["py"]: v for v in variants} + assert by_py["3.12"].matrix["jax_ref"] == "rocm-jaxlib-v0.11.0" + assert by_py["3.12"].status is Status.success + assert by_py["3.13"].matrix["jax_ref"] == "0.11.0" + assert by_py["3.13"].status is Status.cancelled + + def test_skip_workflow_names_are_all_disregarded(tmp_path: Path) -> None: # Guards the generic `_SKIP_WORKFLOW_NAMES` mechanism itself, not just the # one workflow it was introduced for: whatever is in the set must be diff --git a/scripts/receive_therock/therock_update_status_json.py b/scripts/receive_therock/therock_update_status_json.py index 2ea1edae..cc74af91 100644 --- a/scripts/receive_therock/therock_update_status_json.py +++ b/scripts/receive_therock/therock_update_status_json.py @@ -326,8 +326,14 @@ def _update_document_metadata( # Matrix-cell job name for fan-out builds, e.g. TheRock's # "Build | py 3.12 | torch release/2.10" (pytorch) # "Build | py 3.12 | jax rocm-jaxlib-v0.9" (jax) -# `.search` (not fullmatch) so a reusable-workflow prefix/suffix still matches. -_MATRIX_JOB_RE = re.compile(r"py\s+(?P\S+)\s*\|\s*(?:torch|jax)\s+(?P\S+)") +# Case-insensitive: a calling orchestrator's own composite job name can wrap +# this in a differently-cased ancestor segment, e.g. rockrel's +# "Release | py 3.12 | JAX 0.11.0 / Build | py 3.12 | jax rocm-jaxlib-v0.11.0" +# -- the nested Test sub-job under that same cell has no (py, ref) of its own +# and relies entirely on that ancestor segment to be recognized. +_MATRIX_JOB_RE = re.compile( + r"py\s+(?P\S+)\s*\|\s*(?:torch|jax)\s+(?P\S+)", re.IGNORECASE +) # pipeline_type -> the matrix axis key used in the variant (reference schema: # pytorch cells key the ref as "torch", jax cells as "jax_ref"). @@ -382,9 +388,17 @@ def _variants_from_jobs( cells: dict[tuple[str, str], list[WorkflowJobRecord]] = {} order: list[tuple[str, str]] = [] for j in jobs: - match = _MATRIX_JOB_RE.search(j.name) - if not match: + # Take the *last* match, not the first: a nested job's composite name + # is "ancestor segment(s) / ... / own segment", and the own segment + # (closest to the actual job) is the authoritative (py, ref) -- e.g. + # a build job's own tail carries the full ref, while an orchestrator + # ancestor segment upstream of it may carry a shorter/looser one. A + # job with no segment of its own (a nested test sub-job) falls back + # to whichever ancestor segment matched. + matches = list(_MATRIX_JOB_RE.finditer(j.name)) + if not matches: continue + match = matches[-1] key = (match.group("py"), match.group("ref")) if key not in cells: cells[key] = [] @@ -492,11 +506,30 @@ def _refresh_same_run_fanout_tests( ) -> bool: """Refresh same-run test leaves from a completed fan-out workflow snapshot. - Delegated PyTorch/JAX release workflows report the shared entry run id. - Early notifications can project the run's job list into per-arch test - leaves while some matrix cells are still in progress; the final top-level - completion is classified as the build phase, so it would otherwise leave - those same-run test leaves stale. + PyTorch/JAX test coverage (`test_pytorch_wheels.yml` / `test_linux_jax_wheels.yml`) + is invoked as a reusable `workflow_call` nested inside the delegated release + workflow -- not dispatched as its own top-level run -- so its jobs land in + the *same* run id, job list, and webhook notifications as the entry build. + There is no job-name-level split between "build" and "test" jobs: the + registry classifies the whole run as `pipeline_type`/`pipeline_phase="build"` + (see `WORKFLOW_SPECS`), and `_variants_from_jobs` already groups every job + sharing a (py, ref) cell -- build and nested test alike -- into one + `Variant` per cell (see `test_reusable_matrix_nested_jobs_collapse_to_one_variant_per_cell`). + Early notifications can project that job-list snapshot into per-arch test + leaves (keyed by the same run id) while some cells are still in progress; + the final notification is still classified as the build phase, so without + this function those same-run test leaves would go stale once the build + itself is done. + + `leaf.status` is this run's own top-level GitHub conclusion. It is not + necessarily the worst-of its `variants` (e.g. a matrix cell whose nested + test job failed/cancelled does not always flip the run's own conclusion, + or a cell can be entirely missing from `variants` if its job never + started). Fold it into the rollup rather than only using it as an + empty-variants fallback, so a terminal failure/cancellation at the run + level cannot be masked by whatever the individual cells happened to report + -- mirroring what `_merge_matrix_build_leaf` does for the build leaf + itself. """ cls = workflow_run.classification if ( @@ -508,6 +541,9 @@ def _refresh_same_run_fanout_tests( ): return False + projected_status = rollup_statuses( + (*(v.status for v in leaf.variants), leaf.status), leaf.status + ) pipeline = getattr(doc.pipelines, cls.pipeline_type) wrote = False for phase_map in (pipeline.test, pipeline.test_full): @@ -519,7 +555,7 @@ def _refresh_same_run_fanout_tests( continue if not existing.should_replace(leaf): continue - existing.status = leaf.status + existing.status = projected_status existing.completed_at = leaf.completed_at existing.variants = leaf.variants wrote = True