From 8c30d52506ececd3e2361976d8e175e439ae14f6 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Sat, 19 Sep 2026 12:21:26 -0700 Subject: [PATCH 1/2] Export coverage as Cobertura XML GitHub Code Quality stores coverage itself now -- a bot comment on the pull request with aggregate and per-file line coverage against the default branch, and a `Restrict code coverage` ruleset that can gate merges on it. It takes Cobertura XML and nothing else, and this package could only write LCOV. `write_cobertura(io_or_path, result; root)` is the sibling of `write_lcov`: same `root` relativization through `_report_path`, same `/` separators, same "returns `false` and writes nothing when the run collected no coverage" contract, so a caller can drive both from one `if`. Cobertura describes branch and method coverage too. Julia's line-count instrumentation produces neither and the consumer this exists for reads line coverage only, so those attributes are present -- the format's consumers expect them -- and always zero. A line Julia could not instrument is left out rather than reported as a line nobody ran: Cobertura has no "not applicable" hit count. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 1 + Project.toml | 2 +- src/TestItemControllers.jl | 3 +- src/cobertura.jl | 168 +++++++++++++++++++++++++++++++++++++ test/test_cobertura.jl | 135 +++++++++++++++++++++++++++++ 5 files changed, 307 insertions(+), 2 deletions(-) create mode 100644 src/cobertura.jl create mode 100644 test/test_cobertura.jl diff --git a/CHANGELOG.md b/CHANGELOG.md index d59acc6..4720c03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The JSON-RPC protocol gained a client → controller `shutdown` notification for a graceful shutdown: every run is cancelled, every test process is terminated (force-killed if it does not exit within the grace period) and the controller exits once they are gone. - `write_junit_xml(io_or_path, ::TestrunResult; root)` writes a test run as JUnit XML, the one report format every CI system ingests. Test items are grouped into one `` per source file and one `` per (item × run profile); captured output goes to `` with ANSI escape sequences stripped, and performance statistics become ``. It is a pure function of a `TestrunResult`, so a result file written by one process can be converted by another. - `write_lcov(io_or_path, ::TestrunResult; root)` writes a run's merged coverage in LCOV info format, so consumers no longer have to reach into the vendored `CoverageTools`. `root` is a directory path or `file:` URI that the `SF:` paths are relativized against, mirroring `write_junit_xml` — coverage services match `SF:` paths against paths in the repository, and the absolute paths of a CI runner match nothing at all, which is one way a fully covered package comes out at 0%. A file outside `root` keeps its absolute path rather than a `..`-heavy one, the same choice the JUnit writer makes, and paths always use `/` separators so a Windows leg and a Linux leg of the same matrix contribute the same file names to a merged report. +- `write_cobertura(io_or_path, ::TestrunResult; root)` writes a run's merged coverage as a Cobertura XML report — the sibling of `write_lcov`, with the same `root` relativization, the same `/` separators and the same "returns `false` and writes nothing when the run collected no coverage" contract, so a caller can drive both from one `if`. It exists because GitHub Code Quality's coverage feature takes Cobertura and nothing else. Cobertura describes branch and method coverage too; Julia's line-count instrumentation produces neither, so those attributes are present and always zero, and a line Julia could not instrument is left out rather than reported as a line nobody ran. - `TestrunResultTestitemProfile` gained a `perf` field and `TestrunResult` a `coverage` field. Both are optional and both are read tolerantly: a result file written before these existed still parses, which matters because `julia-report-ci-results` merges files produced by every leg of a CI matrix and those legs are not necessarily on the same version. - `TestItemDetail` gained `option_skip`, carrying the `@testitem` `skip` kwarg — either a literal `Bool` or the source text of an expression, which is evaluated in the test process rather than the controller so it sees the test process's Julia version and platform. - The terminal test item callbacks take an optional trailing argument: performance statistics for `passed`/`failed`/`errored`, and a reason for `skipped`. Callbacks written against the previous signatures are unaffected — the controller falls back to the arity the callback accepts. diff --git a/Project.toml b/Project.toml index a7bd985..fa1eb0b 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "TestItemControllers" uuid = "340407ab-12a4-4a78-8da2-3de3b3fd1448" authors = ["David Anthoff "] -version = "1.11.3-DEV" +version = "1.12.0-DEV" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" diff --git a/src/TestItemControllers.jl b/src/TestItemControllers.jl index 6d65b69..692c7c1 100644 --- a/src/TestItemControllers.jl +++ b/src/TestItemControllers.jl @@ -27,7 +27,7 @@ export execute_testrun export TestrunResult, TestrunResultTestitem, TestrunResultTestitemProfile, TestrunResultMessage, TestrunResultStackFrame, TestrunResultDefinitionError, TestrunResultPerfStats, TestrunResultFileCoverage -export write_junit_xml, write_lcov +export write_junit_xml, write_lcov, write_cobertura include("json_protocol.jl") include("../shared/testserver_protocol.jl") @@ -40,6 +40,7 @@ include("results.jl") using .Results include("junit.jl") include("lcov.jl") +include("cobertura.jl") include("testenvironment.jl") include("fsm.jl") diff --git a/src/cobertura.jl b/src/cobertura.jl new file mode 100644 index 0000000..c0c8d4e --- /dev/null +++ b/src/cobertura.jl @@ -0,0 +1,168 @@ +""" +Cobertura XML export for the coverage carried on a [`TestrunResult`](@ref). + +The sibling of [`write_lcov`](@ref), for the consumers that take Cobertura rather than +LCOV — GitHub Code Quality's coverage feature takes Cobertura and nothing else. + +Cobertura describes branch and method coverage too. Julia's line-count instrumentation +produces neither, and the one consumer this exists for reads line coverage only, so the +branch and complexity attributes are present (the format's consumers expect them) and +always zero. +""" + +""" + write_cobertura(io_or_path, result::TestrunResult; root=nothing) + +Write the merged coverage of `result` as a Cobertura XML report. + +Returns `false` and writes nothing when the run collected no coverage — that is the normal +outcome for a run that was not started in coverage mode, not an error. The same contract as +[`write_lcov`](@ref), so a caller can drive both from the same `if`. + +`root` is a directory path or `file:` URI to relativize the `filename` attributes against; +without it they are absolute. Coverage services match those paths against paths in the +repository, and the absolute paths of a CI runner match nothing at all — which is one way a +fully covered package comes out at 0%. A file outside `root` keeps its absolute path rather +than a `..`-heavy one, the same choice [`write_junit_xml`](@ref) makes. + +Paths always use `/` separators, so a Windows leg and a Linux leg of the same matrix +contribute the same file names to a merged report. + +Entries whose URI is not a `file:` URI are skipped. +""" +function write_cobertura(io::IO, result::TestrunResult; root::Union{Nothing,AbstractString}=nothing) + files = _cobertura_files(result, root) + files === nothing && return false + _write_cobertura(io, files) + return true +end + +function write_cobertura(path::AbstractString, result::TestrunResult; root::Union{Nothing,AbstractString}=nothing) + files = _cobertura_files(result, root) + files === nothing && return false + open(path, "w") do io + _write_cobertura(io, files) + end + return true +end + +# `(filename, lines)` per source file, where `lines` is the `(number, hits)` of every +# instrumentable line. A line Julia could not instrument has no representation in Cobertura +# — there is no "not applicable" hit count — so it is left out rather than reported as a +# line nobody ran. +function _cobertura_files(result::TestrunResult, root::Union{Nothing,AbstractString}) + result.coverage === nothing && return nothing + isempty(result.coverage) && return nothing + + files = Tuple{String,Vector{Tuple{Int,Int}}}[] + for fc in result.coverage + # Shared with the JUnit and LCOV writers: same relativization, same `/` separators, + # same refusal to walk out of the root with `..`. + filename = _report_path(fc.uri, root) + filename === nothing && continue + lines = Tuple{Int,Int}[] + for (i, hits) in enumerate(fc.coverage) + hits === nothing && continue + push!(lines, (i, hits)) + end + push!(files, (filename, lines)) + end + + return isempty(files) ? nothing : files +end + +# Cobertura's `line-rate` is covered lines over instrumentable lines. A file with nothing to +# instrument is not 0% covered, it is undefined; 1.0 is what every producer emits for it and +# what keeps it from dragging a project's rate down. +function _cobertura_rate(covered::Integer, valid::Integer) + valid == 0 && return 1.0 + return covered / valid +end + +_cobertura_counts(lines) = (count(l -> l[2] > 0, lines), length(lines)) + +# Cobertura groups classes into packages. Nothing reads the grouping for a language without +# packages, but a report that puts every file in one anonymous bucket is unreadable in the +# viewers that do show it, so the directory is used — dotted, as the format's Java ancestry +# expects. +function _cobertura_package_name(filename::AbstractString) + dir = dirname(filename) + isempty(dir) && return "." + return replace(dir, '/' => '.') +end + +function _write_cobertura(io::IO, files::Vector{Tuple{String,Vector{Tuple{Int,Int}}}}) + total_covered = 0 + total_valid = 0 + for (_, lines) in files + covered, valid = _cobertura_counts(lines) + total_covered += covered + total_valid += valid + end + + # Cobertura timestamps are milliseconds since the epoch. + timestamp = round(Int, time() * 1000) + + println(io, "") + println(io, "") + println(io, "") + # The filenames are already relative to the root the caller gave us, so the source root + # a consumer should join them against is wherever it checked the repository out. + println(io, " ") + println(io, " .") + println(io, " ") + println(io, " ") + + for (package, members) in _group_by_package(files) + pkg_covered = 0 + pkg_valid = 0 + for (_, lines) in members + covered, valid = _cobertura_counts(lines) + pkg_covered += covered + pkg_valid += valid + end + println(io, " ") + println(io, " ") + for (filename, lines) in members + covered, valid = _cobertura_counts(lines) + println(io, " ") + println(io, " ") + println(io, " ") + for (number, hits) in lines + println(io, " ") + end + println(io, " ") + println(io, " ") + end + println(io, " ") + println(io, " ") + end + + println(io, " ") + println(io, "") + + return nothing +end + +# Package order follows first appearance, and files keep the order the result gave them, so +# the same run always produces the same bytes. +function _group_by_package(files::Vector{Tuple{String,Vector{Tuple{Int,Int}}}}) + order = String[] + groups = Dict{String,Vector{Tuple{String,Vector{Tuple{Int,Int}}}}}() + for entry in files + package = _cobertura_package_name(entry[1]) + members = get!(groups, package) do + push!(order, package) + Tuple{String,Vector{Tuple{Int,Int}}}[] + end + push!(members, entry) + end + return [(package, groups[package]) for package in order] +end diff --git a/test/test_cobertura.jl b/test/test_cobertura.jl new file mode 100644 index 0000000..bec6c3c --- /dev/null +++ b/test/test_cobertura.jl @@ -0,0 +1,135 @@ +@testitem "Cobertura export" begin + using TestItemControllers: write_cobertura + using TestItemControllers.Results + + empty_result = TestrunResult(TestrunResultDefinitionError[], TestrunResultTestitem[], Dict{String,String}()) + + io = IOBuffer() + @test write_cobertura(io, empty_result) == false + @test isempty(take!(io)) + + covered = TestrunResult( + TestrunResultDefinitionError[], + TestrunResultTestitem[], + Dict{String,String}(), + [TestrunResultFileCoverage("file:///c%3A/pkg/src/f.jl", Union{Nothing,Int}[nothing, 3, 0, nothing])], + ) + + io = IOBuffer() + @test write_cobertura(io, covered; root="file:///c%3A/pkg") == true + xml = String(take!(io)) + + @test startswith(xml, "") + @test occursin("", xml) + @test occursin("filename=\"src/f.jl\"", xml) + @test occursin("name=\"f.jl\"", xml) + @test occursin("", xml) + @test occursin("", xml) + # A line Julia could not instrument has no representation in Cobertura, so it is left + # out rather than reported as a line nobody ran. + @test !occursin("number=\"1\"", xml) + @test !occursin("number=\"4\"", xml) +end + +@testitem "Cobertura export escapes and skips what it cannot report" begin + using TestItemControllers: write_cobertura + using TestItemControllers.Results + + # The consumers parse this with a real XML parser, so an unescaped character in a path + # is a silently rejected report rather than a visibly broken one. + result = TestrunResult( + TestrunResultDefinitionError[], + TestrunResultTestitem[], + Dict{String,String}(), + [ + TestrunResultFileCoverage("file:///c%3A/pkg/src/a%20%26%20b.jl", Union{Nothing,Int}[1, 0]), + TestrunResultFileCoverage("file:///c%3A/pkg/src/sub/c.jl", Union{Nothing,Int}[nothing]), + # Not a `file:` URI, so it has no path to report and is skipped. + TestrunResultFileCoverage("untitled:Untitled-1", Union{Nothing,Int}[1]), + ], + ) + + io = IOBuffer() + @test write_cobertura(io, result; root="file:///c%3A/pkg") == true + xml = String(take!(io)) + + @test occursin("filename=\"src/a & b.jl\"", xml) + @test !occursin("Untitled-1", xml) + # A directory becomes a dotted package name, the way the format's Java ancestry expects. + # A file with nothing to instrument is not 0% covered, it is undefined -- reporting it + # as zero would drag the project's rate down for a file with no code in it. + @test occursin(" "/"), xml) + + # ...and a relative root is resolved against the working directory, which `relpath` + # does not do on its own. + io = IOBuffer() + cd(dir) do + write_cobertura(io, result; root="pkg") + end + @test occursin("filename=\"src/f.jl\"", String(take!(io))) + end +end + +@testitem "Cobertura export writes to a path" begin + using TestItemControllers: write_cobertura + using TestItemControllers.Results + + empty_result = TestrunResult(TestrunResultDefinitionError[], TestrunResultTestitem[], Dict{String,String}()) + + covered = TestrunResult( + TestrunResultDefinitionError[], + TestrunResultTestitem[], + Dict{String,String}(), + [TestrunResultFileCoverage("file:///c%3A/pkg/src/f.jl", Union{Nothing,Int}[1])], + ) + + mktempdir() do dir + path = joinpath(dir, "cobertura.xml") + + # A run without coverage must not leave an empty file behind: an empty report is + # indistinguishable from a real 0% one to whatever uploads it. + @test write_cobertura(path, empty_result) == false + @test !isfile(path) + + @test write_cobertura(path, covered; root="file:///c%3A/pkg") == true + @test occursin("filename=\"src/f.jl\"", read(path, String)) + end +end From b3a7f94352da058ea221135ccca905e0754bb63e Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Sat, 19 Sep 2026 15:43:15 -0700 Subject: [PATCH 2/2] Write the Cobertura timestamp as Int64, not Int `round(Int, time() * 1000)` is `round(Int32, ...)` on a 32-bit Julia, and epoch milliseconds passed what an Int32 can hold in 1970. Every x86 leg therefore threw `InexactError: Int32(1.7898e12)` out of `write_cobertura` and produced no report at all. The test now pins the attribute: it has to parse as an integer larger than `typemax(Int32)`, which is true on every platform and is what the 32-bit legs were failing to reach. Co-Authored-By: Claude Opus 5 --- src/cobertura.jl | 6 ++++-- test/test_cobertura.jl | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/cobertura.jl b/src/cobertura.jl index c0c8d4e..7fb342d 100644 --- a/src/cobertura.jl +++ b/src/cobertura.jl @@ -100,8 +100,10 @@ function _write_cobertura(io::IO, files::Vector{Tuple{String,Vector{Tuple{Int,In total_valid += valid end - # Cobertura timestamps are milliseconds since the epoch. - timestamp = round(Int, time() * 1000) + # Cobertura timestamps are milliseconds since the epoch. `Int64` explicitly, not + # `Int`: on a 32-bit Julia that is `Int32`, and epoch milliseconds passed what an + # `Int32` can hold in 1970. + timestamp = round(Int64, time() * 1000) println(io, "") println(io, "") diff --git a/test/test_cobertura.jl b/test/test_cobertura.jl index bec6c3c..6210458 100644 --- a/test/test_cobertura.jl +++ b/test/test_cobertura.jl @@ -35,6 +35,12 @@ # out rather than reported as a line nobody ran. @test !occursin("number=\"1\"", xml) @test !occursin("number=\"4\"", xml) + + # Epoch milliseconds overflow an `Int32`, so on a 32-bit Julia a `round(Int, ...)` + # here throws an InexactError and no report gets written at all. + stamp = match(r"timestamp=\"(\d+)\"", xml) + @test stamp !== nothing + @test parse(Int64, stamp[1]) > typemax(Int32) end @testitem "Cobertura export escapes and skips what it cannot report" begin