From 8667d6b2b454cf5fe8e1e898ad2459df3dc620e8 Mon Sep 17 00:00:00 2001
From: Michel Schanen <michel.schanen@gmail.com>
Date: Tue, 21 Jul 2026 10:47:27 -0500
Subject: [PATCH 01/20] Add recycle_on_failure and retries options to runtests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A test that corrupts process-wide state — the motivating case is a GPU
whose driver ends up in a state where every subsequent allocation in the
process fails — poisons every later test scheduled onto the same worker,
turning one bad test into a cascade of failed files. The Distributed-
based harness this package was extracted from recycled a worker after
any failed test; restore that behavior behind `recycle_on_failure =
true`, alongside the existing max-rss and crash recycling.

With `retries = N`, tests that did not pass are re-run up to N times
after the main run completes: sequentially, on a single fresh worker,
with all other workers stopped. Parallel test runs create resource
contention (several workers sharing one GPU or a limited amount of RAM),
so a failure can mean "lost the resource race" rather than "broken":
re-running on an otherwise-idle system distinguishes the two. Tests that
failed due to contention reliably pass on the idle retry, while
deterministic failures fail again and are reported exactly once — only
the final attempt of each test enters the results, and retried tests are
visibly marked in the output.

Both options default to off.
---
 src/ParallelTestRunner.jl | 73 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 73 insertions(+)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 7c1723e..a433360 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -999,6 +999,17 @@ runtests(MyPackage, ARGS; serial=["big_alloc_test", "huge_matrix"])
 
 Workers are automatically recycled when they exceed memory limits to prevent out-of-memory
 issues during long test runs. The memory limit is set based on system architecture.
+
+## Failure Handling
+
+With `recycle_on_failure = true`, a worker is recycled after any test that did not pass, so
+a test that corrupts process-wide state (e.g. wedges a GPU driver) cannot poison subsequent
+tests on the same worker.
+
+With `retries = N` (default 0), tests that did not pass are re-run up to `N` times after
+the main run completes — sequentially, on a single fresh worker, with all other workers
+stopped — so tests that failed due to resource pressure from concurrent workers get an
+otherwise-idle system. Only the final attempt of each test is reported.
 """
 function runtests(mod::Module, args::ParsedArgs;
                   testsuite::Dict{String,Expr} = find_tests(pwd()),
@@ -1013,6 +1024,8 @@ function runtests(mod::Module, args::ParsedArgs;
                   stdout = Base.stdout,
                   stderr = Base.stderr,
                   max_worker_rss = get_max_worker_rss(),
+                  recycle_on_failure::Bool = false,
+                  retries::Integer = 0,
                   )
     #
     # set-up
@@ -1070,6 +1083,8 @@ function runtests(mod::Module, args::ParsedArgs;
         stdout,
         stderr,
         max_worker_rss,
+        recycle_on_failure,
+        retries,
     )
 end
 
@@ -1092,6 +1107,8 @@ function _runtests(mod::Module, args::ParsedArgs;
                    stdout = Base.stdout,
                    stderr = Base.stderr,
                    max_worker_rss = get_max_worker_rss(),
+                   recycle_on_failure::Bool = false,
+                   retries::Integer = 0,
                    )
 
     # partition into serial and parallel groups
@@ -1319,6 +1336,7 @@ function _runtests(mod::Module, args::ParsedArgs;
     #
 
     tests_to_start = Threads.Atomic{Int}(length(tests))
+    interrupted = false
     # After parallel-before-serial: stop extra workers so only one process is alive for
     # serial tests, but keep one parallel worker so we do not add a third addworker (ID_COUNTER).
     function drain_pool_leaving_one_worker!(pool, njobs)
@@ -1416,6 +1434,11 @@ function _runtests(mod::Module, args::ParsedArgs;
                                   # the worker has reached the max-rss limit, recycle it
                                   # so future tests start with a smaller working set
                                   Malt.stop(wrkr)
+                              elseif recycle_on_failure && anynonpass(result[])
+                                  # a failing test may have left the worker in a bad state
+                                  # (e.g. a wedged GPU driver whose every later allocation
+                                  # fails); recycle it so future tests get a fresh process
+                                  Malt.stop(wrkr)
                               end
                           else
                               # One of Malt.TerminatedWorkerException, Malt.RemoteException, or ErrorException
@@ -1482,6 +1505,7 @@ function _runtests(mod::Module, args::ParsedArgs;
             end
         end
     catch err
+        interrupted = true
         if !(err isa InterruptException)
             println(io_ctx.stderr, "\nCaught an error, stopping...")
         end
@@ -1519,6 +1543,55 @@ function _runtests(mod::Module, args::ParsedArgs;
         end
     end
 
+    # retry failed tests, if requested: sequentially, on a single fresh worker, with every
+    # other worker gone — tests that failed due to resource pressure (e.g. GPU memory
+    # oversubscription from concurrent workers) reliably pass on an otherwise-idle system.
+    # only the retried result is reported; persistent failures fail again and are reported
+    # exactly once.
+    if retries > 0 && !interrupted && args.quickfail === nothing
+        local retry_wrkr = nothing
+        for round in 1:retries
+            retryable = [r.test for r in results.value
+                         if r.result isa Exception || anynonpass(r.result[])]
+            isempty(retryable) && break
+            println(io_ctx.stdout)
+            printstyled(io_ctx.stdout,
+                        "Retrying $(length(retryable)) failed test(s) on a fresh worker...\n";
+                        color = :yellow)
+            for test in retryable
+                if retry_wrkr === nothing || !Malt.isrunning(retry_wrkr)
+                    retry_wrkr = addworker(; init_worker_code, io_ctx.color, exename,
+                                           exeflags, env)
+                end
+                test_t0 = time()
+                result = try
+                    Malt.remote_eval_wait(Main, retry_wrkr.w, :(import ParallelTestRunner))
+                    Malt.remote_call_fetch(invokelatest, retry_wrkr.w, runtest,
+                                           RecordType, testsuite[test], test,
+                                           init_code, test_t0, custom_args)
+                catch ex
+                    isa(ex, InterruptException) && rethrow()
+                    ex
+                end
+                test_t1 = time()
+                output = @lock retry_wrkr.io String(take!(retry_wrkr.io[]))
+                filter!(r -> r.test != test, results.value)
+                push!(results.value, (; test, result, output, test_t0, test_t1))
+                if result isa AbstractTestRecord && !anynonpass(result[])
+                    printstyled(io_ctx.stdout, "  $test passed on retry\n"; color = :green)
+                else
+                    printstyled(io_ctx.stdout, "  $test failed again\n"; color = :red)
+                    # don't let a failure contaminate the next retry
+                    Malt.stop(retry_wrkr)
+                    retry_wrkr = nothing
+                end
+            end
+        end
+        if retry_wrkr !== nothing && Malt.isrunning(retry_wrkr)
+            Malt.stop(retry_wrkr)
+        end
+    end
+
     # print the output generated by each testset
     # (`@sync` above joined all writers, so `results` is quiescent from here on)
     for (testname, result, output, _start, _stop) in results.value

From 78d28adf69b8d5c81f164821b83f6fbd1e9323d9 Mon Sep 17 00:00:00 2001
From: Michel Schanen <michel.schanen@gmail.com>
Date: Mon, 3 Aug 2026 11:07:56 -0500
Subject: [PATCH 02/20] Document recycle_on_failure and retries

Add a "Failure Handling" section to the advanced usage guide covering both
options: why worker recycling after a failure is useful (process-wide state
corruption cascading onto later tests on the same worker) and what the retry
environment guarantees (all other workers stopped, sequential re-run on a
fresh worker, only the final attempt reported, retry worker recycled after a
repeat failure).

Also mention them in the feature list on the front page, and add a best
practice warning against using retries to paper over genuinely broken tests.
---
 docs/src/advanced.md | 51 ++++++++++++++++++++++++++++++++++++++++++++
 docs/src/index.md    |  9 ++++++++
 2 files changed, 60 insertions(+)

diff --git a/docs/src/advanced.md b/docs/src/advanced.md
index 2ac2e93..b749b7d 100644
--- a/docs/src/advanced.md
+++ b/docs/src/advanced.md
@@ -174,6 +174,55 @@ duration, longest first) and their results appear in the same overall summary.
     If the user filters tests via positional arguments (e.g. `julia test/runtests.jl unit`),
     any serial test names that were filtered out are silently removed from the serial list.
 
+## Failure Handling
+
+Both options described in this section are opt-in and default to off.
+
+### Recycling Workers after a Failure
+
+Workers are reused across tests, so a test that corrupts process-wide state — a wedged GPU driver whose every subsequent allocation fails, a global left in an inconsistent state, a library put in an unusable configuration — can make every later test scheduled on that same worker fail too.
+
+Setting `recycle_on_failure=true` stops the worker after any test that did not pass, so the next test gets a fresh process:
+
+```julia
+runtests(MyPackage, ARGS; recycle_on_failure=true)
+```
+
+This complements the existing recycling of workers exceeding `max_worker_rss` and of workers that crashed outright.
+The cost is worker start-up time (plus re-running `init_worker_code`) after each failure, which is why it is off by default: for a suite whose failures are self-contained it is pure overhead.
+
+### Retrying Failed Tests
+
+When several workers compete for a limited resource — GPU memory, RAM, a device that only allows so many contexts — a failure can mean "lost the race for the resource" rather than "the code is broken".
+Such a test typically passes when run on its own.
+
+The `retries` keyword argument re-runs tests that did not pass, up to `N` times, after the main run has completed:
+
+```julia
+runtests(MyPackage, ARGS; retries=1)
+```
+
+The retry environment is deliberately quiesced: all parallel workers have been stopped by then, and the retried tests run **sequentially on a single fresh worker**, so a test that failed only because of concurrent resource pressure gets an otherwise-idle system.
+If a test fails again, its worker is stopped before the next retry, so one failure cannot contaminate the following one.
+
+Only the final attempt of each test is recorded in the results, so a test that passes on retry is reported as passing and a persistently broken test is reported as failing exactly once.
+Retries are visible in the output, so flakiness is surfaced rather than hidden:
+
+```
+Retrying 2 failed test(s) on a fresh worker...
+  gpu/memory passed on retry
+  broken_test failed again
+```
+
+!!! note
+    Retries are skipped when the run was interrupted (e.g. `Ctrl+C`) or when `--quickfail` is
+    in effect, since in both cases the run stopped early on purpose.
+
+!!! tip
+    `recycle_on_failure` and `retries` address different halves of the same problem and work
+    well together: recycling keeps one bad test from cascading onto its worker during the run,
+    while retries give the tests that did fail a contention-free second chance.
+
 ## Custom Workers
 
 For tests that require specific environment variables or Julia flags, you can use the `test_worker` keyword argument to [`runtests`](@ref) to assign tests to custom workers:
@@ -303,3 +352,5 @@ function jltest {
 1. **Use custom workers sparingly**: Custom workers add overhead. Only use them when tests genuinely require different configurations.
 
 1. **Use `serial` for resource-intensive tests**: If a test allocates significant memory or uses exclusive hardware resources, mark it as serial rather than reducing `--jobs` globally. This keeps the rest of your suite running in parallel.
+
+1. **Don't paper over real failures with `retries`**: Retries are meant for failures caused by contention between concurrent workers, not for tests that are genuinely broken. Persistent failures still fail after their retries, and retried tests are reported as such, so keep an eye on which tests keep needing a second attempt.
diff --git a/docs/src/index.md b/docs/src/index.md
index cb89f41..41a948a 100644
--- a/docs/src/index.md
+++ b/docs/src/index.md
@@ -114,6 +114,15 @@ The `serial` keyword argument to [`runtests`](@ref) lets you designate specific
 for sequential execution, either before or after the parallel batch.
 See [Serial Tests](@ref) in the advanced usage guide for details.
 
+### Failure Recycling and Retries
+
+Workers are recycled when they crash or exceed the memory threshold.
+Additionally, `recycle_on_failure=true` recycles a worker after any failed test, so a test
+that corrupts process-wide state cannot poison later tests, and `retries=N` re-runs failed
+tests on an otherwise-idle system, to tell tests broken by resource contention apart from
+genuinely broken ones.
+See [Failure Handling](@ref) in the advanced usage guide for details.
+
 ### Real-time Progress
 
 The test runner provides real-time output showing:

From 01e4e6505d93fd4b0c2aa19f75876a80f43554ed Mon Sep 17 00:00:00 2001
From: Michel Schanen <michel.schanen@gmail.com>
Date: Mon, 3 Aug 2026 11:22:11 -0500
Subject: [PATCH 03/20] Add tests for recycle_on_failure and retries

For `recycle_on_failure`, run a fixed sequence of failing and passing tests
with a single job and count the workers created: the default reuses one
worker for all of them, while `recycle_on_failure=true` needs a fresh worker
after each failure.

For `retries`, use a test that fails on its first attempt and passes on any
subsequent one (recording attempts in a file, since each attempt runs in a
different process) to check that a test rescued by a retry is reported as
passing, and that it is the only worker alive while it runs. A persistently
failing test is checked to exhaust its retries and still be reported exactly
once. Also cover that retries are off by default and skipped under
`--quickfail`.
---
 test/runtests.jl | 197 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 197 insertions(+)

diff --git a/test/runtests.jl b/test/runtests.jl
index 1cb7314..be1ca53 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1463,6 +1463,203 @@ end
     end
 end
 
+@testset "recycle_on_failure" begin
+    # Call `_runtests` throughout, so that we can enforce a run order, and use a single job,
+    # so that all tests share the same pool slot: a test only gets a new worker if the
+    # previous one was recycled.
+    testsuite = Dict(
+        "fail1" => :( @test false ),
+        "pass1" => :( @test true ),
+        "fail2" => :( @test false ),
+        "pass2" => :( @test true ),
+    )
+    tests = ["fail1", "pass1", "fail2", "pass2"]
+
+    @testset "workers are reused across failures by default" begin
+        io = IOBuffer()
+        old_id_counter = ParallelTestRunner.ID_COUNTER[]
+        @test_throws Test.FallbackTestSetException begin
+            ParallelTestRunner._runtests(
+                ParallelTestRunner, parse_args(["--jobs=1"]);
+                testsuite,
+                tests,
+                historical_durations=Dict{String, Float64}(),
+                stdout=io,
+                stderr=io,
+            )
+        end
+        str = String(take!(io))
+        @test contains(str, "FAILURE")
+        # A failing test does not recycle its worker, so a single one runs all four tests.
+        @test ParallelTestRunner.ID_COUNTER[] == old_id_counter + 1
+    end
+
+    @testset "worker is recycled after a failed test" begin
+        io = IOBuffer()
+        old_id_counter = ParallelTestRunner.ID_COUNTER[]
+        @test_throws Test.FallbackTestSetException begin
+            ParallelTestRunner._runtests(
+                ParallelTestRunner, parse_args(["--jobs=1"]);
+                testsuite,
+                tests,
+                historical_durations=Dict{String, Float64}(),
+                stdout=io,
+                stderr=io,
+                recycle_on_failure=true,
+            )
+        end
+        str = String(take!(io))
+        @test contains(str, "FAILURE")
+        # `fail1` and `fail2` recycle their worker, so `pass1` and `pass2` each need a fresh
+        # one: 1 initial worker + 2 replacements.
+        @test ParallelTestRunner.ID_COUNTER[] == old_id_counter + 3
+    end
+end
+
+@testset "retries" begin
+    # A test that fails on its first attempt and passes on any subsequent one, by recording
+    # attempts in a file: the worker running the retry is a different process, so the marker
+    # has to live outside of it.
+    flaky_test(marker, body=:( @test true )) = quote
+        if isfile($marker)
+            $body
+        else
+            touch($marker)
+            @test false
+        end
+    end
+
+    @testset "failed test passing on retry is reported as passing" begin
+        mktempdir() do dir
+            testsuite = Dict(
+                "flaky" => flaky_test(joinpath(dir, "flaky")),
+                "passes" => :( @test true ),
+            )
+            io = IOBuffer()
+            @show_if_error io ParallelTestRunner._runtests(
+                ParallelTestRunner, parse_args(["--jobs=1"]);
+                testsuite,
+                tests=["flaky", "passes"],
+                historical_durations=Dict{String, Float64}(),
+                stdout=io,
+                stderr=io,
+                retries=1,
+            )
+            str = String(take!(io))
+            # Only the failed test is retried, and its retried result is the one reported.
+            @test contains(str, "Retrying 1 failed test(s)")
+            @test contains(str, "flaky passed on retry")
+            @test !contains(str, "passes passed on retry")
+            @test contains(str, "SUCCESS")
+            # Two results in total: the failed attempt of `flaky` was replaced by the
+            # retried one, rather than reported next to it.
+            @test contains(str, r"Overall +\| +2 +2 ")
+        end
+    end
+
+    @testset "persistent failure is retried and reported once" begin
+        testsuite = Dict(
+            "always_fails" => :( @test false ),
+            "passes" => :( @test true ),
+        )
+        io = IOBuffer()
+        @test_throws Test.FallbackTestSetException begin
+            ParallelTestRunner._runtests(
+                ParallelTestRunner, parse_args(["--jobs=1"]);
+                testsuite,
+                tests=["always_fails", "passes"],
+                historical_durations=Dict{String, Float64}(),
+                stdout=io,
+                stderr=io,
+                retries=2,
+            )
+        end
+        str = String(take!(io))
+        @test contains(str, "FAILURE")
+        # Both retry rounds run, and each of them fails again.
+        @test length(collect(eachmatch(r"always_fails failed again", str))) == 2
+        # Despite the three attempts, the test is reported exactly once, as a failure.
+        @test contains(str, r"always_fails +\| +1 +1 ")
+    end
+
+    @testset "retried test runs alone" begin
+        mktempdir() do dir
+            # On its retry, the flaky test checks it is the only worker left alive.
+            check_alone = quote
+                children = _count_child_pids($(getpid()))
+                if children >= 0
+                    @test children == 1
+                end
+            end
+            testsuite = Dict(
+                "flaky" => flaky_test(joinpath(dir, "flaky"), check_alone),
+                "pass1" => :( @test true ),
+                "pass2" => :( @test true ),
+                "pass3" => :( @test true ),
+            )
+            io = IOBuffer()
+            @show_if_error io ParallelTestRunner._runtests(
+                ParallelTestRunner, parse_args(["--jobs=3"]);
+                testsuite,
+                tests=["flaky", "pass1", "pass2", "pass3"],
+                historical_durations=Dict{String, Float64}(),
+                init_code=:(include($(joinpath(@__DIR__, "utils.jl")))),
+                stdout=io,
+                stderr=io,
+                retries=1,
+            )
+            str = String(take!(io))
+            @test contains(str, "flaky passed on retry")
+            @test contains(str, "SUCCESS")
+        end
+    end
+
+    @testset "no retries by default" begin
+        mktempdir() do dir
+            testsuite = Dict("flaky" => flaky_test(joinpath(dir, "flaky")))
+            io = IOBuffer()
+            @test_throws Test.FallbackTestSetException begin
+                ParallelTestRunner._runtests(
+                    ParallelTestRunner, parse_args(["--jobs=1"]);
+                    testsuite,
+                    tests=["flaky"],
+                    historical_durations=Dict{String, Float64}(),
+                    stdout=io,
+                    stderr=io,
+                )
+            end
+            str = String(take!(io))
+            @test !contains(str, "Retrying")
+            @test contains(str, "FAILURE")
+        end
+    end
+
+    @testset "quickfail skips retries" begin
+        mktempdir() do dir
+            testsuite = Dict(
+                "flaky" => flaky_test(joinpath(dir, "flaky")),
+                "passes" => :( @test true ),
+            )
+            io = IOBuffer()
+            @test_throws Test.FallbackTestSetException begin
+                ParallelTestRunner._runtests(
+                    ParallelTestRunner, parse_args(["--quickfail", "--jobs=1"]);
+                    testsuite,
+                    tests=["flaky", "passes"],
+                    historical_durations=Dict{String, Float64}(),
+                    stdout=io,
+                    stderr=io,
+                    retries=1,
+                )
+            end
+            str = String(take!(io))
+            # The run stopped early on purpose, retrying would defeat that.
+            @test !contains(str, "Retrying")
+            @test contains(str, "FAILURE")
+        end
+    end
+end
+
 # This testset should always be the last one, don't add anything after this.
 # We want to make sure there are no running workers at the end of the tests.
 @testset "no workers running" begin

From 3080b24c0eab05e82227ef00caa69c1c646544b2 Mon Sep 17 00:00:00 2001
From: Michel Schanen <michel.schanen@gmail.com>
Date: Fri, 7 Aug 2026 10:34:03 -0500
Subject: [PATCH 04/20] Respect the test_worker hook when retrying failed tests

---
 src/ParallelTestRunner.jl | 37 ++++++++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index a433360..3a5970e 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1559,14 +1559,27 @@ function _runtests(mod::Module, args::ParsedArgs;
                         "Retrying $(length(retryable)) failed test(s) on a fresh worker...\n";
                         color = :yellow)
             for test in retryable
-                if retry_wrkr === nothing || !Malt.isrunning(retry_wrkr)
-                    retry_wrkr = addworker(; init_worker_code, io_ctx.color, exename,
-                                           exeflags, env)
+                # pass in init_worker_code to custom worker function if defined
+                wrkr = if init_worker_code == :()
+                    test_worker(test)
+                else
+                    test_worker(test, init_worker_code)
+                end
+                if wrkr !== nothing && !Malt.isrunning(wrkr)
+                    wrkr = nothing
+                end
+                custom = wrkr !== nothing
+                if !custom
+                    if retry_wrkr === nothing || !Malt.isrunning(retry_wrkr)
+                        retry_wrkr = addworker(; init_worker_code, io_ctx.color, exename,
+                                               exeflags, env)
+                    end
+                    wrkr = retry_wrkr
                 end
                 test_t0 = time()
                 result = try
-                    Malt.remote_eval_wait(Main, retry_wrkr.w, :(import ParallelTestRunner))
-                    Malt.remote_call_fetch(invokelatest, retry_wrkr.w, runtest,
+                    Malt.remote_eval_wait(Main, wrkr.w, :(import ParallelTestRunner))
+                    Malt.remote_call_fetch(invokelatest, wrkr.w, runtest,
                                            RecordType, testsuite[test], test,
                                            init_code, test_t0, custom_args)
                 catch ex
@@ -1574,16 +1587,22 @@ function _runtests(mod::Module, args::ParsedArgs;
                     ex
                 end
                 test_t1 = time()
-                output = @lock retry_wrkr.io String(take!(retry_wrkr.io[]))
+                output = @lock wrkr.io String(take!(wrkr.io[]))
                 filter!(r -> r.test != test, results.value)
                 push!(results.value, (; test, result, output, test_t0, test_t1))
                 if result isa AbstractTestRecord && !anynonpass(result[])
                     printstyled(io_ctx.stdout, "  $test passed on retry\n"; color = :green)
                 else
                     printstyled(io_ctx.stdout, "  $test failed again\n"; color = :red)
-                    # don't let a failure contaminate the next retry
-                    Malt.stop(retry_wrkr)
-                    retry_wrkr = nothing
+                    if !custom
+                        # don't let a failure contaminate the next retry
+                        Malt.stop(retry_wrkr)
+                        retry_wrkr = nothing
+                    end
+                end
+                # get rid of the custom worker
+                if custom && Malt.isrunning(wrkr)
+                    Malt.stop(wrkr)
                 end
             end
         end

From 192af09e1adeb8d498b31312fd62c59f38f02780 Mon Sep 17 00:00:00 2001
From: Michel Schanen <michel.schanen@gmail.com>
Date: Fri, 7 Aug 2026 10:34:11 -0500
Subject: [PATCH 05/20] Remove redundant ParallelTestRunner import in the retry
 loop

---
 src/ParallelTestRunner.jl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 3a5970e..863fb4e 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1578,7 +1578,6 @@ function _runtests(mod::Module, args::ParsedArgs;
                 end
                 test_t0 = time()
                 result = try
-                    Malt.remote_eval_wait(Main, wrkr.w, :(import ParallelTestRunner))
                     Malt.remote_call_fetch(invokelatest, wrkr.w, runtest,
                                            RecordType, testsuite[test], test,
                                            init_code, test_t0, custom_args)

From 400d94675579d66627af01e2b200101486e941f1 Mon Sep 17 00:00:00 2001
From: Michel Schanen <michel.schanen@gmail.com>
Date: Fri, 7 Aug 2026 10:34:27 -0500
Subject: [PATCH 06/20] Document recycle_on_failure and retries in the runtests
 signature

---
 src/ParallelTestRunner.jl | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 863fb4e..f8439f4 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -872,7 +872,9 @@ end
              stderr = Base.stderr,
              max_worker_rss = get_max_worker_rss(),
              serial = String[],
-             serial_position::Symbol = :before)
+             serial_position::Symbol = :before,
+             recycle_on_failure::Bool = false,
+             retries::Integer = 0)
     runtests(mod::Module, ARGS; ...)
 
 Run Julia tests in parallel across multiple worker processes.
@@ -920,6 +922,10 @@ Several keyword arguments are also supported:
   testsuite; names that are valid but deselected by command-line filtering are ignored.
 - `serial_position`: When to run serial tests relative to the parallel batch.
   Must be `:before` (default) or `:after`.
+- `recycle_on_failure`: Whether to recycle a worker after any test that did not pass
+  (default: `false`). See the Failure Handling section below.
+- `retries`: How many times to re-run tests that did not pass after the main run completes
+  (default: `0`). See the Failure Handling section below.
 
 ## Command Line Options
 

From 98e8cfff17588919a954206a81f2266b3175f328 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Fri, 7 Aug 2026 17:04:06 -0300
Subject: [PATCH 07/20] Remove no longer necessary arguments

---
 test/runtests.jl | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/test/runtests.jl b/test/runtests.jl
index be1ca53..c8cf204 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1483,7 +1483,6 @@ end
                 ParallelTestRunner, parse_args(["--jobs=1"]);
                 testsuite,
                 tests,
-                historical_durations=Dict{String, Float64}(),
                 stdout=io,
                 stderr=io,
             )
@@ -1502,7 +1501,6 @@ end
                 ParallelTestRunner, parse_args(["--jobs=1"]);
                 testsuite,
                 tests,
-                historical_durations=Dict{String, Float64}(),
                 stdout=io,
                 stderr=io,
                 recycle_on_failure=true,
@@ -1540,7 +1538,6 @@ end
                 ParallelTestRunner, parse_args(["--jobs=1"]);
                 testsuite,
                 tests=["flaky", "passes"],
-                historical_durations=Dict{String, Float64}(),
                 stdout=io,
                 stderr=io,
                 retries=1,
@@ -1568,7 +1565,6 @@ end
                 ParallelTestRunner, parse_args(["--jobs=1"]);
                 testsuite,
                 tests=["always_fails", "passes"],
-                historical_durations=Dict{String, Float64}(),
                 stdout=io,
                 stderr=io,
                 retries=2,
@@ -1602,7 +1598,6 @@ end
                 ParallelTestRunner, parse_args(["--jobs=3"]);
                 testsuite,
                 tests=["flaky", "pass1", "pass2", "pass3"],
-                historical_durations=Dict{String, Float64}(),
                 init_code=:(include($(joinpath(@__DIR__, "utils.jl")))),
                 stdout=io,
                 stderr=io,
@@ -1623,7 +1618,6 @@ end
                     ParallelTestRunner, parse_args(["--jobs=1"]);
                     testsuite,
                     tests=["flaky"],
-                    historical_durations=Dict{String, Float64}(),
                     stdout=io,
                     stderr=io,
                 )
@@ -1646,7 +1640,6 @@ end
                     ParallelTestRunner, parse_args(["--quickfail", "--jobs=1"]);
                     testsuite,
                     tests=["flaky", "passes"],
-                    historical_durations=Dict{String, Float64}(),
                     stdout=io,
                     stderr=io,
                     retries=1,

From 9ed6910e19eccd8a90a269eb3f5e06a605783fba Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Fri, 7 Aug 2026 17:21:27 -0300
Subject: [PATCH 08/20] Use existing test phase machinery to implement retries

---
 src/ParallelTestRunner.jl | 146 +++++++++++++++++++++-----------------
 1 file changed, 80 insertions(+), 66 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index f8439f4..1845448 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1510,6 +1510,20 @@ function _runtests(mod::Module, args::ParsedArgs;
                 end
             end
         end
+
+        # retries
+        for i in 1:retries
+            retry_tests = [r.test for r in results.value
+                                if r.result isa Exception || anynonpass(r.result[])]
+            isempty(retry_tests) && break
+
+            println(io_ctx.stdout, "Retrying $(length(retry_tests)) failed tests ($i)")
+            sem = Base.Semaphore(1)
+            shared_worker = serial_worker
+            filter!(r -> r.test ∉ retry_tests, results.value)
+
+            run_test_phase(retry_tests, sem, shared_worker)
+        end
     catch err
         interrupted = true
         if !(err isa InterruptException)
@@ -1549,72 +1563,72 @@ function _runtests(mod::Module, args::ParsedArgs;
         end
     end
 
-    # retry failed tests, if requested: sequentially, on a single fresh worker, with every
-    # other worker gone — tests that failed due to resource pressure (e.g. GPU memory
-    # oversubscription from concurrent workers) reliably pass on an otherwise-idle system.
-    # only the retried result is reported; persistent failures fail again and are reported
-    # exactly once.
-    if retries > 0 && !interrupted && args.quickfail === nothing
-        local retry_wrkr = nothing
-        for round in 1:retries
-            retryable = [r.test for r in results.value
-                         if r.result isa Exception || anynonpass(r.result[])]
-            isempty(retryable) && break
-            println(io_ctx.stdout)
-            printstyled(io_ctx.stdout,
-                        "Retrying $(length(retryable)) failed test(s) on a fresh worker...\n";
-                        color = :yellow)
-            for test in retryable
-                # pass in init_worker_code to custom worker function if defined
-                wrkr = if init_worker_code == :()
-                    test_worker(test)
-                else
-                    test_worker(test, init_worker_code)
-                end
-                if wrkr !== nothing && !Malt.isrunning(wrkr)
-                    wrkr = nothing
-                end
-                custom = wrkr !== nothing
-                if !custom
-                    if retry_wrkr === nothing || !Malt.isrunning(retry_wrkr)
-                        retry_wrkr = addworker(; init_worker_code, io_ctx.color, exename,
-                                               exeflags, env)
-                    end
-                    wrkr = retry_wrkr
-                end
-                test_t0 = time()
-                result = try
-                    Malt.remote_call_fetch(invokelatest, wrkr.w, runtest,
-                                           RecordType, testsuite[test], test,
-                                           init_code, test_t0, custom_args)
-                catch ex
-                    isa(ex, InterruptException) && rethrow()
-                    ex
-                end
-                test_t1 = time()
-                output = @lock wrkr.io String(take!(wrkr.io[]))
-                filter!(r -> r.test != test, results.value)
-                push!(results.value, (; test, result, output, test_t0, test_t1))
-                if result isa AbstractTestRecord && !anynonpass(result[])
-                    printstyled(io_ctx.stdout, "  $test passed on retry\n"; color = :green)
-                else
-                    printstyled(io_ctx.stdout, "  $test failed again\n"; color = :red)
-                    if !custom
-                        # don't let a failure contaminate the next retry
-                        Malt.stop(retry_wrkr)
-                        retry_wrkr = nothing
-                    end
-                end
-                # get rid of the custom worker
-                if custom && Malt.isrunning(wrkr)
-                    Malt.stop(wrkr)
-                end
-            end
-        end
-        if retry_wrkr !== nothing && Malt.isrunning(retry_wrkr)
-            Malt.stop(retry_wrkr)
-        end
-    end
+    # # retry failed tests, if requested: sequentially, on a single fresh worker, with every
+    # # other worker gone — tests that failed due to resource pressure (e.g. GPU memory
+    # # oversubscription from concurrent workers) reliably pass on an otherwise-idle system.
+    # # only the retried result is reported; persistent failures fail again and are reported
+    # # exactly once.
+    # if retries > 0 && !interrupted && args.quickfail === nothing
+    #     local retry_wrkr = nothing
+    #     for round in 1:retries
+    #         retryable = [r.test for r in results.value
+    #                      if r.result isa Exception || anynonpass(r.result[])]
+    #         isempty(retryable) && break
+    #         println(io_ctx.stdout)
+    #         printstyled(io_ctx.stdout,
+    #                     "Retrying $(length(retryable)) failed test(s) on a fresh worker...\n";
+    #                     color = :yellow)
+    #         for test in retryable
+    #             # pass in init_worker_code to custom worker function if defined
+    #             wrkr = if init_worker_code == :()
+    #                 test_worker(test)
+    #             else
+    #                 test_worker(test, init_worker_code)
+    #             end
+    #             if wrkr !== nothing && !Malt.isrunning(wrkr)
+    #                 wrkr = nothing
+    #             end
+    #             custom = wrkr !== nothing
+    #             if !custom
+    #                 if retry_wrkr === nothing || !Malt.isrunning(retry_wrkr)
+    #                     retry_wrkr = addworker(; init_worker_code, io_ctx.color, exename,
+    #                                            exeflags, env)
+    #                 end
+    #                 wrkr = retry_wrkr
+    #             end
+    #             test_t0 = time()
+    #             result = try
+    #                 Malt.remote_call_fetch(invokelatest, wrkr.w, runtest,
+    #                                        RecordType, testsuite[test], test,
+    #                                        init_code, test_t0, custom_args)
+    #             catch ex
+    #                 isa(ex, InterruptException) && rethrow()
+    #                 ex
+    #             end
+    #             test_t1 = time()
+    #             output = @lock wrkr.io String(take!(wrkr.io[]))
+    #             filter!(r -> r.test != test, results.value)
+    #             push!(results.value, (; test, result, output, test_t0, test_t1))
+    #             if result isa AbstractTestRecord && !anynonpass(result[])
+    #                 printstyled(io_ctx.stdout, "  $test passed on retry\n"; color = :green)
+    #             else
+    #                 printstyled(io_ctx.stdout, "  $test failed again\n"; color = :red)
+    #                 if !custom
+    #                     # don't let a failure contaminate the next retry
+    #                     Malt.stop(retry_wrkr)
+    #                     retry_wrkr = nothing
+    #                 end
+    #             end
+    #             # get rid of the custom worker
+    #             if custom && Malt.isrunning(wrkr)
+    #                 Malt.stop(wrkr)
+    #             end
+    #         end
+    #     end
+    #     if retry_wrkr !== nothing && Malt.isrunning(retry_wrkr)
+    #         Malt.stop(retry_wrkr)
+    #     end
+    # end
 
     # print the output generated by each testset
     # (`@sync` above joined all writers, so `results` is quiescent from here on)

From b06f704b2b3311c0d9ec5fb08fa440463020d4d9 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Fri, 7 Aug 2026 17:29:16 -0300
Subject: [PATCH 09/20] Print failures of non-final attempts yellow

---
 src/ParallelTestRunner.jl | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 1845448..1caf32f 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -167,6 +167,7 @@ struct TestIOContext
     alloc_align::Int
     rss_align::Int
     max_worker_rss::Int
+    nonpass_color::Ref{Symbol}
 end
 
 function test_IOContext(::Type{<:AbstractTestRecord}, stdout::IO, stderr::IO, lock::ReentrantLock, name_align::Int, verbose::Bool, max_worker_rss::Int)
@@ -181,7 +182,7 @@ function test_IOContext(::Type{<:AbstractTestRecord}, stdout::IO, stderr::IO, lo
 
     return TestIOContext(
         stdout, stderr, color, verbose, lock, name_align, elapsed_align, compile_align, gc_align, percent_align,
-        alloc_align, rss_align, max_worker_rss
+        alloc_align, rss_align, max_worker_rss, Ref(:red)
     )
 end
 
@@ -268,26 +269,26 @@ function print_test_failed(record::AbstractTestRecord, wrkr, test, ctx::TestIOCo
     base = parent(record)
     lock(ctx.lock)
     try
-        printstyled(ctx.stderr, test, color = :red)
+        printstyled(ctx.stderr, test, color = ctx.nonpass_color[])
         printstyled(
             ctx.stderr,
             lpad("($wrkr)", ctx.name_align - textwidth(test) + 1, " "), " │"
-            , color = :red
+            , color = ctx.nonpass_color[]
         )
 
         time_str = @sprintf("%7.2f", base.time)
-        printstyled(ctx.stderr, lpad(time_str, ctx.elapsed_align + 1, " "), " │", color = :red)
+        printstyled(ctx.stderr, lpad(time_str, ctx.elapsed_align + 1, " "), " │", color = ctx.nonpass_color[])
 
         if ctx.verbose
             init_time_str = @sprintf("%7.2f", base.total_time - base.time)
-            printstyled(ctx.stderr, lpad(init_time_str, ctx.elapsed_align + 1, " "), " │ ", color = :red)
+            printstyled(ctx.stderr, lpad(init_time_str, ctx.elapsed_align + 1, " "), " │ ", color = ctx.nonpass_color[])
         end
 
         failed_str = "failed at $(now())\n"
         # 11 -> 3 from " │ " 3x and 2 for each " " on either side
         fail_align = (11 + ctx.gc_align + ctx.percent_align + ctx.alloc_align + ctx.rss_align - textwidth(failed_str)) ÷ 2 + textwidth(failed_str)
         failed_str = lpad(failed_str, fail_align, " ")
-        printstyled(ctx.stderr, failed_str, color = :red)
+        printstyled(ctx.stderr, failed_str, color = ctx.nonpass_color[])
 
         # TODO: print other stats?
 
@@ -300,11 +301,11 @@ end
 function print_test_crashed(::Type{<:AbstractTestRecord}, wrkr, test, ctx::TestIOContext)
     lock(ctx.lock)
     try
-        printstyled(ctx.stderr, test, color = :red)
+        printstyled(ctx.stderr, test, color = ctx.nonpass_color[])
         printstyled(
             ctx.stderr,
             lpad("($wrkr)", ctx.name_align - textwidth(test) + 1, " "), " │",
-            " "^ctx.elapsed_align, " crashed at $(now())\n", color = :red
+            " "^ctx.elapsed_align, " crashed at $(now())\n", color = ctx.nonpass_color[]
         )
 
         flush(ctx.stderr)
@@ -1495,6 +1496,7 @@ function _runtests(mod::Module, args::ParsedArgs;
     end
     try
         phases = test_phases
+        retries > 0 && (io_ctx.nonpass_color[] = :yellow)
         for i in 1:length(phases)
             phase_tests, sem, shared_worker = phases[i]
             isempty(phase_tests) && continue
@@ -1513,6 +1515,7 @@ function _runtests(mod::Module, args::ParsedArgs;
 
         # retries
         for i in 1:retries
+            retries == i && (io_ctx.nonpass_color[] = :red)
             retry_tests = [r.test for r in results.value
                                 if r.result isa Exception || anynonpass(r.result[])]
             isempty(retry_tests) && break

From 6a13c5fd76f070031b1fcf86c136626453bea139 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Fri, 7 Aug 2026 17:54:56 -0300
Subject: [PATCH 10/20] Print properly

---
 src/ParallelTestRunner.jl | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 1caf32f..f624c2e 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1307,6 +1307,18 @@ function _runtests(mod::Module, args::ParsedArgs;
 
                         clear_status()
                         print_test_crashed(RecordType, wrkr, test_name, io_ctx)
+
+                    elseif msg_type === :retry
+                        tests_n, retry_n = msg[2], msg[3]
+
+                        clear_status()
+                        lock(io_ctx.lock)
+                        try
+                            printstyled(io_ctx.stdout, "Retrying $tests_n failed test$(tests_n > 1 ? "s" : " ") ($retry_n)\n", color=:white)
+                            flush(io_ctx.stdout)
+                        finally
+                            unlock(io_ctx.lock)
+                        end
                     end
                 end
 
@@ -1520,7 +1532,7 @@ function _runtests(mod::Module, args::ParsedArgs;
                                 if r.result isa Exception || anynonpass(r.result[])]
             isempty(retry_tests) && break
 
-            println(io_ctx.stdout, "Retrying $(length(retry_tests)) failed tests ($i)")
+            put!(printer_channel, (:retry, length(retry_tests), i))
             sem = Base.Semaphore(1)
             shared_worker = serial_worker
             filter!(r -> r.test ∉ retry_tests, results.value)

From 9e0783e4ce0a5fe96dd5460f1933a690192d6f6b Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Fri, 7 Aug 2026 18:00:32 -0300
Subject: [PATCH 11/20] Fix most tests

---
 test/runtests.jl | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/test/runtests.jl b/test/runtests.jl
index c8cf204..ee7062a 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1544,9 +1544,7 @@ end
             )
             str = String(take!(io))
             # Only the failed test is retried, and its retried result is the one reported.
-            @test contains(str, "Retrying 1 failed test(s)")
-            @test contains(str, "flaky passed on retry")
-            @test !contains(str, "passes passed on retry")
+            @test contains(str, "Retrying 1 failed test")
             @test contains(str, "SUCCESS")
             # Two results in total: the failed attempt of `flaky` was replaced by the
             # retried one, rather than reported next to it.
@@ -1573,7 +1571,7 @@ end
         str = String(take!(io))
         @test contains(str, "FAILURE")
         # Both retry rounds run, and each of them fails again.
-        @test length(collect(eachmatch(r"always_fails failed again", str))) == 2
+        @test length(collect(eachmatch(r"always_fails.*failed", str))) == 3
         # Despite the three attempts, the test is reported exactly once, as a failure.
         @test contains(str, r"always_fails +\| +1 +1 ")
     end
@@ -1604,7 +1602,7 @@ end
                 retries=1,
             )
             str = String(take!(io))
-            @test contains(str, "flaky passed on retry")
+            @test length(collect(eachmatch(r"failed", str))) == 2
             @test contains(str, "SUCCESS")
         end
     end

From d39598a9ef57bc7acfc76b29cb06ac9e39e102f9 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Sat, 8 Aug 2026 14:54:20 -0300
Subject: [PATCH 12/20] Remove old code

---
 src/ParallelTestRunner.jl | 67 ---------------------------------------
 1 file changed, 67 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index f624c2e..beef7fa 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1578,73 +1578,6 @@ function _runtests(mod::Module, args::ParsedArgs;
         end
     end
 
-    # # retry failed tests, if requested: sequentially, on a single fresh worker, with every
-    # # other worker gone — tests that failed due to resource pressure (e.g. GPU memory
-    # # oversubscription from concurrent workers) reliably pass on an otherwise-idle system.
-    # # only the retried result is reported; persistent failures fail again and are reported
-    # # exactly once.
-    # if retries > 0 && !interrupted && args.quickfail === nothing
-    #     local retry_wrkr = nothing
-    #     for round in 1:retries
-    #         retryable = [r.test for r in results.value
-    #                      if r.result isa Exception || anynonpass(r.result[])]
-    #         isempty(retryable) && break
-    #         println(io_ctx.stdout)
-    #         printstyled(io_ctx.stdout,
-    #                     "Retrying $(length(retryable)) failed test(s) on a fresh worker...\n";
-    #                     color = :yellow)
-    #         for test in retryable
-    #             # pass in init_worker_code to custom worker function if defined
-    #             wrkr = if init_worker_code == :()
-    #                 test_worker(test)
-    #             else
-    #                 test_worker(test, init_worker_code)
-    #             end
-    #             if wrkr !== nothing && !Malt.isrunning(wrkr)
-    #                 wrkr = nothing
-    #             end
-    #             custom = wrkr !== nothing
-    #             if !custom
-    #                 if retry_wrkr === nothing || !Malt.isrunning(retry_wrkr)
-    #                     retry_wrkr = addworker(; init_worker_code, io_ctx.color, exename,
-    #                                            exeflags, env)
-    #                 end
-    #                 wrkr = retry_wrkr
-    #             end
-    #             test_t0 = time()
-    #             result = try
-    #                 Malt.remote_call_fetch(invokelatest, wrkr.w, runtest,
-    #                                        RecordType, testsuite[test], test,
-    #                                        init_code, test_t0, custom_args)
-    #             catch ex
-    #                 isa(ex, InterruptException) && rethrow()
-    #                 ex
-    #             end
-    #             test_t1 = time()
-    #             output = @lock wrkr.io String(take!(wrkr.io[]))
-    #             filter!(r -> r.test != test, results.value)
-    #             push!(results.value, (; test, result, output, test_t0, test_t1))
-    #             if result isa AbstractTestRecord && !anynonpass(result[])
-    #                 printstyled(io_ctx.stdout, "  $test passed on retry\n"; color = :green)
-    #             else
-    #                 printstyled(io_ctx.stdout, "  $test failed again\n"; color = :red)
-    #                 if !custom
-    #                     # don't let a failure contaminate the next retry
-    #                     Malt.stop(retry_wrkr)
-    #                     retry_wrkr = nothing
-    #                 end
-    #             end
-    #             # get rid of the custom worker
-    #             if custom && Malt.isrunning(wrkr)
-    #                 Malt.stop(wrkr)
-    #             end
-    #         end
-    #     end
-    #     if retry_wrkr !== nothing && Malt.isrunning(retry_wrkr)
-    #         Malt.stop(retry_wrkr)
-    #     end
-    # end
-
     # print the output generated by each testset
     # (`@sync` above joined all writers, so `results` is quiescent from here on)
     for (testname, result, output, _start, _stop) in results.value

From 2a22b82dda454c983c587f33a7890e6b8542c99c Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Sat, 8 Aug 2026 15:03:39 -0300
Subject: [PATCH 13/20] Fix test

---
 src/ParallelTestRunner.jl | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index beef7fa..bc79c21 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1526,18 +1526,20 @@ function _runtests(mod::Module, args::ParsedArgs;
         end
 
         # retries
-        for i in 1:retries
-            retries == i && (io_ctx.nonpass_color[] = :red)
-            retry_tests = [r.test for r in results.value
-                                if r.result isa Exception || anynonpass(r.result[])]
-            isempty(retry_tests) && break
-
-            put!(printer_channel, (:retry, length(retry_tests), i))
-            sem = Base.Semaphore(1)
-            shared_worker = serial_worker
-            filter!(r -> r.test ∉ retry_tests, results.value)
-
-            run_test_phase(retry_tests, sem, shared_worker)
+        if retries > 0 && !interrupted && args.quickfail === nothing
+            for i in 1:retries
+                retries == i && (io_ctx.nonpass_color[] = :red)
+                retry_tests = [r.test for r in results.value
+                                    if r.result isa Exception || anynonpass(r.result[])]
+                isempty(retry_tests) && break
+
+                put!(printer_channel, (:retry, length(retry_tests), i))
+                sem = Base.Semaphore(1)
+                shared_worker = serial_worker
+                filter!(r -> r.test ∉ retry_tests, results.value)
+
+                run_test_phase(retry_tests, sem, shared_worker)
+            end
         end
     catch err
         interrupted = true

From f11fd263d129e24d1734dfbadddfd7f4e41dd135 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Sun, 9 Aug 2026 20:09:27 -0300
Subject: [PATCH 14/20] Tweak documentation

---
 docs/src/advanced.md      | 14 ++++++--------
 docs/src/index.md         |  9 +++++----
 src/ParallelTestRunner.jl |  6 ++----
 3 files changed, 13 insertions(+), 16 deletions(-)

diff --git a/docs/src/advanced.md b/docs/src/advanced.md
index b749b7d..e49acab 100644
--- a/docs/src/advanced.md
+++ b/docs/src/advanced.md
@@ -189,11 +189,10 @@ runtests(MyPackage, ARGS; recycle_on_failure=true)
 ```
 
 This complements the existing recycling of workers exceeding `max_worker_rss` and of workers that crashed outright.
-The cost is worker start-up time (plus re-running `init_worker_code`) after each failure, which is why it is off by default: for a suite whose failures are self-contained it is pure overhead.
 
 ### Retrying Failed Tests
 
-When several workers compete for a limited resource — GPU memory, RAM, a device that only allows so many contexts — a failure can mean "lost the race for the resource" rather than "the code is broken".
+When several workers compete for a limited resource (usually memory), a failure can mean "lost the race for the resource" rather than "the code is broken".
 Such a test typically passes when run on its own.
 
 The `retries` keyword argument re-runs tests that did not pass, up to `N` times, after the main run has completed:
@@ -202,16 +201,15 @@ The `retries` keyword argument re-runs tests that did not pass, up to `N` times,
 runtests(MyPackage, ARGS; retries=1)
 ```
 
-The retry environment is deliberately quiesced: all parallel workers have been stopped by then, and the retried tests run **sequentially on a single fresh worker**, so a test that failed only because of concurrent resource pressure gets an otherwise-idle system.
+Retried tests run **sequentially on a single fresh worker**, so a test that failed only because of concurrent resource pressure gets an otherwise-idle system.
 If a test fails again, its worker is stopped before the next retry, so one failure cannot contaminate the following one.
 
-Only the final attempt of each test is recorded in the results, so a test that passes on retry is reported as passing and a persistently broken test is reported as failing exactly once.
+Only the final attempt of each test is recorded in the results, so a test that passes on retry is reported as passing and a persistently broken test is reported as failing.
 Retries are visible in the output, so flakiness is surfaced rather than hidden:
 
 ```
-Retrying 2 failed test(s) on a fresh worker...
-  gpu/memory passed on retry
-  broken_test failed again
+Retrying 1 failed test  (1)
+fails      (8) │     0.05 │   failed at 2026-08-08T15:10:15.526
 ```
 
 !!! note
@@ -353,4 +351,4 @@ function jltest {
 
 1. **Use `serial` for resource-intensive tests**: If a test allocates significant memory or uses exclusive hardware resources, mark it as serial rather than reducing `--jobs` globally. This keeps the rest of your suite running in parallel.
 
-1. **Don't paper over real failures with `retries`**: Retries are meant for failures caused by contention between concurrent workers, not for tests that are genuinely broken. Persistent failures still fail after their retries, and retried tests are reported as such, so keep an eye on which tests keep needing a second attempt.
+1. **Only use `retries` for worker contention-related failures**: Not all intermittent failures are caused by parallel worker resource contention. Ensure you aren't masking real test failures when using this feature.
diff --git a/docs/src/index.md b/docs/src/index.md
index 41a948a..b7d569c 100644
--- a/docs/src/index.md
+++ b/docs/src/index.md
@@ -117,10 +117,11 @@ See [Serial Tests](@ref) in the advanced usage guide for details.
 ### Failure Recycling and Retries
 
 Workers are recycled when they crash or exceed the memory threshold.
-Additionally, `recycle_on_failure=true` recycles a worker after any failed test, so a test
-that corrupts process-wide state cannot poison later tests, and `retries=N` re-runs failed
-tests on an otherwise-idle system, to tell tests broken by resource contention apart from
-genuinely broken ones.
+Additionally, [`runtests`](@ref) has two keyword arguments to further customize
+failure hanlding. Setting `recycle_on_failure=true` recycles a worker after any
+failed test, so a test that corrupts process-wide state cannot poison later tests,
+and `retries=N` re-runs failed tests sequentially up to `N` times to reduce false
+failures caused by resource contention.
 See [Failure Handling](@ref) in the advanced usage guide for details.
 
 ### Real-time Progress
diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index bc79c21..f30548c 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1013,10 +1013,8 @@ With `recycle_on_failure = true`, a worker is recycled after any test that did n
 a test that corrupts process-wide state (e.g. wedges a GPU driver) cannot poison subsequent
 tests on the same worker.
 
-With `retries = N` (default 0), tests that did not pass are re-run up to `N` times after
-the main run completes — sequentially, on a single fresh worker, with all other workers
-stopped — so tests that failed due to resource pressure from concurrent workers get an
-otherwise-idle system. Only the final attempt of each test is reported.
+With `retries = N` (default 0), tests that did not pass are re-run sequantially up to `N`
+times after the main run completes. Only the final attempt of each test is reported.
 """
 function runtests(mod::Module, args::ParsedArgs;
                   testsuite::Dict{String,Expr} = find_tests(pwd()),

From acfd1a26bd0c963aa24b30082f12e6f98ff3d7ec Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Sun, 9 Aug 2026 20:14:38 -0300
Subject: [PATCH 15/20] Failed tests print red when --quickfail is set

---
 src/ParallelTestRunner.jl | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index f30548c..a357d7f 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1506,7 +1506,10 @@ function _runtests(mod::Module, args::ParsedArgs;
     end
     try
         phases = test_phases
-        retries > 0 && (io_ctx.nonpass_color[] = :yellow)
+
+        potential_retries = retries > 0 && !interrupted && args.quickfail === nothing
+
+        potential_retries && (io_ctx.nonpass_color[] = :yellow)
         for i in 1:length(phases)
             phase_tests, sem, shared_worker = phases[i]
             isempty(phase_tests) && continue
@@ -1524,7 +1527,7 @@ function _runtests(mod::Module, args::ParsedArgs;
         end
 
         # retries
-        if retries > 0 && !interrupted && args.quickfail === nothing
+        if potential_retries
             for i in 1:retries
                 retries == i && (io_ctx.nonpass_color[] = :red)
                 retry_tests = [r.test for r in results.value

From a5187dfbbe2ca04b93a243e673038320e0229e25 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Sun, 9 Aug 2026 20:44:43 -0300
Subject: [PATCH 16/20] Typo

---
 src/ParallelTestRunner.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index a357d7f..05811c1 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1013,7 +1013,7 @@ With `recycle_on_failure = true`, a worker is recycled after any test that did n
 a test that corrupts process-wide state (e.g. wedges a GPU driver) cannot poison subsequent
 tests on the same worker.
 
-With `retries = N` (default 0), tests that did not pass are re-run sequantially up to `N`
+With `retries = N` (default 0), tests that did not pass are re-run sequentially up to `N`
 times after the main run completes. Only the final attempt of each test is reported.
 """
 function runtests(mod::Module, args::ParsedArgs;

From 500ea91b405bd05cac028105404a5adde1b5e4ad Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 10 Aug 2026 13:50:57 -0300
Subject: [PATCH 17/20] Document failure printing colour

---
 docs/src/advanced.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/src/advanced.md b/docs/src/advanced.md
index e49acab..6da9bab 100644
--- a/docs/src/advanced.md
+++ b/docs/src/advanced.md
@@ -212,6 +212,10 @@ Retrying 1 failed test  (1)
 fails      (8) │     0.05 │   failed at 2026-08-08T15:10:15.526
 ```
 
+While a test still has an attempt left, its failure is printed in yellow, and the final
+attempt is printed in red. A red line therefore always marks the result that will be reported,
+and a yellow one marks a result that may still be replaced.
+
 !!! note
     Retries are skipped when the run was interrupted (e.g. `Ctrl+C`) or when `--quickfail` is
     in effect, since in both cases the run stopped early on purpose.

From cb55a2368c5040359220ff851ac670d3f57d5e8d Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 10 Aug 2026 13:58:37 -0300
Subject: [PATCH 18/20] Address "Single fresh worker is no longer guaranteed"

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 src/ParallelTestRunner.jl | 32 +++++++++++++++++++-------------
 test/runtests.jl          | 35 ++++++++++++++++++++++++++++++++++-
 2 files changed, 53 insertions(+), 14 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 05811c1..7dbbb3d 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1354,9 +1354,9 @@ function _runtests(mod::Module, args::ParsedArgs;
 
     tests_to_start = Threads.Atomic{Int}(length(tests))
     interrupted = false
-    # After parallel-before-serial: stop extra workers so only one process is alive for
-    # serial tests, but keep one parallel worker so we do not add a third addworker (ID_COUNTER).
-    function drain_pool_leaving_one_worker!(pool, njobs)
+    # Stop every all-but-`n` workers in the pool.Only safe at a
+    # phase boundary, where all `njobs` slots have been returned.
+    function drain_pool_leaving_n_workers!(pool, njobs, n)
         alive = PTRWorker[]
         for _ in 1:njobs
             p = take!(pool)
@@ -1364,18 +1364,17 @@ function _runtests(mod::Module, args::ParsedArgs;
                 push!(alive, p)
             end
         end
-        while length(alive) > 1
+        while length(alive) > n
             Malt.stop(pop!(alive))
         end
-        kept = isempty(alive) ? nothing : alive[1]
-        if kept !== nothing
-            put!(pool, kept)
+        for p in alive
+            put!(pool, p)
         end
-        for _ in 1:(njobs - (kept === nothing ? 0 : 1))
+        for _ in 1:(njobs - length(alive))
             put!(pool, nothing)
         end
     end
-    function run_test_phase(phase_tests, sem, shared_worker)
+    function run_test_phase(phase_tests, sem, shared_worker; force_recycle::Bool=false)
         # for serial phases, reserve one pool slot for the shared worker
         if !isnothing(shared_worker)
             shared_worker[] = take!(worker_pool)
@@ -1451,7 +1450,7 @@ function _runtests(mod::Module, args::ParsedArgs;
                                   # the worker has reached the max-rss limit, recycle it
                                   # so future tests start with a smaller working set
                                   Malt.stop(wrkr)
-                              elseif recycle_on_failure && anynonpass(result[])
+                              elseif (recycle_on_failure || force_recycle) && anynonpass(result[])
                                   # a failing test may have left the worker in a bad state
                                   # (e.g. a wedged GPU driver whose every later allocation
                                   # fails); recycle it so future tests get a fresh process
@@ -1517,11 +1516,12 @@ function _runtests(mod::Module, args::ParsedArgs;
             run_test_phase(phase_tests, sem, shared_worker)
 
             # parallel workers are not stopped while serial tests remain (tests_to_start > 0);
-            # drain before serial-after so only one worker is alive for the serial phase
+            # drain before serial-after so only one worker is alive for the serial phase.
+            # one is kept rather than none so we do not add a third addworker (ID_COUNTER).
             if isnothing(shared_worker) && i < length(phases)
                 next_tests, _, next_sw = phases[i+1]
                 if !isempty(next_tests) && !isnothing(next_sw)
-                    drain_pool_leaving_one_worker!(worker_pool, jobs)
+                    drain_pool_leaving_n_workers!(worker_pool, jobs, 1)
                 end
             end
         end
@@ -1539,7 +1539,13 @@ function _runtests(mod::Module, args::ParsedArgs;
                 shared_worker = serial_worker
                 filter!(r -> r.test ∉ retry_tests, results.value)
 
-                run_test_phase(retry_tests, sem, shared_worker)
+                # retries run on an otherwise-idle system: stop every worker left over
+                # from the previous phase, so the retry worker is spawned fresh below and
+                # no sibling process competes with it. `force_recycle` keeps it that way
+                # after each test that does not pass.
+                drain_pool_leaving_n_workers!(worker_pool, jobs, 0)
+
+                run_test_phase(retry_tests, sem, shared_worker; force_recycle=true)
             end
         end
     catch err
diff --git a/test/runtests.jl b/test/runtests.jl
index ee7062a..b52d4cc 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1576,7 +1576,13 @@ end
         @test contains(str, r"always_fails +\| +1 +1 ")
     end
 
-    @testset "retried test runs alone" begin
+    # `serial_position=:after` returns the live serial worker to the pool immediately
+    # before the retry phase, so it is the configuration where the "alone" invariant is
+    # easiest to break.
+    @testset "retried test runs alone (serial=$serial, $serial_position)" for
+            (serial, serial_position) in ((String[], :before),
+                                          (["pass3"], :before),
+                                          (["pass3"], :after))
         mktempdir() do dir
             # On its retry, the flaky test checks it is the only worker left alive.
             check_alone = quote
@@ -1597,6 +1603,8 @@ end
                 testsuite,
                 tests=["flaky", "pass1", "pass2", "pass3"],
                 init_code=:(include($(joinpath(@__DIR__, "utils.jl")))),
+                serial,
+                serial_position,
                 stdout=io,
                 stderr=io,
                 retries=1,
@@ -1607,6 +1615,31 @@ end
         end
     end
 
+    @testset "failing retry does not reuse its worker" begin
+        testsuite = Dict(
+            "failA" => :( @test false ),
+            "failB" => :( @test false ),
+        )
+        io = IOBuffer()
+        @test_throws Test.FallbackTestSetException begin
+            ParallelTestRunner._runtests(
+                ParallelTestRunner, parse_args(["--jobs=1"]);
+                testsuite,
+                tests=["failA", "failB"],
+                stdout=io,
+                stderr=io,
+                retries=1,
+            )
+        end
+        str = String(take!(io))
+        main, retry = split(str, "Retrying")
+        ids(s) = [m[1] for m in eachmatch(r"fail[AB] +\((\d+)\)", s)]
+        # the main run reuses one worker: `recycle_on_failure` is off by default
+        @test length(ids(main)) == 2 && allequal(ids(main))
+        # the retry round recycles after every non-pass, so each test gets its own worker
+        @test length(ids(retry)) == 2 && allunique(ids(retry))
+    end
+
     @testset "no retries by default" begin
         mktempdir() do dir
             testsuite = Dict("flaky" => flaky_test(joinpath(dir, "flaky")))

From ae28e708a9833ca5e1ac8b722b5b211bf4bfd923 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 10 Aug 2026 14:13:40 -0300
Subject: [PATCH 19/20] Remove unnecessary `interrupted`

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 src/ParallelTestRunner.jl | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 7dbbb3d..6fec835 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1353,7 +1353,6 @@ function _runtests(mod::Module, args::ParsedArgs;
     #
 
     tests_to_start = Threads.Atomic{Int}(length(tests))
-    interrupted = false
     # Stop every all-but-`n` workers in the pool.Only safe at a
     # phase boundary, where all `njobs` slots have been returned.
     function drain_pool_leaving_n_workers!(pool, njobs, n)
@@ -1374,7 +1373,9 @@ function _runtests(mod::Module, args::ParsedArgs;
             put!(pool, nothing)
         end
     end
-    function run_test_phase(phase_tests, sem, shared_worker; force_recycle::Bool=false)
+    # `retry_mode` forces worker recycling after every test and enables
+    # deletion of an old failed run of the test that just finished
+    function run_test_phase(phase_tests, sem, shared_worker; retry_mode::Bool=false)
         # for serial phases, reserve one pool slot for the shared worker
         if !isnothing(shared_worker)
             shared_worker[] = take!(worker_pool)
@@ -1436,7 +1437,13 @@ function _runtests(mod::Module, args::ParsedArgs;
                           end
                           test_t1 = time()
                           output = @lock wrkr.io String(take!(wrkr.io[]))
-                          @lock results push!(results[], (; test, result, output, test_t0, test_t1))
+                          # a retry drops the record of the attempt it re-runs only once it
+                          # has one to put in its place: dropping them up front would lose
+                          # them outright if the phase is interrupted
+                          @lock results begin
+                              retry_mode && filter!(r -> r.test != test, results[])
+                              push!(results[], (; test, result, output, test_t0, test_t1))
+                          end
 
                           # act on the results
                           if result isa AbstractTestRecord
@@ -1450,7 +1457,7 @@ function _runtests(mod::Module, args::ParsedArgs;
                                   # the worker has reached the max-rss limit, recycle it
                                   # so future tests start with a smaller working set
                                   Malt.stop(wrkr)
-                              elseif (recycle_on_failure || force_recycle) && anynonpass(result[])
+                              elseif (recycle_on_failure || retry_mode) && anynonpass(result[])
                                   # a failing test may have left the worker in a bad state
                                   # (e.g. a wedged GPU driver whose every later allocation
                                   # fails); recycle it so future tests get a fresh process
@@ -1506,7 +1513,7 @@ function _runtests(mod::Module, args::ParsedArgs;
     try
         phases = test_phases
 
-        potential_retries = retries > 0 && !interrupted && args.quickfail === nothing
+        potential_retries = retries > 0 && args.quickfail === nothing
 
         potential_retries && (io_ctx.nonpass_color[] = :yellow)
         for i in 1:length(phases)
@@ -1529,6 +1536,11 @@ function _runtests(mod::Module, args::ParsedArgs;
         # retries
         if potential_retries
             for i in 1:retries
+                # `stop_work()` may have been called from a worker task or the printer
+                # without any exception reaching the `catch` below, so we cannot assume we
+                # got here normally; there is no point retrying a run being torn down.
+                done[] && break
+
                 retries == i && (io_ctx.nonpass_color[] = :red)
                 retry_tests = [r.test for r in results.value
                                     if r.result isa Exception || anynonpass(r.result[])]
@@ -1537,19 +1549,17 @@ function _runtests(mod::Module, args::ParsedArgs;
                 put!(printer_channel, (:retry, length(retry_tests), i))
                 sem = Base.Semaphore(1)
                 shared_worker = serial_worker
-                filter!(r -> r.test ∉ retry_tests, results.value)
 
                 # retries run on an otherwise-idle system: stop every worker left over
                 # from the previous phase, so the retry worker is spawned fresh below and
-                # no sibling process competes with it. `force_recycle` keeps it that way
+                # no sibling process competes with it. `retry_mode` keeps it that way
                 # after each test that does not pass.
                 drain_pool_leaving_n_workers!(worker_pool, jobs, 0)
 
-                run_test_phase(retry_tests, sem, shared_worker; force_recycle=true)
+                run_test_phase(retry_tests, sem, shared_worker; retry_mode=true)
             end
         end
     catch err
-        interrupted = true
         if !(err isa InterruptException)
             println(io_ctx.stderr, "\nCaught an error, stopping...")
         end

From d6ac82ceb0493cdcf5bbcb3676376a664ec39ec3 Mon Sep 17 00:00:00 2001
From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com>
Date: Mon, 10 Aug 2026 14:45:35 -0300
Subject: [PATCH 20/20] Fix potential race in nonpass print colour

---
 src/ParallelTestRunner.jl | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl
index 6fec835..d67a945 100644
--- a/src/ParallelTestRunner.jl
+++ b/src/ParallelTestRunner.jl
@@ -1268,6 +1268,8 @@ function _runtests(mod::Module, args::ParsedArgs;
     # (:started, test_name, worker_id)
     # (:finished, test_name, worker_id, record)
     # (:crashed, test_name, worker_id, test_time)
+    # (:retry, tests_n, retry_n)
+    # (:nonpass_color, color)
     printer_channel = Channel{Tuple}(100)
 
     printer_task = @async begin
@@ -1317,6 +1319,12 @@ function _runtests(mod::Module, args::ParsedArgs;
                         finally
                             unlock(io_ctx.lock)
                         end
+
+                    elseif msg_type === :nonpass_color
+                        # routed through the channel rather than set directly so it lands
+                        # in order with the results it applies to: the coordinator flips it
+                        # while this task may still be draining the previous round
+                        io_ctx.nonpass_color[] = msg[2]
                     end
                 end
 
@@ -1515,7 +1523,7 @@ function _runtests(mod::Module, args::ParsedArgs;
 
         potential_retries = retries > 0 && args.quickfail === nothing
 
-        potential_retries && (io_ctx.nonpass_color[] = :yellow)
+        potential_retries && put!(printer_channel, (:nonpass_color, :yellow))
         for i in 1:length(phases)
             phase_tests, sem, shared_worker = phases[i]
             isempty(phase_tests) && continue
@@ -1541,11 +1549,12 @@ function _runtests(mod::Module, args::ParsedArgs;
                 # got here normally; there is no point retrying a run being torn down.
                 done[] && break
 
-                retries == i && (io_ctx.nonpass_color[] = :red)
                 retry_tests = [r.test for r in results.value
                                     if r.result isa Exception || anynonpass(r.result[])]
                 isempty(retry_tests) && break
 
+                # the last attempt of a test is the one that gets reported, so print it red
+                retries == i && put!(printer_channel, (:nonpass_color, :red))
                 put!(printer_channel, (:retry, length(retry_tests), i))
                 sem = Base.Semaphore(1)
                 shared_worker = serial_worker