From 5697586273e9a22a9a73ae717f44b38da8cfaf70 Mon Sep 17 00:00:00 2001
From: Tobias Pfeiffer <tobias.pfeiffer@supabase.io>
Date: Wed, 2 Sep 2026 18:08:20 +0200
Subject: [PATCH 1/4] fix: Test Manifest, parameterized do not override sibling
 failures

Fixes #15820

Alternative approach to #15825.

In short, parameterized tests aen't accounted for in the test manifest,
so if a variant first failed it could get overridden by a later run
with different parameters as the key is just {module, test}.

Based on [feedback](https://github.com/elixir-lang/elixir/pull/15825#issuecomment-5490847997),
this PR tries to solve this by keeping _all_ parameterized tests marked
as failing.
So for variants 1, 2, 3, 4 ran and only 2 failed, all 4 get rerun which isn't
too much of a cost.
For this small cost, we can keep the format of the manifest file,
as it was before.

There is one wrinkle to this, which is I couldn't find a good way
to stop the override ("do not delete a failing test from the manifest,
if I have the same id") while still ever deleting a failing test
(which we need, otherwise `mix test --failed` would be ever growing)
other than to track which tests failed _this run_. So, we only
refuse to clear it if we know the test failed _this run_.

The additional work to track is quite minimal though.

It adds an additional argument to `put_test` though, which I
decided to give a default to not increase the splash radius of the
PR. Worth potentially removing, unless we think it's fair game
for external callers.
---
 lib/ex_unit/lib/ex_unit/failures_manifest.ex  | 21 ++++++--
 lib/ex_unit/lib/ex_unit/runner_stats.ex       | 25 +++++++++-
 .../test/ex_unit/failures_manifest_test.exs   | 30 ++++++++++++
 .../test/ex_unit/runner_stats_test.exs        | 49 +++++++++++++++++++
 .../fixtures/test_failed_parameterize/mix.exs | 11 +++++
 .../test/parameterized_test_failed.exs        |  8 +++
 .../test/test_helper.exs                      |  1 +
 lib/mix/test/mix/tasks/test_test.exs          | 13 +++++
 8 files changed, 151 insertions(+), 7 deletions(-)
 create mode 100644 lib/mix/test/fixtures/test_failed_parameterize/mix.exs
 create mode 100644 lib/mix/test/fixtures/test_failed_parameterize/test/parameterized_test_failed.exs
 create mode 100644 lib/mix/test/fixtures/test_failed_parameterize/test/test_helper.exs

diff --git a/lib/ex_unit/lib/ex_unit/failures_manifest.ex b/lib/ex_unit/lib/ex_unit/failures_manifest.ex
index 239b63201fd..f88b65bcbc3 100644
--- a/lib/ex_unit/lib/ex_unit/failures_manifest.ex
+++ b/lib/ex_unit/lib/ex_unit/failures_manifest.ex
@@ -12,16 +12,27 @@ defmodule ExUnit.FailuresManifest do
   @spec new() :: t
   def new, do: %{}
 
-  @spec put_test(t, ExUnit.Test.t()) :: t
-  def put_test(%{} = manifest, %ExUnit.Test{state: {ignored_state, _}})
+  @spec put_test(t, ExUnit.Test.t(), MapSet.t(ExUnit.test_id())) :: t
+  def put_test(manifest, test, failed_this_run \\ MapSet.new())
+
+  def put_test(%{} = manifest, %ExUnit.Test{state: {ignored_state, _}}, _failed_this_run)
       when ignored_state in [:skipped, :excluded],
       do: manifest
 
-  def put_test(%{} = manifest, %ExUnit.Test{state: nil} = test) do
-    Map.delete(manifest, {test.module, test.name})
+  def put_test(%{} = manifest, %ExUnit.Test{state: nil} = test, failed_this_run) do
+    key = {test.module, test.name}
+
+    if MapSet.member?(failed_this_run, key) do
+      # We have an entry that this test failed earlier in the same run
+      # --> must have been a parameterized variant, in which case keep
+      # it as failed (See #15820)
+      manifest
+    else
+      Map.delete(manifest, key)
+    end
   end
 
-  def put_test(%{} = manifest, %ExUnit.Test{state: {failed_state, _}} = test)
+  def put_test(%{} = manifest, %ExUnit.Test{state: {failed_state, _}} = test, _failed_this_run)
       when failed_state in [:failed, :invalid] do
     Map.put(manifest, {test.module, test.name}, test.tags.file)
   end
diff --git a/lib/ex_unit/lib/ex_unit/runner_stats.ex b/lib/ex_unit/lib/ex_unit/runner_stats.ex
index 7c61f5fe7da..4bb5bbdfd2f 100644
--- a/lib/ex_unit/lib/ex_unit/runner_stats.ex
+++ b/lib/ex_unit/lib/ex_unit/runner_stats.ex
@@ -42,6 +42,12 @@ defmodule ExUnit.RunnerStats do
       excluded: 0,
       failures_manifest_path: opts[:failures_manifest_path],
       failures_manifest: FailuresManifest.new(),
+      # We need to keep track of the tests that specifically failed this run,
+      # so we can prevent a parameterized test variant from overriding another
+      # failing variant.
+      # If we just always didn't override it, we'd never get rid of fails.
+      # See `ExUnit.FailuresManifest.put_test/3`.
+      failed_this_run: MapSet.new(),
       failure_counter: 0,
       pids: []
     }
@@ -64,9 +70,14 @@ defmodule ExUnit.RunnerStats do
   end
 
   def handle_cast({:test_finished, %Test{} = test}, state) do
+    state = track_failed_this_run(state, test)
+
     state =
       state
-      |> Map.update!(:failures_manifest, &FailuresManifest.put_test(&1, test))
+      |> Map.update!(
+        :failures_manifest,
+        &FailuresManifest.put_test(&1, test, state.failed_this_run)
+      )
       |> Map.update!(:total, &(&1 + 1))
       |> increment_status_counter(test.state)
 
@@ -96,7 +107,10 @@ defmodule ExUnit.RunnerStats do
     state =
       Enum.reduce(successful_tests, state, fn test, acc ->
         acc
-        |> Map.update!(:failures_manifest, &FailuresManifest.put_test(&1, test))
+        |> Map.update!(
+          :failures_manifest,
+          &FailuresManifest.put_test(&1, test, acc.failed_this_run)
+        )
         |> Map.update!(:failures, &(&1 + 1))
         |> Map.update!(:passed, &(&1 - 1))
       end)
@@ -108,6 +122,13 @@ defmodule ExUnit.RunnerStats do
     {:noreply, state}
   end
 
+  defp track_failed_this_run(state, %Test{state: {failed_state, _}} = test)
+       when failed_state in [:failed, :invalid] do
+    Map.update!(state, :failed_this_run, &MapSet.put(&1, {test.module, test.name}))
+  end
+
+  defp track_failed_this_run(state, %Test{}), do: state
+
   defp increment_status_counter(state, tag) when tag in [nil, :passed] do
     Map.update!(state, :passed, &(&1 + 1))
   end
diff --git a/lib/ex_unit/test/ex_unit/failures_manifest_test.exs b/lib/ex_unit/test/ex_unit/failures_manifest_test.exs
index 219f723d36a..e17632aeee4 100644
--- a/lib/ex_unit/test/ex_unit/failures_manifest_test.exs
+++ b/lib/ex_unit/test/ex_unit/failures_manifest_test.exs
@@ -108,6 +108,36 @@ defmodule ExUnit.FailuresManifestTest do
     end
   end
 
+  describe "put_test/3 with a failed_this_run set" do
+    setup do
+      failed_test = new_test(@failed)
+      manifest = put_test(new(), failed_test)
+      {:ok, %{failed_test: failed_test, manifest: manifest}}
+    end
+
+    test "keeps a newly passed test in the manifest when its id is in failed_this_run",
+         context do
+      test = %{context.failed_test | state: @passed}
+      failed_this_run = MapSet.new([test_id(test)])
+
+      assert put_test(context.manifest, test, failed_this_run) == context.manifest
+    end
+
+    test "removes a newly passed test when its id is not in failed_this_run", context do
+      test = %{context.failed_test | state: @passed}
+
+      assert put_test(context.manifest, test, MapSet.new()) == new()
+    end
+
+    test "stores a failed test regardless of failed_this_run", context do
+      expected_manifest = %{test_id(context.failed_test) => file(context.failed_test)}
+
+      failed_this_run = MapSet.new([test_id(context.failed_test)])
+      assert put_test(new(), context.failed_test, failed_this_run) == expected_manifest
+      assert put_test(new(), context.failed_test, MapSet.new()) == expected_manifest
+    end
+  end
+
   describe "write!/2" do
     @tag :tmp_dir
     test "stores a manifest that can later be read with read/1", context do
diff --git a/lib/ex_unit/test/ex_unit/runner_stats_test.exs b/lib/ex_unit/test/ex_unit/runner_stats_test.exs
index 05db0642eea..d1114758e3c 100644
--- a/lib/ex_unit/test/ex_unit/runner_stats_test.exs
+++ b/lib/ex_unit/test/ex_unit/runner_stats_test.exs
@@ -115,6 +115,55 @@ defmodule ExUnit.RunnerStatsTest do
     end
   end
 
+  describe "parameterized test variants sharing a test id (#15820)" do
+    @tag :tmp_dir
+    test "keeps the manifest entry when a failing variant runs before a passing sibling", %{
+      tmp_dir: tmp_dir
+    } do
+      File.cd!(tmp_dir, fn ->
+        simulate_suite(fn formatter ->
+          simulate_test(formatter, :test_1, :failed)
+          simulate_test(formatter, :test_1, :passed)
+        end)
+
+        assert read_failures_manifest() == %{{TestModule, :test_1} => __ENV__.file}
+      end)
+    end
+
+    @tag :tmp_dir
+    test "keeps the manifest entry when a failing variant runs after a passing sibling", %{
+      tmp_dir: tmp_dir
+    } do
+      File.cd!(tmp_dir, fn ->
+        simulate_suite(fn formatter ->
+          simulate_test(formatter, :test_1, :passed)
+          simulate_test(formatter, :test_1, :failed)
+        end)
+
+        assert read_failures_manifest() == %{{TestModule, :test_1} => __ENV__.file}
+      end)
+    end
+
+    @tag :tmp_dir
+    test "clears the manifest entry once every variant passes in a later run", %{
+      tmp_dir: tmp_dir
+    } do
+      File.cd!(tmp_dir, fn ->
+        simulate_suite(fn formatter ->
+          simulate_test(formatter, :test_1, :failed)
+          simulate_test(formatter, :test_1, :passed)
+        end)
+
+        simulate_suite(fn formatter ->
+          simulate_test(formatter, :test_1, :passed)
+          simulate_test(formatter, :test_1, :passed)
+        end)
+
+        assert read_failures_manifest() == %{}
+      end)
+    end
+  end
+
   defp simulate_suite(opts \\ [failures_manifest_path: @failures_manifest_path], fun) do
     {:ok, pid} = GenServer.start_link(RunnerStats, opts)
     GenServer.cast(pid, {:suite_started, opts})
diff --git a/lib/mix/test/fixtures/test_failed_parameterize/mix.exs b/lib/mix/test/fixtures/test_failed_parameterize/mix.exs
new file mode 100644
index 00000000000..18e0e8e1ab4
--- /dev/null
+++ b/lib/mix/test/fixtures/test_failed_parameterize/mix.exs
@@ -0,0 +1,11 @@
+defmodule TestFailedParameterize.MixProject do
+  use Mix.Project
+
+  def project do
+    [
+      app: :test_failed_parameterize,
+      version: "0.0.1",
+      test_load_filters: [~r/.*_test_failed\.exs/]
+    ]
+  end
+end
diff --git a/lib/mix/test/fixtures/test_failed_parameterize/test/parameterized_test_failed.exs b/lib/mix/test/fixtures/test_failed_parameterize/test/parameterized_test_failed.exs
new file mode 100644
index 00000000000..66bda88c3a6
--- /dev/null
+++ b/lib/mix/test/fixtures/test_failed_parameterize/test/parameterized_test_failed.exs
@@ -0,0 +1,8 @@
+defmodule ParameterizedTest do
+  use ExUnit.Case,
+    parameterize: [%{value: :a}, %{value: :b}]
+
+  test "checks value", %{value: value} do
+    assert value == :b
+  end
+end
diff --git a/lib/mix/test/fixtures/test_failed_parameterize/test/test_helper.exs b/lib/mix/test/fixtures/test_failed_parameterize/test/test_helper.exs
new file mode 100644
index 00000000000..869559e709e
--- /dev/null
+++ b/lib/mix/test/fixtures/test_failed_parameterize/test/test_helper.exs
@@ -0,0 +1 @@
+ExUnit.start()
diff --git a/lib/mix/test/mix/tasks/test_test.exs b/lib/mix/test/mix/tasks/test_test.exs
index 29bebd7f145..865e327bb85 100644
--- a/lib/mix/test/mix/tasks/test_test.exs
+++ b/lib/mix/test/mix/tasks/test_test.exs
@@ -285,6 +285,19 @@ defmodule Mix.Tasks.TestTest do
         assert output =~ "** (RuntimeError) oops"
       end)
     end
+
+    test "keeps a failing parameterized variant in the manifest even when a passing sibling variant runs after it" do
+      in_fixture("test_failed_parameterize", fn ->
+        # The `:a` variant fails and the `:b` variant passes. Whichever order
+        # they run in, `:a` must stay in the manifest and be retried. See #15820.
+        output = mix(["test"])
+        assert output =~ "Failed: 1 test"
+
+        output = mix(["test", "--failed"])
+        refute output =~ "There are no tests to run"
+        assert output =~ "Failed: 1 test"
+      end)
+    end
   end
 
   describe "--listen-on-stdin" do

From d93fedee4ec6fda2f24ecdcd7429b2593b6eb321 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Valim?= <jose.valim@gmail.com>
Date: Wed, 2 Sep 2026 20:57:06 +0200
Subject: [PATCH 2/4] Keep logic within failures manifest

---
 lib/ex_unit/lib/ex_unit/failures_manifest.ex  |  49 ++++----
 lib/ex_unit/lib/ex_unit/runner_stats.ex       |  33 +-----
 .../test/ex_unit/failures_manifest_test.exs   | 108 ++++++++----------
 3 files changed, 76 insertions(+), 114 deletions(-)

diff --git a/lib/ex_unit/lib/ex_unit/failures_manifest.ex b/lib/ex_unit/lib/ex_unit/failures_manifest.ex
index f88b65bcbc3..63f0e4e59f3 100644
--- a/lib/ex_unit/lib/ex_unit/failures_manifest.ex
+++ b/lib/ex_unit/lib/ex_unit/failures_manifest.ex
@@ -5,41 +5,42 @@
 defmodule ExUnit.FailuresManifest do
   @moduledoc false
 
-  @opaque t :: %{optional(ExUnit.test_id()) => test_file :: Path.t()}
+  @opaque t :: {passed, failed}
+
+  @typep passed :: [ExUnit.test_id()]
+  @typep failed :: %{optional(ExUnit.test_id()) => test_file :: Path.t()}
 
   @manifest_vsn 1
 
   @spec new() :: t
-  def new, do: %{}
-
-  @spec put_test(t, ExUnit.Test.t(), MapSet.t(ExUnit.test_id())) :: t
-  def put_test(manifest, test, failed_this_run \\ MapSet.new())
+  def new, do: {[], %{}}
 
-  def put_test(%{} = manifest, %ExUnit.Test{state: {ignored_state, _}}, _failed_this_run)
+  @spec put_test(t, ExUnit.Test.t()) :: t
+  def put_test({_passed, _failed} = manifest, %ExUnit.Test{state: {ignored_state, _}})
       when ignored_state in [:skipped, :excluded],
       do: manifest
 
-  def put_test(%{} = manifest, %ExUnit.Test{state: nil} = test, failed_this_run) do
-    key = {test.module, test.name}
-
-    if MapSet.member?(failed_this_run, key) do
-      # We have an entry that this test failed earlier in the same run
-      # --> must have been a parameterized variant, in which case keep
-      # it as failed (See #15820)
-      manifest
-    else
-      Map.delete(manifest, key)
-    end
+  def put_test({passed, failed}, %ExUnit.Test{state: nil} = test) do
+    test_id = {test.module, test.name}
+    {[test_id | passed], failed}
   end
 
-  def put_test(%{} = manifest, %ExUnit.Test{state: {failed_state, _}} = test, _failed_this_run)
+  def put_test({passed, failed}, %ExUnit.Test{state: {failed_state, _}} = test)
       when failed_state in [:failed, :invalid] do
-    Map.put(manifest, {test.module, test.name}, test.tags.file)
+    test_id = {test.module, test.name}
+
+    {passed, Map.put(failed, test_id, test.tags.file)}
   end
 
-  @spec write!(t, Path.t()) :: :ok
-  def write!(manifest, file) when is_binary(file) do
-    manifest = prune_deleted_tests(manifest)
+  @spec update!(t, Path.t()) :: :ok
+  def update!({passed, failed}, file) when is_binary(file) do
+    manifest =
+      file
+      |> read()
+      |> prune_deleted_tests()
+      |> Map.drop(passed)
+      |> Map.merge(failed)
+
     binary = :erlang.term_to_binary({@manifest_vsn, manifest})
     Path.dirname(file) |> File.mkdir_p!()
     File.write!(file, binary)
@@ -52,13 +53,13 @@ defmodule ExUnit.FailuresManifest do
     File.write!(file, binary)
   end
 
-  @spec read(Path.t()) :: t
+  @spec read(Path.t()) :: failed
   def read(file) when is_binary(file) do
     with {:ok, binary} <- File.read(file),
          {:ok, {@manifest_vsn, %{} = manifest}} <- safe_binary_to_term(binary) do
       manifest
     else
-      _ -> new()
+      _ -> %{}
     end
   end
 
diff --git a/lib/ex_unit/lib/ex_unit/runner_stats.ex b/lib/ex_unit/lib/ex_unit/runner_stats.ex
index 4bb5bbdfd2f..4a8de3222a1 100644
--- a/lib/ex_unit/lib/ex_unit/runner_stats.ex
+++ b/lib/ex_unit/lib/ex_unit/runner_stats.ex
@@ -42,12 +42,6 @@ defmodule ExUnit.RunnerStats do
       excluded: 0,
       failures_manifest_path: opts[:failures_manifest_path],
       failures_manifest: FailuresManifest.new(),
-      # We need to keep track of the tests that specifically failed this run,
-      # so we can prevent a parameterized test variant from overriding another
-      # failing variant.
-      # If we just always didn't override it, we'd never get rid of fails.
-      # See `ExUnit.FailuresManifest.put_test/3`.
-      failed_this_run: MapSet.new(),
       failure_counter: 0,
       pids: []
     }
@@ -70,29 +64,18 @@ defmodule ExUnit.RunnerStats do
   end
 
   def handle_cast({:test_finished, %Test{} = test}, state) do
-    state = track_failed_this_run(state, test)
-
     state =
       state
-      |> Map.update!(
-        :failures_manifest,
-        &FailuresManifest.put_test(&1, test, state.failed_this_run)
-      )
+      |> Map.update!(:failures_manifest, &FailuresManifest.put_test(&1, test))
       |> Map.update!(:total, &(&1 + 1))
       |> increment_status_counter(test.state)
 
     {:noreply, state}
   end
 
-  def handle_cast({:suite_started, _opts}, %{failures_manifest_path: file} = state)
-      when is_binary(file) do
-    state = %{state | failures_manifest: FailuresManifest.read(file)}
-    {:noreply, state}
-  end
-
   def handle_cast({:suite_finished, _}, %{failures_manifest_path: file} = state)
       when is_binary(file) do
-    FailuresManifest.write!(state.failures_manifest, file)
+    FailuresManifest.update!(state.failures_manifest, file)
     {:noreply, state}
   end
 
@@ -107,10 +90,7 @@ defmodule ExUnit.RunnerStats do
     state =
       Enum.reduce(successful_tests, state, fn test, acc ->
         acc
-        |> Map.update!(
-          :failures_manifest,
-          &FailuresManifest.put_test(&1, test, acc.failed_this_run)
-        )
+        |> Map.update!(:failures_manifest, &FailuresManifest.put_test(&1, test))
         |> Map.update!(:failures, &(&1 + 1))
         |> Map.update!(:passed, &(&1 - 1))
       end)
@@ -122,13 +102,6 @@ defmodule ExUnit.RunnerStats do
     {:noreply, state}
   end
 
-  defp track_failed_this_run(state, %Test{state: {failed_state, _}} = test)
-       when failed_state in [:failed, :invalid] do
-    Map.update!(state, :failed_this_run, &MapSet.put(&1, {test.module, test.name}))
-  end
-
-  defp track_failed_this_run(state, %Test{}), do: state
-
   defp increment_status_counter(state, tag) when tag in [nil, :passed] do
     Map.update!(state, :passed, &(&1 + 1))
   end
diff --git a/lib/ex_unit/test/ex_unit/failures_manifest_test.exs b/lib/ex_unit/test/ex_unit/failures_manifest_test.exs
index e17632aeee4..2a01ac6a3e0 100644
--- a/lib/ex_unit/test/ex_unit/failures_manifest_test.exs
+++ b/lib/ex_unit/test/ex_unit/failures_manifest_test.exs
@@ -27,7 +27,7 @@ defmodule ExUnit.FailuresManifestTest do
         |> put_test(invalid_1 = new_test(@invalid, context))
 
       File.cd!(context.tmp_dir, fn ->
-        write!(manifest, @manifest_path)
+        update!(manifest, @manifest_path)
 
         assert info(@manifest_path) ==
                  {MapSet.new([context.file]),
@@ -52,18 +52,19 @@ defmodule ExUnit.FailuresManifestTest do
   end
 
   describe "put_test/2 when the test is not already in the manifest" do
-    test "ignores passed tests since we only care to store failures" do
-      assert put_test(new(), new_test(@passed)) == new()
+    test "records passed tests so they clear failures from a prior run" do
+      test = new_test(@passed)
+      assert put_test(new(), test) == {[test_id(test)], %{}}
     end
 
     test "stores failed tests" do
       test = new_test(@failed)
-      assert put_test(new(), test) == %{test_id(test) => file(test)}
+      assert put_test(new(), test) == {[], %{test_id(test) => file(test)}}
     end
 
     test "stores invalid tests" do
       test = new_test(@invalid)
-      assert put_test(new(), test) == %{test_id(test) => file(test)}
+      assert put_test(new(), test) == {[], %{test_id(test) => file(test)}}
     end
 
     test "ignores skipped tests since we know nothing about their pass/fail status" do
@@ -82,19 +83,21 @@ defmodule ExUnit.FailuresManifestTest do
       {:ok, %{failed_test: failed_test, manifest: manifest}}
     end
 
-    test "removes a newly passed test, since it is no longer failing", context do
+    test "records a passing parameterized sibling", context do
       test = %{context.failed_test | state: @passed}
-      assert put_test(context.manifest, test) == new()
+      assert put_test(context.manifest, test) == {[test_id(test)], elem(context.manifest, 1)}
     end
 
     test "stores failed tests, updating the stored file value", context do
       test = %{context.failed_test | tags: %{file: "some-other-file"}}
-      assert put_test(context.manifest, test) == %{test_id(test) => file(test)}
+
+      assert put_test(context.manifest, test) == {[], %{test_id(test) => file(test)}}
     end
 
     test "stores invalid tests, updating the stored file value", context do
       test = %{context.failed_test | tags: %{file: "some-other-file"}, state: @invalid}
-      assert put_test(context.manifest, test) == %{test_id(test) => file(test)}
+
+      assert put_test(context.manifest, test) == {[], %{test_id(test) => file(test)}}
     end
 
     test "ignores skipped tests since we know nothing about their pass/fail status", context do
@@ -108,57 +111,40 @@ defmodule ExUnit.FailuresManifestTest do
     end
   end
 
-  describe "put_test/3 with a failed_this_run set" do
-    setup do
-      failed_test = new_test(@failed)
-      manifest = put_test(new(), failed_test)
-      {:ok, %{failed_test: failed_test, manifest: manifest}}
-    end
-
-    test "keeps a newly passed test in the manifest when its id is in failed_this_run",
-         context do
-      test = %{context.failed_test | state: @passed}
-      failed_this_run = MapSet.new([test_id(test)])
-
-      assert put_test(context.manifest, test, failed_this_run) == context.manifest
-    end
-
-    test "removes a newly passed test when its id is not in failed_this_run", context do
-      test = %{context.failed_test | state: @passed}
-
-      assert put_test(context.manifest, test, MapSet.new()) == new()
-    end
-
-    test "stores a failed test regardless of failed_this_run", context do
-      expected_manifest = %{test_id(context.failed_test) => file(context.failed_test)}
+  describe "update!/2" do
+    @tag :tmp_dir
+    test "stores a manifest that can later be read with read/1", context do
+      manifest = non_blank_manifest(context)
 
-      failed_this_run = MapSet.new([test_id(context.failed_test)])
-      assert put_test(new(), context.failed_test, failed_this_run) == expected_manifest
-      assert put_test(new(), context.failed_test, MapSet.new()) == expected_manifest
+      File.cd!(context.tmp_dir, fn ->
+        assert update!(manifest, @manifest_path) == :ok
+        assert read(@manifest_path) == elem(manifest, 1)
+      end)
     end
-  end
 
-  describe "write!/2" do
     @tag :tmp_dir
-    test "stores a manifest that can later be read with read/1", context do
-      manifest = non_blank_manifest(context)
+    test "merges the results from this run with the prior manifest", context do
+      failed_test = new_test(@failed, context)
+      passed_test = %{failed_test | state: @passed}
 
       File.cd!(context.tmp_dir, fn ->
-        assert write!(manifest, @manifest_path) == :ok
-        assert read(@manifest_path) == manifest
+        assert update!(put_test(new(), failed_test), @manifest_path) == :ok
+        assert update!(put_test(new(), passed_test), @manifest_path) == :ok
+        assert read(@manifest_path) == %{}
       end)
     end
 
     @tag :tmp_dir
-    test "prunes tests from files that no longer exist", context do
+    test "prunes tests from files that no longer exist in the prior manifest", context do
       test = new_test(@failed, %{context | file: "missing_file.exs"})
 
       File.cd!(context.tmp_dir, fn ->
-        new()
-        |> put_test(test)
-        |> write!(@manifest_path)
+        binary = :erlang.term_to_binary({1, %{test_id(test) => file(test)}})
+        File.write!(@manifest_path, binary)
 
-        assert read(@manifest_path) == new()
+        update!(new(), @manifest_path)
+
+        assert read(@manifest_path) == %{}
       end)
     end
 
@@ -168,21 +154,22 @@ defmodule ExUnit.FailuresManifestTest do
       manifest = new() |> put_test(test)
 
       File.cd!(context.tmp_dir, fn ->
-        write!(manifest, @manifest_path)
-        assert read(@manifest_path) == manifest
+        update!(manifest, @manifest_path)
+        assert read(@manifest_path) == elem(manifest, 1)
       end)
     end
 
     @tag :tmp_dir
-    test "prunes tests defined in a function that no longer exists", context do
+    test "prunes tests from functions that no longer exist in the prior manifest", context do
       test = new_test(@failed, %{context | test: :not_a_function_anymore})
 
       File.cd!(context.tmp_dir, fn ->
-        new()
-        |> put_test(test)
-        |> write!(@manifest_path)
+        binary = :erlang.term_to_binary({1, %{test_id(test) => file(test)}})
+        File.write!(@manifest_path, binary)
+
+        update!(new(), @manifest_path)
 
-        assert read(@manifest_path) == new()
+        assert read(@manifest_path) == %{}
       end)
     end
   end
@@ -192,7 +179,7 @@ defmodule ExUnit.FailuresManifestTest do
     test "returns a blank manifest when loading a file that does not exit", context do
       path = Path.join(context.tmp_dir, "missing.manifest")
       refute File.exists?(path)
-      assert read(path) == new()
+      assert read(path) == %{}
     end
 
     @tag :tmp_dir
@@ -200,10 +187,10 @@ defmodule ExUnit.FailuresManifestTest do
       manifest = non_blank_manifest(context)
 
       File.cd!(context.tmp_dir, fn ->
-        assert write!(manifest, @manifest_path) == :ok
+        assert update!(manifest, @manifest_path) == :ok
         corrupted = "corrupted" <> File.read!(@manifest_path)
         File.write!(@manifest_path, corrupted)
-        assert read(@manifest_path) == new()
+        assert read(@manifest_path) == %{}
       end)
     end
 
@@ -212,10 +199,11 @@ defmodule ExUnit.FailuresManifestTest do
       manifest = non_blank_manifest(context)
 
       File.cd!(context.tmp_dir, fn ->
-        assert write!(manifest, @manifest_path) == :ok
-        assert {vsn, ^manifest} = @manifest_path |> File.read!() |> :erlang.binary_to_term()
-        File.write!(@manifest_path, :erlang.term_to_binary({vsn + 1, manifest}))
-        assert read(@manifest_path) == new()
+        assert update!(manifest, @manifest_path) == :ok
+        assert {vsn, failures} = @manifest_path |> File.read!() |> :erlang.binary_to_term()
+        assert failures == elem(manifest, 1)
+        File.write!(@manifest_path, :erlang.term_to_binary({vsn + 1, failures}))
+        assert read(@manifest_path) == %{}
       end)
     end
   end

From 3dafb18cef06796252b93f3c705f81a5827377fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Valim?= <jose.valim@gmail.com>
Date: Wed, 2 Sep 2026 20:58:29 +0200
Subject: [PATCH 3/4] Remove unneeded integration test

---
 lib/ex_unit/lib/ex_unit/failures_manifest.ex        |  1 -
 .../test/fixtures/test_failed_parameterize/mix.exs  | 11 -----------
 .../test/parameterized_test_failed.exs              |  8 --------
 .../test_failed_parameterize/test/test_helper.exs   |  1 -
 lib/mix/test/mix/tasks/test_test.exs                | 13 -------------
 5 files changed, 34 deletions(-)
 delete mode 100644 lib/mix/test/fixtures/test_failed_parameterize/mix.exs
 delete mode 100644 lib/mix/test/fixtures/test_failed_parameterize/test/parameterized_test_failed.exs
 delete mode 100644 lib/mix/test/fixtures/test_failed_parameterize/test/test_helper.exs

diff --git a/lib/ex_unit/lib/ex_unit/failures_manifest.ex b/lib/ex_unit/lib/ex_unit/failures_manifest.ex
index 63f0e4e59f3..7d2af1c2cb3 100644
--- a/lib/ex_unit/lib/ex_unit/failures_manifest.ex
+++ b/lib/ex_unit/lib/ex_unit/failures_manifest.ex
@@ -28,7 +28,6 @@ defmodule ExUnit.FailuresManifest do
   def put_test({passed, failed}, %ExUnit.Test{state: {failed_state, _}} = test)
       when failed_state in [:failed, :invalid] do
     test_id = {test.module, test.name}
-
     {passed, Map.put(failed, test_id, test.tags.file)}
   end
 
diff --git a/lib/mix/test/fixtures/test_failed_parameterize/mix.exs b/lib/mix/test/fixtures/test_failed_parameterize/mix.exs
deleted file mode 100644
index 18e0e8e1ab4..00000000000
--- a/lib/mix/test/fixtures/test_failed_parameterize/mix.exs
+++ /dev/null
@@ -1,11 +0,0 @@
-defmodule TestFailedParameterize.MixProject do
-  use Mix.Project
-
-  def project do
-    [
-      app: :test_failed_parameterize,
-      version: "0.0.1",
-      test_load_filters: [~r/.*_test_failed\.exs/]
-    ]
-  end
-end
diff --git a/lib/mix/test/fixtures/test_failed_parameterize/test/parameterized_test_failed.exs b/lib/mix/test/fixtures/test_failed_parameterize/test/parameterized_test_failed.exs
deleted file mode 100644
index 66bda88c3a6..00000000000
--- a/lib/mix/test/fixtures/test_failed_parameterize/test/parameterized_test_failed.exs
+++ /dev/null
@@ -1,8 +0,0 @@
-defmodule ParameterizedTest do
-  use ExUnit.Case,
-    parameterize: [%{value: :a}, %{value: :b}]
-
-  test "checks value", %{value: value} do
-    assert value == :b
-  end
-end
diff --git a/lib/mix/test/fixtures/test_failed_parameterize/test/test_helper.exs b/lib/mix/test/fixtures/test_failed_parameterize/test/test_helper.exs
deleted file mode 100644
index 869559e709e..00000000000
--- a/lib/mix/test/fixtures/test_failed_parameterize/test/test_helper.exs
+++ /dev/null
@@ -1 +0,0 @@
-ExUnit.start()
diff --git a/lib/mix/test/mix/tasks/test_test.exs b/lib/mix/test/mix/tasks/test_test.exs
index 865e327bb85..29bebd7f145 100644
--- a/lib/mix/test/mix/tasks/test_test.exs
+++ b/lib/mix/test/mix/tasks/test_test.exs
@@ -285,19 +285,6 @@ defmodule Mix.Tasks.TestTest do
         assert output =~ "** (RuntimeError) oops"
       end)
     end
-
-    test "keeps a failing parameterized variant in the manifest even when a passing sibling variant runs after it" do
-      in_fixture("test_failed_parameterize", fn ->
-        # The `:a` variant fails and the `:b` variant passes. Whichever order
-        # they run in, `:a` must stay in the manifest and be retried. See #15820.
-        output = mix(["test"])
-        assert output =~ "Failed: 1 test"
-
-        output = mix(["test", "--failed"])
-        refute output =~ "There are no tests to run"
-        assert output =~ "Failed: 1 test"
-      end)
-    end
   end
 
   describe "--listen-on-stdin" do

From 0a2a94b81572697b6a50dee238b45779c70db058 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jos=C3=A9=20Valim?= <jose.valim@gmail.com>
Date: Wed, 2 Sep 2026 21:03:54 +0200
Subject: [PATCH 4/4] Update test name

---
 lib/ex_unit/test/ex_unit/failures_manifest_test.exs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/lib/ex_unit/test/ex_unit/failures_manifest_test.exs b/lib/ex_unit/test/ex_unit/failures_manifest_test.exs
index 2a01ac6a3e0..569e5d27d03 100644
--- a/lib/ex_unit/test/ex_unit/failures_manifest_test.exs
+++ b/lib/ex_unit/test/ex_unit/failures_manifest_test.exs
@@ -83,20 +83,18 @@ defmodule ExUnit.FailuresManifestTest do
       {:ok, %{failed_test: failed_test, manifest: manifest}}
     end
 
-    test "records a passing parameterized sibling", context do
+    test "stores passing test separately", context do
       test = %{context.failed_test | state: @passed}
-      assert put_test(context.manifest, test) == {[test_id(test)], elem(context.manifest, 1)}
+      assert put_test(context.manifest, test) == {[test_id(test)], %{test_id(test) => "file"}}
     end
 
     test "stores failed tests, updating the stored file value", context do
       test = %{context.failed_test | tags: %{file: "some-other-file"}}
-
       assert put_test(context.manifest, test) == {[], %{test_id(test) => file(test)}}
     end
 
     test "stores invalid tests, updating the stored file value", context do
       test = %{context.failed_test | tags: %{file: "some-other-file"}, state: @invalid}
-
       assert put_test(context.manifest, test) == {[], %{test_id(test) => file(test)}}
     end