From 8a7245a2b3912c2a91adbc7b04b1f26e68733e39 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 27 Oct 2026 21:31:08 -0700 Subject: [PATCH 1/2] fix(runner): accept earlyoom's SIGKILL in the memory-hog subtest earlyoom sends SIGKILL instead of SIGTERM when its poll first lands below the 5% kill threshold, so the subtest's SIGTERM grep failed whenever the hog filled memory faster than earlyoom polled. Assert that earlyoom killed the hog with either signal, and print full build logs from the flake check so a failing VM subtest names itself. Co-Authored-By: Claude Opus 5.5 --- nix/runner/test.nix | 4 +++- sh/dispatch-query.test.py | 14 +++++++------- sh/dispatch.sh | 4 +++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/nix/runner/test.nix b/nix/runner/test.nix index cd74e6bfd53..84e338830b3 100644 --- a/nix/runner/test.nix +++ b/nix/runner/test.nix @@ -296,7 +296,9 @@ pkgs.testers.runNixOSTest { # re-reads its inputs from disk until the job times out. assert elapsed < 60, "the guest thrashed instead of killing memory-hog" time.sleep(1) - print(guest.succeed("journalctl -u earlyoom | grep -F 'sending SIGTERM to process'")) + # earlyoom sends SIGKILL instead of SIGTERM when its poll first lands + # below the 5% kill threshold, which depends on how fast the hog fills. + print(guest.succeed("journalctl -u earlyoom | grep -E 'sending SIG(TERM|KILL) to process .*${./hog.py}'")) guest.succeed("rm /var/tmp/hog-inputs") router.wait_until_succeeds("${pkgs.openssh}/bin/ssh-keyscan -t ed25519 -T 5 10.231.0.2 | grep -F ssh-ed25519", timeout=timedelta(minutes=1)) diff --git a/sh/dispatch-query.test.py b/sh/dispatch-query.test.py index 0dab2a9a3b6..b846bc0756a 100644 --- a/sh/dispatch-query.test.py +++ b/sh/dispatch-query.test.py @@ -279,7 +279,7 @@ def test_local_flake_namespace_refusal_is_reported_as_skipped(self): self.assertIn("local Nix builder cannot create user namespaces", result.stdout + result.stderr) self.assertRegex(result.stderr, r"\d+ check flake \(skipped\)") calls = (self.root / "calls").read_text().splitlines() - self.assertNotIn("nix flake check", calls) + self.assertNotIn("nix flake check --print-build-logs", calls) self.assertIn("gh test", calls) probe, = [call for call in calls if call.startswith("nix build ")] self.assertIn("--builders --max-jobs 1", probe) @@ -297,7 +297,7 @@ def test_local_capable_builder_runs_the_full_flake(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertNotIn("(skipped)", result.stderr) calls = (self.root / "calls").read_text().splitlines() - self.assertIn("nix flake check", calls) + self.assertIn("nix flake check --print-build-logs", calls) self.assertEqual(sum(call.startswith("nix build ") for call in calls), 1) @unittest.skipUnless(sys.platform == "linux" and os.uname().machine == "x86_64", "Linux VM checks only") @@ -305,7 +305,7 @@ def test_local_flake_check_exiting_77_fails_instead_of_skipping(self): self.stub_ci() self.change("flake.nix") nix = self.bin / "nix" - nix.write_text(nix.read_text() + 'case "$*" in "flake check") exit 77 ;; esac\n') + nix.write_text(nix.read_text() + 'case "$*" in "flake check --print-build-logs") exit 77 ;; esac\n') self.env.pop("CI_STRICT") result = subprocess.run(["bash", "sh/dispatch.sh", "check", "origin/main"], cwd=self.repo, env=self.env, text=True, capture_output=True) @@ -326,13 +326,13 @@ def test_local_flake_probe_errors_are_failures_without_cancelling_other_modules( self.assertIn("check: check flake failed", result.stderr) self.assertNotIn("(skipped)", result.stderr) calls = (self.root / "calls").read_text().splitlines() - self.assertNotIn("nix flake check", calls) + self.assertNotIn("nix flake check --print-build-logs", calls) self.assertIn("gh test", calls) def test_strict_ci_runs_the_full_flake_without_a_probe(self): self.change("flake.nix") calls = self.run_ci() - self.assertIn("nix flake check", calls) + self.assertIn("nix flake check --print-build-logs", calls) self.assertFalse(any(call.startswith("nix build ") for call in calls)) nix = self.bin / "nix" nix.write_text(nix.read_text() + 'exit 77\n') @@ -392,8 +392,8 @@ def test_flake_checks_follow_nix_inputs_instead_of_rust_source(self): self.change(path) calls = self.run_ci() (self.repo / path).unlink() - self.assertEqual("nix flake check" in calls, expected) - self.assertIn("nix flake check", self.run_ci("--all")) + self.assertEqual("nix flake check --print-build-logs" in calls, expected) + self.assertIn("nix flake check --print-build-logs", self.run_ci("--all")) def benchmark(): diff --git a/sh/dispatch.sh b/sh/dispatch.sh index c08d42d6629..f71c815df4f 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -312,7 +312,9 @@ step() { fi fi # 77 marks the probe's skip, so the check's own 77 is a failure. - bash sh/local.sh nix flake check || { + # Full build logs name a failing VM subtest; Nix otherwise keeps + # only the last 25 lines, which are QEMU's cleanup. + bash sh/local.sh nix flake check --print-build-logs || { local status=$? [[ -n "${CI_STRICT:-}" ]] || ((status != 77)) || status=1 return "$status" From af189853c63aed6734089e212b57dc34d4e2e012 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 27 Oct 2026 22:21:44 -0700 Subject: [PATCH 2/2] quest: finish the runner VM test quest Co-Authored-By: Claude Opus 5.5 --- quest/m0/test-flakes-2/README.md | 1 - quest/m0/test-flakes-2/runner-vm.md | 29 ----------------------------- 2 files changed, 30 deletions(-) delete mode 100644 quest/m0/test-flakes-2/runner-vm.md diff --git a/quest/m0/test-flakes-2/README.md b/quest/m0/test-flakes-2/README.md index 9df1c1b33da..c76d5928ec6 100644 --- a/quest/m0/test-flakes-2/README.md +++ b/quest/m0/test-flakes-2/README.md @@ -52,4 +52,3 @@ Public API: none. Wire: none. - [IETF22 fallback](/quest/m0/test-flakes-2/ietf22-fallback.md) - an IETF22 browser viewer follows a fallback within the media lane's bound, keeping its session and its audio - [Swapped Rust to JS Opus](/quest/m0/test-flakes-2/swapped-js-opus.md) - the `rust-swapped -> js` interop cell stops timing out under load - [JS host-task yields follow the clock](/quest/m0/test-flakes-2/js-host-yields.md) - `Budget` and `nextTask` yield through the injected clock, so `Manual` tests stop racing them -- [Runner VM test under load](/quest/m0/test-flakes-2/runner-vm.md) - the runner VM check passes when it rebuilds in a merge group, and a failed flake check prints its full log diff --git a/quest/m0/test-flakes-2/runner-vm.md b/quest/m0/test-flakes-2/runner-vm.md deleted file mode 100644 index 1bbb120ccc9..00000000000 --- a/quest/m0/test-flakes-2/runner-vm.md +++ /dev/null @@ -1,29 +0,0 @@ -# [S] Runner VM test under load - -## Goal - -`checks.x86_64-linux.github-runner-vm` (`nix/runner/test.nix`) passes in -every merge group it runs in, and a failed flake check prints the full log -of the failing derivation so the failing subtest is named. - -## Plan - -Decided 2026-10-28 (maintainer): an m0 flake quest, fixed at its cause. - -- It failed once, in merge-group run - [40609795466](https://github.com/moq-dev/moq/actions/runs/40609795466), - which dropped [moq-dev/moq#6934](https://github.com/moq-dev/moq/pull/6934) - from the queue. The PR did not change the derivation, and it passed when - built locally (10 subtests, about 4 minutes). -- It rarely runs: in 18 earlier merge groups on `nixos-moq-ci` the result was - already in the runner's store (`built=0`). This time it had been collected, - so it rebuilt, probably alongside the heavy `check rs` and `test rs` work. -- `sh/dispatch.sh` runs `nix flake check` without `-L`, so the log kept only - the last 25 lines (QEMU cleanup). First make a failed check print the full - build log (`-L`, or `nix log` on failure), so the next failure names its - subtest. -- Then reproduce under the same contention (rebuild with the store result - removed while a Rust test run loads the host) and fix the subtest at its - cause, not with a longer timeout. - -Public API: none. Wire: none.