diff --git a/nix/runner/test.nix b/nix/runner/test.nix index cd74e6bfd53..84e338830b3 100644 --- a/nix/runner/test.nix +++ b/nix/runner/test.nix @@ -296,7 +296,9 @@ pkgs.testers.runNixOSTest { # re-reads its inputs from disk until the job times out. assert elapsed < 60, "the guest thrashed instead of killing memory-hog" time.sleep(1) - print(guest.succeed("journalctl -u earlyoom | grep -F 'sending SIGTERM to process'")) + # earlyoom sends SIGKILL instead of SIGTERM when its poll first lands + # below the 5% kill threshold, which depends on how fast the hog fills. + print(guest.succeed("journalctl -u earlyoom | grep -E 'sending SIG(TERM|KILL) to process .*${./hog.py}'")) guest.succeed("rm /var/tmp/hog-inputs") router.wait_until_succeeds("${pkgs.openssh}/bin/ssh-keyscan -t ed25519 -T 5 10.231.0.2 | grep -F ssh-ed25519", timeout=timedelta(minutes=1)) diff --git a/quest/m0/test-flakes-2/README.md b/quest/m0/test-flakes-2/README.md index 9df1c1b33da..c76d5928ec6 100644 --- a/quest/m0/test-flakes-2/README.md +++ b/quest/m0/test-flakes-2/README.md @@ -52,4 +52,3 @@ Public API: none. Wire: none. - [IETF22 fallback](/quest/m0/test-flakes-2/ietf22-fallback.md) - an IETF22 browser viewer follows a fallback within the media lane's bound, keeping its session and its audio - [Swapped Rust to JS Opus](/quest/m0/test-flakes-2/swapped-js-opus.md) - the `rust-swapped -> js` interop cell stops timing out under load - [JS host-task yields follow the clock](/quest/m0/test-flakes-2/js-host-yields.md) - `Budget` and `nextTask` yield through the injected clock, so `Manual` tests stop racing them -- [Runner VM test under load](/quest/m0/test-flakes-2/runner-vm.md) - the runner VM check passes when it rebuilds in a merge group, and a failed flake check prints its full log diff --git a/quest/m0/test-flakes-2/runner-vm.md b/quest/m0/test-flakes-2/runner-vm.md deleted file mode 100644 index 1bbb120ccc9..00000000000 --- a/quest/m0/test-flakes-2/runner-vm.md +++ /dev/null @@ -1,29 +0,0 @@ -# [S] Runner VM test under load - -## Goal - -`checks.x86_64-linux.github-runner-vm` (`nix/runner/test.nix`) passes in -every merge group it runs in, and a failed flake check prints the full log -of the failing derivation so the failing subtest is named. - -## Plan - -Decided 2026-10-28 (maintainer): an m0 flake quest, fixed at its cause. - -- It failed once, in merge-group run - [40609795466](https://github.com/moq-dev/moq/actions/runs/40609795466), - which dropped [moq-dev/moq#6934](https://github.com/moq-dev/moq/pull/6934) - from the queue. The PR did not change the derivation, and it passed when - built locally (10 subtests, about 4 minutes). -- It rarely runs: in 18 earlier merge groups on `nixos-moq-ci` the result was - already in the runner's store (`built=0`). This time it had been collected, - so it rebuilt, probably alongside the heavy `check rs` and `test rs` work. -- `sh/dispatch.sh` runs `nix flake check` without `-L`, so the log kept only - the last 25 lines (QEMU cleanup). First make a failed check print the full - build log (`-L`, or `nix log` on failure), so the next failure names its - subtest. -- Then reproduce under the same contention (rebuild with the store result - removed while a Rust test run loads the host) and fix the subtest at its - cause, not with a longer timeout. - -Public API: none. Wire: none. diff --git a/sh/dispatch-query.test.py b/sh/dispatch-query.test.py index 0dab2a9a3b6..b846bc0756a 100644 --- a/sh/dispatch-query.test.py +++ b/sh/dispatch-query.test.py @@ -279,7 +279,7 @@ def test_local_flake_namespace_refusal_is_reported_as_skipped(self): self.assertIn("local Nix builder cannot create user namespaces", result.stdout + result.stderr) self.assertRegex(result.stderr, r"\d+ check flake \(skipped\)") calls = (self.root / "calls").read_text().splitlines() - self.assertNotIn("nix flake check", calls) + self.assertNotIn("nix flake check --print-build-logs", calls) self.assertIn("gh test", calls) probe, = [call for call in calls if call.startswith("nix build ")] self.assertIn("--builders --max-jobs 1", probe) @@ -297,7 +297,7 @@ def test_local_capable_builder_runs_the_full_flake(self): self.assertEqual(result.returncode, 0, result.stderr) self.assertNotIn("(skipped)", result.stderr) calls = (self.root / "calls").read_text().splitlines() - self.assertIn("nix flake check", calls) + self.assertIn("nix flake check --print-build-logs", calls) self.assertEqual(sum(call.startswith("nix build ") for call in calls), 1) @unittest.skipUnless(sys.platform == "linux" and os.uname().machine == "x86_64", "Linux VM checks only") @@ -305,7 +305,7 @@ def test_local_flake_check_exiting_77_fails_instead_of_skipping(self): self.stub_ci() self.change("flake.nix") nix = self.bin / "nix" - nix.write_text(nix.read_text() + 'case "$*" in "flake check") exit 77 ;; esac\n') + nix.write_text(nix.read_text() + 'case "$*" in "flake check --print-build-logs") exit 77 ;; esac\n') self.env.pop("CI_STRICT") result = subprocess.run(["bash", "sh/dispatch.sh", "check", "origin/main"], cwd=self.repo, env=self.env, text=True, capture_output=True) @@ -326,13 +326,13 @@ def test_local_flake_probe_errors_are_failures_without_cancelling_other_modules( self.assertIn("check: check flake failed", result.stderr) self.assertNotIn("(skipped)", result.stderr) calls = (self.root / "calls").read_text().splitlines() - self.assertNotIn("nix flake check", calls) + self.assertNotIn("nix flake check --print-build-logs", calls) self.assertIn("gh test", calls) def test_strict_ci_runs_the_full_flake_without_a_probe(self): self.change("flake.nix") calls = self.run_ci() - self.assertIn("nix flake check", calls) + self.assertIn("nix flake check --print-build-logs", calls) self.assertFalse(any(call.startswith("nix build ") for call in calls)) nix = self.bin / "nix" nix.write_text(nix.read_text() + 'exit 77\n') @@ -392,8 +392,8 @@ def test_flake_checks_follow_nix_inputs_instead_of_rust_source(self): self.change(path) calls = self.run_ci() (self.repo / path).unlink() - self.assertEqual("nix flake check" in calls, expected) - self.assertIn("nix flake check", self.run_ci("--all")) + self.assertEqual("nix flake check --print-build-logs" in calls, expected) + self.assertIn("nix flake check --print-build-logs", self.run_ci("--all")) def benchmark(): diff --git a/sh/dispatch.sh b/sh/dispatch.sh index c08d42d6629..f71c815df4f 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -312,7 +312,9 @@ step() { fi fi # 77 marks the probe's skip, so the check's own 77 is a failure. - bash sh/local.sh nix flake check || { + # Full build logs name a failing VM subtest; Nix otherwise keeps + # only the last 25 lines, which are QEMU's cleanup. + bash sh/local.sh nix flake check --print-build-logs || { local status=$? [[ -n "${CI_STRICT:-}" ]] || ((status != 77)) || status=1 return "$status"