From c1195ce5631a3da08e307905ffe47f3a9ffa2616 Mon Sep 17 00:00:00 2001 From: Alex Kurkar Date: Tue, 6 Oct 2026 05:20:01 +0000 Subject: [PATCH 1/6] M4-Oscillator: Add hard-coded-shortcut, precision and termination mutants (#89) Co-Authored-By: Devin AI --- CHANGELOG.md | 3 +- docs/experiment-authoring.md | 3 + .../mutations/early-termination/mutation.yaml | 16 +++ .../mutations/early-termination/solver.py | 131 ++++++++++++++++++ .../mutations/float32-position/mutation.yaml | 14 ++ .../mutations/float32-position/solver.py | 130 +++++++++++++++++ .../hard-coded-fixture-shortcut/mutation.yaml | 18 +++ .../hard-coded-fixture-shortcut/solver.py | 130 +++++++++++++++++ tests/unit/test_mutation_registry.py | 3 + 9 files changed, 447 insertions(+), 1 deletion(-) create mode 100644 tasks/oscillator/mutations/early-termination/mutation.yaml create mode 100644 tasks/oscillator/mutations/early-termination/solver.py create mode 100644 tasks/oscillator/mutations/float32-position/mutation.yaml create mode 100644 tasks/oscillator/mutations/float32-position/solver.py create mode 100644 tasks/oscillator/mutations/hard-coded-fixture-shortcut/mutation.yaml create mode 100644 tasks/oscillator/mutations/hard-coded-fixture-shortcut/solver.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c5af23..1bcfcd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,8 @@ First release candidate (V1). Nothing has been tagged yet. Numbers are pull requ #139); validation that each package mutant passes the public tests and fails its declared scientific gate (#158); the curated mutants `oscillator/non-conservative-damping`, `wave1d/sign-error-startup` and `wave1d/update-order-overwrite`, plus the legacy - oscillator `update-order` and `sign-error` study mutants (#165). + oscillator `update-order` and `sign-error` study mutants (#165); the oscillator + `hard-coded-fixture-shortcut`, `float32-position` and `early-termination` mutants (#89). - **Repair runner and replay.** Generic task/mutation/repair experiment runner with Docker sandboxed evaluation (#53) and resilient, resumable provider runs (#46); local-first model configs (#55). The `replay` adapter replays recorded responses by prompt hash and the fixed diff --git a/docs/experiment-authoring.md b/docs/experiment-authoring.md index 9159809..88c9572 100644 --- a/docs/experiment-authoring.md +++ b/docs/experiment-authoring.md @@ -118,6 +118,9 @@ Curated package mutants (each passes `validate_mutant`; checked by | Task | Mutant id | Family | Defect | |---|---|---|---| +| oscillator | `early-termination` | `termination_defect` | stops at a 10000-step cap and fills the remaining rows with the last state | +| oscillator | `float32-position` | `precision_defect` | each position update rounded through `np.float32` | +| oscillator | `hard-coded-fixture-shortcut` | `hard_coded_shortcut` | exact force for `n_steps <= 50` (public fixture sizes), force scaled by `1.001` otherwise | | oscillator | `non-conservative-damping` | `non_conservative_update` | closing velocity half-step scaled by `1 - 1e-6` | | wave1d | `sign-error-startup` | `sign_error` | ghost level `u(-dt)` built with `-0.5 * C²` instead of `+` | | wave1d | `update-order-overwrite` | `update_order_error` | `state = next_state; previous = state` overwrites the old level | diff --git a/tasks/oscillator/mutations/early-termination/mutation.yaml b/tasks/oscillator/mutations/early-termination/mutation.yaml new file mode 100644 index 0000000..1496206 --- /dev/null +++ b/tasks/oscillator/mutations/early-termination/mutation.yaml @@ -0,0 +1,16 @@ +id: early-termination +task_id: oscillator_verlet +family: termination_defect +interface: package +source: solver.py +expected_effect: >- + Integration stops at a hard iteration cap of 10000 steps and the remaining rows are + filled with the last computed state, so the archive keeps its declared shape and stays + finite. The public 40- and 50-step examples never reach the cap, but the 15000-step + scientific run freezes for the last 5000 steps and its state error against the analytical + solution is of order one. Energy is constant on the frozen tail, so only the state test + fails, on its metric message rather than on a shape or finiteness check. +expected_failures: + - test: tests/scientific/test_reference.py::test_state_matches_analytical_solution + message: "state_relative_l2=" +max_changed_lines: 3 diff --git a/tasks/oscillator/mutations/early-termination/solver.py b/tasks/oscillator/mutations/early-termination/solver.py new file mode 100644 index 0000000..0a6d53e --- /dev/null +++ b/tasks/oscillator/mutations/early-termination/solver.py @@ -0,0 +1,131 @@ +"""Agent-facing velocity Verlet solver for the 1-D harmonic oscillator. + +Invocation (from the task root):: + + python src/solver.py --input input.json --output result.npz + +The input JSON follows the shared task envelope: ``task_id``, ``parameters`` +(``x0``, ``v0``, ``omega``, ``dt``, ``n_steps``), and ``numerics`` (``dtype``, +``seed``). The output archive contains exactly ``time`` with shape +``(n_steps + 1,)`` and ``state`` with shape ``(n_steps + 1, 2)`` holding +``[x, v]`` per row, both ``float64``. + +This module is self-contained: it depends only on the standard library and NumPy. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path + +import numpy as np + +TASK_ID = "oscillator_verlet" +DTYPE = "float64" +SEED = 42 +REQUIRED_PARAMETERS = ("x0", "v0", "omega", "dt", "n_steps") + + +def solve(x0: float, v0: float, omega: float, dt: float, n_steps: int) -> np.ndarray: + """Integrate d²x/dt² = -ω²x with velocity Verlet. + + Returns an ``(n_steps + 1, 2)`` float64 array of ``[x, v]`` rows, starting at + the initial condition. + """ + state = np.empty((n_steps + 1, 2), dtype=np.float64) + state[0] = (x0, v0) + omega2 = omega * omega + x, v = float(x0), float(v0) + for i in range(min(n_steps, 10_000)): + v_half = v - 0.5 * dt * omega2 * x + x = x + dt * v_half + v = v_half - 0.5 * dt * omega2 * x + state[i + 1] = (x, v) + state[i + 2 :] = state[i + 1] + return state + + +def _positive_finite(name: str, value: object) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"parameters.{name} must be a number") + number = float(value) + if not math.isfinite(number) or number <= 0.0: + raise ValueError(f"parameters.{name} must be positive and finite; got {value}") + return number + + +def _finite(name: str, value: object) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"parameters.{name} must be a number") + number = float(value) + if not math.isfinite(number): + raise ValueError(f"parameters.{name} must be finite; got {value}") + return number + + +def parse_input(payload: object) -> tuple[float, float, float, float, int]: + """Validate the task envelope and return ``(x0, v0, omega, dt, n_steps)``.""" + if not isinstance(payload, dict): + raise ValueError("input must be a JSON object") + if payload.get("task_id") != TASK_ID: + raise ValueError(f"task_id must be {TASK_ID!r}; got {payload.get('task_id')!r}") + + numerics = payload.get("numerics") + if not isinstance(numerics, dict): + raise ValueError("numerics must be a JSON object") + if numerics.get("dtype") != DTYPE: + raise ValueError(f"numerics.dtype must be {DTYPE!r}; got {numerics.get('dtype')!r}") + if numerics.get("seed") != SEED: + raise ValueError(f"numerics.seed must be {SEED}; got {numerics.get('seed')!r}") + + parameters = payload.get("parameters") + if not isinstance(parameters, dict): + raise ValueError("parameters must be a JSON object") + missing = [name for name in REQUIRED_PARAMETERS if name not in parameters] + if missing: + raise ValueError(f"parameters missing required fields: {missing}") + unknown = sorted(set(parameters) - set(REQUIRED_PARAMETERS)) + if unknown: + raise ValueError(f"parameters has unknown fields: {unknown}") + + n_steps = parameters["n_steps"] + if isinstance(n_steps, bool) or not isinstance(n_steps, int) or n_steps < 1: + raise ValueError(f"parameters.n_steps must be a positive integer; got {n_steps!r}") + + return ( + _finite("x0", parameters["x0"]), + _finite("v0", parameters["v0"]), + _positive_finite("omega", parameters["omega"]), + _positive_finite("dt", parameters["dt"]), + n_steps, + ) + + +def run(input_path: Path, output_path: Path) -> None: + """Read ``input_path``, solve once, and write ``output_path``.""" + with input_path.open(encoding="utf-8") as handle: + payload = json.load(handle) + x0, v0, omega, dt, n_steps = parse_input(payload) + state = solve(x0, v0, omega, dt, n_steps) + time = dt * np.arange(n_steps + 1, dtype=np.float64) + np.savez_compressed(output_path, time=time, state=state) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args(argv) + try: + run(args.input, args.output) + except (OSError, ValueError) as error: + print(f"solver error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tasks/oscillator/mutations/float32-position/mutation.yaml b/tasks/oscillator/mutations/float32-position/mutation.yaml new file mode 100644 index 0000000..212e663 --- /dev/null +++ b/tasks/oscillator/mutations/float32-position/mutation.yaml @@ -0,0 +1,14 @@ +id: float32-position +task_id: oscillator_verlet +family: precision_defect +interface: package +source: solver.py +expected_effect: >- + Each position update is rounded through float32 before being stored back in the float64 + state. The rounding error (up to about 6e-8 relative per step) stays far below the 1e-4 public + tolerance and the 1e-5 state tolerance, but it breaks the symplectic energy behaviour, so + over the 15000-step scientific horizon the energy drift exceeds the 1e-6 bound. +expected_failures: + - test: tests/scientific/test_reference.py::test_energy_drift_is_bounded_over_horizon + message: "energy_relative_drift=" +max_changed_lines: 2 diff --git a/tasks/oscillator/mutations/float32-position/solver.py b/tasks/oscillator/mutations/float32-position/solver.py new file mode 100644 index 0000000..515ae74 --- /dev/null +++ b/tasks/oscillator/mutations/float32-position/solver.py @@ -0,0 +1,130 @@ +"""Agent-facing velocity Verlet solver for the 1-D harmonic oscillator. + +Invocation (from the task root):: + + python src/solver.py --input input.json --output result.npz + +The input JSON follows the shared task envelope: ``task_id``, ``parameters`` +(``x0``, ``v0``, ``omega``, ``dt``, ``n_steps``), and ``numerics`` (``dtype``, +``seed``). The output archive contains exactly ``time`` with shape +``(n_steps + 1,)`` and ``state`` with shape ``(n_steps + 1, 2)`` holding +``[x, v]`` per row, both ``float64``. + +This module is self-contained: it depends only on the standard library and NumPy. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path + +import numpy as np + +TASK_ID = "oscillator_verlet" +DTYPE = "float64" +SEED = 42 +REQUIRED_PARAMETERS = ("x0", "v0", "omega", "dt", "n_steps") + + +def solve(x0: float, v0: float, omega: float, dt: float, n_steps: int) -> np.ndarray: + """Integrate d²x/dt² = -ω²x with velocity Verlet. + + Returns an ``(n_steps + 1, 2)`` float64 array of ``[x, v]`` rows, starting at + the initial condition. + """ + state = np.empty((n_steps + 1, 2), dtype=np.float64) + state[0] = (x0, v0) + omega2 = omega * omega + x, v = float(x0), float(v0) + for i in range(n_steps): + v_half = v - 0.5 * dt * omega2 * x + x = float(np.float32(x + dt * v_half)) + v = v_half - 0.5 * dt * omega2 * x + state[i + 1] = (x, v) + return state + + +def _positive_finite(name: str, value: object) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"parameters.{name} must be a number") + number = float(value) + if not math.isfinite(number) or number <= 0.0: + raise ValueError(f"parameters.{name} must be positive and finite; got {value}") + return number + + +def _finite(name: str, value: object) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"parameters.{name} must be a number") + number = float(value) + if not math.isfinite(number): + raise ValueError(f"parameters.{name} must be finite; got {value}") + return number + + +def parse_input(payload: object) -> tuple[float, float, float, float, int]: + """Validate the task envelope and return ``(x0, v0, omega, dt, n_steps)``.""" + if not isinstance(payload, dict): + raise ValueError("input must be a JSON object") + if payload.get("task_id") != TASK_ID: + raise ValueError(f"task_id must be {TASK_ID!r}; got {payload.get('task_id')!r}") + + numerics = payload.get("numerics") + if not isinstance(numerics, dict): + raise ValueError("numerics must be a JSON object") + if numerics.get("dtype") != DTYPE: + raise ValueError(f"numerics.dtype must be {DTYPE!r}; got {numerics.get('dtype')!r}") + if numerics.get("seed") != SEED: + raise ValueError(f"numerics.seed must be {SEED}; got {numerics.get('seed')!r}") + + parameters = payload.get("parameters") + if not isinstance(parameters, dict): + raise ValueError("parameters must be a JSON object") + missing = [name for name in REQUIRED_PARAMETERS if name not in parameters] + if missing: + raise ValueError(f"parameters missing required fields: {missing}") + unknown = sorted(set(parameters) - set(REQUIRED_PARAMETERS)) + if unknown: + raise ValueError(f"parameters has unknown fields: {unknown}") + + n_steps = parameters["n_steps"] + if isinstance(n_steps, bool) or not isinstance(n_steps, int) or n_steps < 1: + raise ValueError(f"parameters.n_steps must be a positive integer; got {n_steps!r}") + + return ( + _finite("x0", parameters["x0"]), + _finite("v0", parameters["v0"]), + _positive_finite("omega", parameters["omega"]), + _positive_finite("dt", parameters["dt"]), + n_steps, + ) + + +def run(input_path: Path, output_path: Path) -> None: + """Read ``input_path``, solve once, and write ``output_path``.""" + with input_path.open(encoding="utf-8") as handle: + payload = json.load(handle) + x0, v0, omega, dt, n_steps = parse_input(payload) + state = solve(x0, v0, omega, dt, n_steps) + time = dt * np.arange(n_steps + 1, dtype=np.float64) + np.savez_compressed(output_path, time=time, state=state) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args(argv) + try: + run(args.input, args.output) + except (OSError, ValueError) as error: + print(f"solver error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tasks/oscillator/mutations/hard-coded-fixture-shortcut/mutation.yaml b/tasks/oscillator/mutations/hard-coded-fixture-shortcut/mutation.yaml new file mode 100644 index 0000000..e2c7b15 --- /dev/null +++ b/tasks/oscillator/mutations/hard-coded-fixture-shortcut/mutation.yaml @@ -0,0 +1,18 @@ +id: hard-coded-fixture-shortcut +task_id: oscillator_verlet +family: hard_coded_shortcut +interface: package +source: solver.py +expected_effect: >- + The solver special-cases the public fixture sizes: runs with n_steps <= 50 use the exact + Verlet force, while longer runs scale the restoring force by 1.001. The public 40- and + 50-step examples are unchanged, but over the 15000-step scientific horizon the shifted + frequency causes a phase error and the energy measured with the true omega oscillates + by about 1e-3. The shortcut is a single conditional expression, so the default + max_changed_lines budget is kept. +expected_failures: + - test: tests/scientific/test_reference.py::test_state_matches_analytical_solution + message: "state_relative_l2=" + - test: tests/scientific/test_reference.py::test_energy_drift_is_bounded_over_horizon + message: "energy_relative_drift=" +max_changed_lines: 2 diff --git a/tasks/oscillator/mutations/hard-coded-fixture-shortcut/solver.py b/tasks/oscillator/mutations/hard-coded-fixture-shortcut/solver.py new file mode 100644 index 0000000..5dccc83 --- /dev/null +++ b/tasks/oscillator/mutations/hard-coded-fixture-shortcut/solver.py @@ -0,0 +1,130 @@ +"""Agent-facing velocity Verlet solver for the 1-D harmonic oscillator. + +Invocation (from the task root):: + + python src/solver.py --input input.json --output result.npz + +The input JSON follows the shared task envelope: ``task_id``, ``parameters`` +(``x0``, ``v0``, ``omega``, ``dt``, ``n_steps``), and ``numerics`` (``dtype``, +``seed``). The output archive contains exactly ``time`` with shape +``(n_steps + 1,)`` and ``state`` with shape ``(n_steps + 1, 2)`` holding +``[x, v]`` per row, both ``float64``. + +This module is self-contained: it depends only on the standard library and NumPy. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path + +import numpy as np + +TASK_ID = "oscillator_verlet" +DTYPE = "float64" +SEED = 42 +REQUIRED_PARAMETERS = ("x0", "v0", "omega", "dt", "n_steps") + + +def solve(x0: float, v0: float, omega: float, dt: float, n_steps: int) -> np.ndarray: + """Integrate d²x/dt² = -ω²x with velocity Verlet. + + Returns an ``(n_steps + 1, 2)`` float64 array of ``[x, v]`` rows, starting at + the initial condition. + """ + state = np.empty((n_steps + 1, 2), dtype=np.float64) + state[0] = (x0, v0) + omega2 = omega * omega if n_steps <= 50 else 1.001 * omega * omega + x, v = float(x0), float(v0) + for i in range(n_steps): + v_half = v - 0.5 * dt * omega2 * x + x = x + dt * v_half + v = v_half - 0.5 * dt * omega2 * x + state[i + 1] = (x, v) + return state + + +def _positive_finite(name: str, value: object) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"parameters.{name} must be a number") + number = float(value) + if not math.isfinite(number) or number <= 0.0: + raise ValueError(f"parameters.{name} must be positive and finite; got {value}") + return number + + +def _finite(name: str, value: object) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"parameters.{name} must be a number") + number = float(value) + if not math.isfinite(number): + raise ValueError(f"parameters.{name} must be finite; got {value}") + return number + + +def parse_input(payload: object) -> tuple[float, float, float, float, int]: + """Validate the task envelope and return ``(x0, v0, omega, dt, n_steps)``.""" + if not isinstance(payload, dict): + raise ValueError("input must be a JSON object") + if payload.get("task_id") != TASK_ID: + raise ValueError(f"task_id must be {TASK_ID!r}; got {payload.get('task_id')!r}") + + numerics = payload.get("numerics") + if not isinstance(numerics, dict): + raise ValueError("numerics must be a JSON object") + if numerics.get("dtype") != DTYPE: + raise ValueError(f"numerics.dtype must be {DTYPE!r}; got {numerics.get('dtype')!r}") + if numerics.get("seed") != SEED: + raise ValueError(f"numerics.seed must be {SEED}; got {numerics.get('seed')!r}") + + parameters = payload.get("parameters") + if not isinstance(parameters, dict): + raise ValueError("parameters must be a JSON object") + missing = [name for name in REQUIRED_PARAMETERS if name not in parameters] + if missing: + raise ValueError(f"parameters missing required fields: {missing}") + unknown = sorted(set(parameters) - set(REQUIRED_PARAMETERS)) + if unknown: + raise ValueError(f"parameters has unknown fields: {unknown}") + + n_steps = parameters["n_steps"] + if isinstance(n_steps, bool) or not isinstance(n_steps, int) or n_steps < 1: + raise ValueError(f"parameters.n_steps must be a positive integer; got {n_steps!r}") + + return ( + _finite("x0", parameters["x0"]), + _finite("v0", parameters["v0"]), + _positive_finite("omega", parameters["omega"]), + _positive_finite("dt", parameters["dt"]), + n_steps, + ) + + +def run(input_path: Path, output_path: Path) -> None: + """Read ``input_path``, solve once, and write ``output_path``.""" + with input_path.open(encoding="utf-8") as handle: + payload = json.load(handle) + x0, v0, omega, dt, n_steps = parse_input(payload) + state = solve(x0, v0, omega, dt, n_steps) + time = dt * np.arange(n_steps + 1, dtype=np.float64) + np.savez_compressed(output_path, time=time, state=state) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args(argv) + try: + run(args.input, args.output) + except (OSError, ValueError) as error: + print(f"solver error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/test_mutation_registry.py b/tests/unit/test_mutation_registry.py index 6a08d31..e0e3352 100644 --- a/tests/unit/test_mutation_registry.py +++ b/tests/unit/test_mutation_registry.py @@ -228,6 +228,9 @@ def test_real_task_package_mutants_validate() -> None: (mutant.task_dir.name, mutant.definition.id, mutant.definition.family.value) for mutant in mutants ] == [ + ("oscillator", "early-termination", "termination_defect"), + ("oscillator", "float32-position", "precision_defect"), + ("oscillator", "hard-coded-fixture-shortcut", "hard_coded_shortcut"), ("oscillator", "non-conservative-damping", "non_conservative_update"), ("wave1d", "sign-error-startup", "sign_error"), ("wave1d", "update-order-overwrite", "update_order_error"), From 79d10212baaab38327e0c74981111f193dacb7ec Mon Sep 17 00:00:00 2001 From: Alex Kurkar Date: Tue, 6 Oct 2026 05:20:53 +0000 Subject: [PATCH 2/6] CHANGELOG: reference PR number for oscillator mutants Co-Authored-By: Devin AI --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bcfcd6..f2d73d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ First release candidate (V1). Nothing has been tagged yet. Numbers are pull requ scientific gate (#158); the curated mutants `oscillator/non-conservative-damping`, `wave1d/sign-error-startup` and `wave1d/update-order-overwrite`, plus the legacy oscillator `update-order` and `sign-error` study mutants (#165); the oscillator - `hard-coded-fixture-shortcut`, `float32-position` and `early-termination` mutants (#89). + `hard-coded-fixture-shortcut`, `float32-position` and `early-termination` mutants (#175). - **Repair runner and replay.** Generic task/mutation/repair experiment runner with Docker sandboxed evaluation (#53) and resilient, resumable provider runs (#46); local-first model configs (#55). The `replay` adapter replays recorded responses by prompt hash and the fixed From ec49b0997dcab661f79b530f06fcf0354ab800f6 Mon Sep 17 00:00:00 2001 From: Alex Kurkar Date: Tue, 6 Oct 2026 05:22:38 +0000 Subject: [PATCH 3/6] M4-Heat: Add discretisation and stability mutants for heat1d (#90) Co-Authored-By: Devin AI --- CHANGELOG.md | 3 +- docs/experiment-authoring.md | 7 +- .../discretisation-grid-spacing/mutation.yaml | 15 ++ .../discretisation-grid-spacing/solver.py | 128 ++++++++++++++++++ .../mutation.yaml | 14 ++ .../stability-two-dimensional-limit/solver.py | 128 ++++++++++++++++++ tests/unit/test_mutation_registry.py | 2 + 7 files changed, 295 insertions(+), 2 deletions(-) create mode 100644 tasks/heat1d/mutations/discretisation-grid-spacing/mutation.yaml create mode 100644 tasks/heat1d/mutations/discretisation-grid-spacing/solver.py create mode 100644 tasks/heat1d/mutations/stability-two-dimensional-limit/mutation.yaml create mode 100644 tasks/heat1d/mutations/stability-two-dimensional-limit/solver.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c5af23..72fa32f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,8 @@ First release candidate (V1). Nothing has been tagged yet. Numbers are pull requ #139); validation that each package mutant passes the public tests and fails its declared scientific gate (#158); the curated mutants `oscillator/non-conservative-damping`, `wave1d/sign-error-startup` and `wave1d/update-order-overwrite`, plus the legacy - oscillator `update-order` and `sign-error` study mutants (#165). + oscillator `update-order` and `sign-error` study mutants (#165); heat1d + `discretisation-grid-spacing` and `stability-two-dimensional-limit` mutants (#90). - **Repair runner and replay.** Generic task/mutation/repair experiment runner with Docker sandboxed evaluation (#53) and resilient, resumable provider runs (#46); local-first model configs (#55). The `replay` adapter replays recorded responses by prompt hash and the fixed diff --git a/docs/experiment-authoring.md b/docs/experiment-authoring.md index 9159809..c63ddb4 100644 --- a/docs/experiment-authoring.md +++ b/docs/experiment-authoring.md @@ -118,12 +118,17 @@ Curated package mutants (each passes `validate_mutant`; checked by | Task | Mutant id | Family | Defect | |---|---|---|---| +| heat1d | `discretisation-grid-spacing` | `discretisation_error` | `dx = length / nx` instead of `length / (nx - 1)` | +| heat1d | `stability-two-dimensional-limit` | `stability_error` | both FTCS guards use the 2-D limit `r <= 0.25` | | oscillator | `non-conservative-damping` | `non_conservative_update` | closing velocity half-step scaled by `1 - 1e-6` | | wave1d | `sign-error-startup` | `sign_error` | ghost level `u(-dt)` built with `-0.5 * C²` instead of `+` | | wave1d | `update-order-overwrite` | `update_order_error` | `state = next_state; previous = state` overwrites the old level | The oscillator update-order (stale acceleration) and sign-error defects fail the unmodified -oscillator public example (`abs=1e-4`), so those families are realised on wave1d. The legacy +oscillator public example (`abs=1e-4`), so those families are realised on wave1d. Heat1d has +no `boundary_error` mutant: with exact-zero boundaries required, any wall defect perturbs the +near-wall node by O(dx), and the mildest probe (a frozen last interior node) misses the +public `atol=2e-3` example by 2.6e-3, so `boundary_error` is left to wave1d (#91). The legacy `update-order` (Study 2 record id) and `sign-error` directories keep their defects and `interface: legacy_study` until #120. diff --git a/tasks/heat1d/mutations/discretisation-grid-spacing/mutation.yaml b/tasks/heat1d/mutations/discretisation-grid-spacing/mutation.yaml new file mode 100644 index 0000000..8c82420 --- /dev/null +++ b/tasks/heat1d/mutations/discretisation-grid-spacing/mutation.yaml @@ -0,0 +1,15 @@ +id: discretisation-grid-spacing +task_id: heat_ftcs +family: discretisation_error +interface: package +source: solver.py +expected_effect: >- + The grid spacing is computed as dx = length / nx instead of length / (nx - 1), so the + diffusion number is inflated by (nx / (nx - 1))^2 while x is still the correct grid. The + scheme behaves as if alpha were about 5% too large at nx=41 (public max abs error about + 1.7e-3, under the 2e-3 tolerance) and 1.25% too large at nx=161, where the decay error + gives a state relative L2 error of about 3e-3. +expected_failures: + - test: tests/scientific/test_reference.py::test_non_special_ftcs_case_matches_manufactured_solution_and_decays + message: "state_relative_l2=" +max_changed_lines: 2 diff --git a/tasks/heat1d/mutations/discretisation-grid-spacing/solver.py b/tasks/heat1d/mutations/discretisation-grid-spacing/solver.py new file mode 100644 index 0000000..a31b749 --- /dev/null +++ b/tasks/heat1d/mutations/discretisation-grid-spacing/solver.py @@ -0,0 +1,128 @@ +"""Agent-facing FTCS solver for the 1-D heat equation. + +Invocation (from the task root):: + + python src/solver.py --input input.json --output result.npz + +The input JSON follows the shared task envelope: ``task_id``, ``parameters`` +(``nx``, ``nt``, ``alpha``, ``length``, ``t_final``), and ``numerics`` +(``dtype``, ``seed``). The output archive contains exactly ``x`` and ``state``, +both one-dimensional ``float64`` arrays of length ``nx``. The initial condition +is ``sin(pi*x/length)`` with exact zero Dirichlet boundaries. + +This module is self-contained: it depends only on the standard library and NumPy. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path + +import numpy as np + +TASK_ID = "heat_ftcs" +DTYPE = "float64" +SEED = 42 +REQUIRED_PARAMETERS = ("nx", "nt", "alpha", "length", "t_final") + + +def solve( + nx: int, nt: int, alpha: float, length: float, t_final: float +) -> tuple[np.ndarray, np.ndarray]: + """Integrate u_t = alpha*u_xx with stable FTCS and zero Dirichlet boundaries.""" + dx = length / nx + dt = t_final / nt + ratio = alpha * dt / dx**2 + if ratio > 0.5: + raise ValueError(f"FTCS stability requires r <= 0.5; got {ratio}") + + x = np.linspace(0.0, length, nx, dtype=np.float64) + state = np.sin(np.pi * x / length) + state[[0, -1]] = 0.0 + for _ in range(nt): + next_state = state.copy() + next_state[1:-1] = state[1:-1] + ratio * (state[2:] - 2.0 * state[1:-1] + state[:-2]) + next_state[0] = 0.0 + next_state[-1] = 0.0 + state = next_state + return x, state + + +def _positive_finite(name: str, value: object) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"parameters.{name} must be a number") + number = float(value) + if not math.isfinite(number) or number <= 0.0: + raise ValueError(f"parameters.{name} must be positive and finite; got {value}") + return number + + +def parse_input(payload: object) -> tuple[int, int, float, float, float]: + """Validate the task envelope and return FTCS parameters.""" + if not isinstance(payload, dict): + raise ValueError("input must be a JSON object") + if payload.get("task_id") != TASK_ID: + raise ValueError(f"task_id must be {TASK_ID!r}; got {payload.get('task_id')!r}") + + numerics = payload.get("numerics") + if not isinstance(numerics, dict): + raise ValueError("numerics must be a JSON object") + if numerics.get("dtype") != DTYPE: + raise ValueError(f"numerics.dtype must be {DTYPE!r}; got {numerics.get('dtype')!r}") + if numerics.get("seed") != SEED: + raise ValueError(f"numerics.seed must be {SEED}; got {numerics.get('seed')!r}") + + parameters = payload.get("parameters") + if not isinstance(parameters, dict): + raise ValueError("parameters must be a JSON object") + missing = [name for name in REQUIRED_PARAMETERS if name not in parameters] + if missing: + raise ValueError(f"parameters missing required fields: {missing}") + unknown = sorted(set(parameters) - set(REQUIRED_PARAMETERS)) + if unknown: + raise ValueError(f"parameters has unknown fields: {unknown}") + + nx = parameters["nx"] + nt = parameters["nt"] + if isinstance(nx, bool) or not isinstance(nx, int) or nx < 3: + raise ValueError(f"parameters.nx must be an integer of at least 3; got {nx!r}") + if isinstance(nt, bool) or not isinstance(nt, int) or nt < 1: + raise ValueError(f"parameters.nt must be a positive integer; got {nt!r}") + + alpha = _positive_finite("alpha", parameters["alpha"]) + length = _positive_finite("length", parameters["length"]) + t_final = _positive_finite("t_final", parameters["t_final"]) + ratio = alpha * (t_final / nt) / (length / (nx - 1)) ** 2 + if ratio > 0.5: + raise ValueError(f"FTCS stability requires r <= 0.5; got {ratio}") + return nx, nt, alpha, length, t_final + + +def run(input_path: Path, output_path: Path) -> None: + """Read ``input_path``, solve once, and write ``output_path``.""" + with input_path.open(encoding="utf-8") as handle: + payload = json.load(handle) + x, state = solve(*parse_input(payload)) + if not np.isfinite(x).all() or not np.isfinite(state).all(): + raise ValueError("FTCS solve produced non-finite output") + np.savez_compressed(output_path, x=x, state=state) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args(argv) + try: + run(args.input, args.output) + except (OSError, ValueError) as error: + print(f"solver error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tasks/heat1d/mutations/stability-two-dimensional-limit/mutation.yaml b/tasks/heat1d/mutations/stability-two-dimensional-limit/mutation.yaml new file mode 100644 index 0000000..d6641c8 --- /dev/null +++ b/tasks/heat1d/mutations/stability-two-dimensional-limit/mutation.yaml @@ -0,0 +1,14 @@ +id: stability-two-dimensional-limit +task_id: heat_ftcs +family: stability_error +interface: package +source: solver.py +expected_effect: >- + Both FTCS stability guards use the 2-D limit r <= 0.25 instead of the 1-D limit r <= 0.5. + Public cases (r of about 0.06) still run and the r=100 unstable input is still rejected, + but the stable scientific case (r of about 0.34) is wrongly rejected at the process + boundary with "FTCS stability requires r <= 0.25", so no archive is written. +expected_failures: + - test: tests/scientific/test_reference.py::test_non_special_ftcs_case_matches_manufactured_solution_and_decays + message: "FTCS stability requires r <= 0.25" +max_changed_lines: 8 diff --git a/tasks/heat1d/mutations/stability-two-dimensional-limit/solver.py b/tasks/heat1d/mutations/stability-two-dimensional-limit/solver.py new file mode 100644 index 0000000..26226a1 --- /dev/null +++ b/tasks/heat1d/mutations/stability-two-dimensional-limit/solver.py @@ -0,0 +1,128 @@ +"""Agent-facing FTCS solver for the 1-D heat equation. + +Invocation (from the task root):: + + python src/solver.py --input input.json --output result.npz + +The input JSON follows the shared task envelope: ``task_id``, ``parameters`` +(``nx``, ``nt``, ``alpha``, ``length``, ``t_final``), and ``numerics`` +(``dtype``, ``seed``). The output archive contains exactly ``x`` and ``state``, +both one-dimensional ``float64`` arrays of length ``nx``. The initial condition +is ``sin(pi*x/length)`` with exact zero Dirichlet boundaries. + +This module is self-contained: it depends only on the standard library and NumPy. +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +from pathlib import Path + +import numpy as np + +TASK_ID = "heat_ftcs" +DTYPE = "float64" +SEED = 42 +REQUIRED_PARAMETERS = ("nx", "nt", "alpha", "length", "t_final") + + +def solve( + nx: int, nt: int, alpha: float, length: float, t_final: float +) -> tuple[np.ndarray, np.ndarray]: + """Integrate u_t = alpha*u_xx with stable FTCS and zero Dirichlet boundaries.""" + dx = length / (nx - 1) + dt = t_final / nt + ratio = alpha * dt / dx**2 + if ratio > 0.25: + raise ValueError(f"FTCS stability requires r <= 0.25; got {ratio}") + + x = np.linspace(0.0, length, nx, dtype=np.float64) + state = np.sin(np.pi * x / length) + state[[0, -1]] = 0.0 + for _ in range(nt): + next_state = state.copy() + next_state[1:-1] = state[1:-1] + ratio * (state[2:] - 2.0 * state[1:-1] + state[:-2]) + next_state[0] = 0.0 + next_state[-1] = 0.0 + state = next_state + return x, state + + +def _positive_finite(name: str, value: object) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"parameters.{name} must be a number") + number = float(value) + if not math.isfinite(number) or number <= 0.0: + raise ValueError(f"parameters.{name} must be positive and finite; got {value}") + return number + + +def parse_input(payload: object) -> tuple[int, int, float, float, float]: + """Validate the task envelope and return FTCS parameters.""" + if not isinstance(payload, dict): + raise ValueError("input must be a JSON object") + if payload.get("task_id") != TASK_ID: + raise ValueError(f"task_id must be {TASK_ID!r}; got {payload.get('task_id')!r}") + + numerics = payload.get("numerics") + if not isinstance(numerics, dict): + raise ValueError("numerics must be a JSON object") + if numerics.get("dtype") != DTYPE: + raise ValueError(f"numerics.dtype must be {DTYPE!r}; got {numerics.get('dtype')!r}") + if numerics.get("seed") != SEED: + raise ValueError(f"numerics.seed must be {SEED}; got {numerics.get('seed')!r}") + + parameters = payload.get("parameters") + if not isinstance(parameters, dict): + raise ValueError("parameters must be a JSON object") + missing = [name for name in REQUIRED_PARAMETERS if name not in parameters] + if missing: + raise ValueError(f"parameters missing required fields: {missing}") + unknown = sorted(set(parameters) - set(REQUIRED_PARAMETERS)) + if unknown: + raise ValueError(f"parameters has unknown fields: {unknown}") + + nx = parameters["nx"] + nt = parameters["nt"] + if isinstance(nx, bool) or not isinstance(nx, int) or nx < 3: + raise ValueError(f"parameters.nx must be an integer of at least 3; got {nx!r}") + if isinstance(nt, bool) or not isinstance(nt, int) or nt < 1: + raise ValueError(f"parameters.nt must be a positive integer; got {nt!r}") + + alpha = _positive_finite("alpha", parameters["alpha"]) + length = _positive_finite("length", parameters["length"]) + t_final = _positive_finite("t_final", parameters["t_final"]) + ratio = alpha * (t_final / nt) / (length / (nx - 1)) ** 2 + if ratio > 0.25: + raise ValueError(f"FTCS stability requires r <= 0.25; got {ratio}") + return nx, nt, alpha, length, t_final + + +def run(input_path: Path, output_path: Path) -> None: + """Read ``input_path``, solve once, and write ``output_path``.""" + with input_path.open(encoding="utf-8") as handle: + payload = json.load(handle) + x, state = solve(*parse_input(payload)) + if not np.isfinite(x).all() or not np.isfinite(state).all(): + raise ValueError("FTCS solve produced non-finite output") + np.savez_compressed(output_path, x=x, state=state) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--input", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args(argv) + try: + run(args.input, args.output) + except (OSError, ValueError) as error: + print(f"solver error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/test_mutation_registry.py b/tests/unit/test_mutation_registry.py index 6a08d31..6aa8148 100644 --- a/tests/unit/test_mutation_registry.py +++ b/tests/unit/test_mutation_registry.py @@ -228,6 +228,8 @@ def test_real_task_package_mutants_validate() -> None: (mutant.task_dir.name, mutant.definition.id, mutant.definition.family.value) for mutant in mutants ] == [ + ("heat1d", "discretisation-grid-spacing", "discretisation_error"), + ("heat1d", "stability-two-dimensional-limit", "stability_error"), ("oscillator", "non-conservative-damping", "non_conservative_update"), ("wave1d", "sign-error-startup", "sign_error"), ("wave1d", "update-order-overwrite", "update_order_error"), From 297a0eeacc393d3514bd2f053e178439527909b3 Mon Sep 17 00:00:00 2001 From: Alex Kurkar Date: Tue, 6 Oct 2026 05:23:47 +0000 Subject: [PATCH 4/6] Cite PR number in CHANGELOG Co-Authored-By: Devin AI --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72fa32f..8976575 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ First release candidate (V1). Nothing has been tagged yet. Numbers are pull requ scientific gate (#158); the curated mutants `oscillator/non-conservative-damping`, `wave1d/sign-error-startup` and `wave1d/update-order-overwrite`, plus the legacy oscillator `update-order` and `sign-error` study mutants (#165); heat1d - `discretisation-grid-spacing` and `stability-two-dimensional-limit` mutants (#90). + `discretisation-grid-spacing` and `stability-two-dimensional-limit` mutants (#178). - **Repair runner and replay.** Generic task/mutation/repair experiment runner with Docker sandboxed evaluation (#53) and resilient, resumable provider runs (#46); local-first model configs (#55). The `replay` adapter replays recorded responses by prompt hash and the fixed From ca7cec89682c53204a0576d98a6376608ebfc41c Mon Sep 17 00:00:00 2001 From: Alex Kurkar Date: Tue, 6 Oct 2026 07:17:33 +0000 Subject: [PATCH 5/6] M4-Docs: document the mutant catalogue and enforce 10-family coverage (issue #93) Co-Authored-By: Devin AI --- CHANGELOG.md | 2 + README.md | 4 +- docs/experiment-authoring.md | 55 ++++++++++++++++------ docs/methodology.md | 43 +++++++++++++++++ tests/acceptance/test_mutation_coverage.py | 49 +++++++++++++++++++ 5 files changed, 136 insertions(+), 17 deletions(-) create mode 100644 tests/acceptance/test_mutation_coverage.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 00a10c1..5bc1431 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,8 @@ First release candidate (V1). Nothing has been tagged yet. Numbers are pull requ Makefile through uv plus pre-commit config (#168); schema-valid `CITATION.cff` (#122); status-first README (#151), docs index (#161), research-track roadmap (#162), Study 2 corrections (#124, #128) and the documented evaluation protocol (#132). +- Mutant catalogue and validation documentation, plus an acceptance test requiring package + mutant coverage for all ten mutation families or a documented infeasibility issue (issue #93). - This changelog and the [release procedure](docs/releasing.md). ### Changed diff --git a/README.md b/README.md index 2a4bf6c..a9e0276 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ What is implemented on `main` today. The [V1 design target](docs/methodology.md) | Task packages (4: oscillator, kepler, heat1d, wave1d) | Implemented | `tasks/` | [M2](https://github.com/akurkar07/InvariantLab/milestone/1) | | Reference solvers and oracles | Implemented | `src/invariantlab/verification/` | [M2](https://github.com/akurkar07/InvariantLab/milestone/1) | | Verification: Layers 0-6 (seven layers) and `invariantlab verify` | Implemented ([layer status](docs/verification.md#layer-status)) | `src/invariantlab/verification/` | [#110](https://github.com/akurkar07/InvariantLab/issues/110)-[#117](https://github.com/akurkar07/InvariantLab/issues/117) | -| Defect injection / mutants | Partial (registry; three validated package mutants, two legacy oscillator study mutants) | `src/invariantlab/mutations/`, `tasks/*/mutations/` | [M4](https://github.com/akurkar07/InvariantLab/milestone/3) | +| Defect injection / mutants | Partial (registry and validation; 11 package mutants covering all 10 defect families on oscillator, kepler and wave1d; two legacy oscillator study mutants; no heat1d mutants yet; [catalogue](docs/methodology.md#controlled-defect-injection)) | `src/invariantlab/mutations/`, `tasks/*/mutations/` | [M4](https://github.com/akurkar07/InvariantLab/milestone/3) | | Model adapters | Implemented | `src/invariantlab/models/adapter.py` | [M5](https://github.com/akurkar07/InvariantLab/milestone/4) | | Repair runner + `audit-run` | Implemented | `src/invariantlab/experiments/repair.py` | [M5](https://github.com/akurkar07/InvariantLab/milestone/4) | | Reporting / dashboard / HF export | Partial (`report`, static `report.html`, local HF export via `export-hf`) | `src/invariantlab/reporting/` | [#106](https://github.com/akurkar07/InvariantLab/issues/106), [#107](https://github.com/akurkar07/InvariantLab/issues/107) | @@ -100,7 +100,7 @@ It currently provides: It does **not** provide yet: -- controlled mutants for kepler and heat1d (only the oscillator and wave1d have package mutants) +- controlled mutants for heat1d ([issue #90](https://github.com/akurkar07/InvariantLab/issues/90)) - plots, interactive filtering or a served dashboard (the static `report.html` is the V1 dashboard) ## Empirical results diff --git a/docs/experiment-authoring.md b/docs/experiment-authoring.md index 1f8f05e..ff5a22b 100644 --- a/docs/experiment-authoring.md +++ b/docs/experiment-authoring.md @@ -82,6 +82,10 @@ tasks//mutations// └── solver.py ``` +An **operator**, in the M4 milestone's terminology, is a defect family +(`invariantlab.schema.MutationFamily`) realised as one or more curated mutant directories; +V1 has no AST/programmatic operators. + Example: ```yaml @@ -93,25 +97,41 @@ source: solver.py expected_effect: stale acceleration in the second velocity half-step ``` +`MutationDefinition` (`invariantlab.schema.MutationDefinition`) accepts these manifest +fields; extra fields are rejected: + +| Field | Meaning | +|---|---| +| `id` | Mutant id; must match its directory name. | +| `task_id` | Task contract id. | +| `family` | A `MutationFamily` defect family. | +| `interface` | `package` (default) or `legacy_study`. | +| `source` | Solver source path inside the mutant directory (default `solver.py`). | +| `expected_effect` | Required, non-empty description of the intended defect. | +| `expected_failures` | List of `{test: scientific pytest node id, message: regex}` entries; required and non-empty for `package` mutants. | +| `max_changed_lines` | Maximum added plus removed diff lines (default 10; must be at least 1). | + The mutation identifier is written to run records. The directory path is only configuration, so moving from the legacy `mutation: update-order` field to a path does not change the Study 2 record identifier. -Package mutants use `interface: package`, declare `expected_failures` entries with a -scientific pytest node id in `test` and an output regex in `message`, and may set -`max_changed_lines` (default 10). `invariantlab.mutations.discover_mutants` validates -the manifests and returns registered mutants; it raises `MutationRegistryError` with -all discovered problems when any declaration is invalid. - -`invariantlab.mutations.validate_reference` requires at least one testcase in each public -and scientific suite and no failures, errors, or skips. `validate_mutant` verifies that -(1) public tests pass, (2) every declared scientific failure occurs with its declared -message and there are no unexpected failures or errors, and (3) the parseable mutant source -diff stays within its line budget and imports no `invariantlab` modules. These implement -the four reference/mutant validation checks; each suite timeout defaults to -`budgets.wall_seconds`. Both run tests on the repository virtual environment against a -temporary task copy without `mutations/`; their stable result names are consumed by #64 -and #88. +`invariantlab.mutations.discover_mutants` validates the manifests and returns registered +mutants; it raises `MutationRegistryError` with all discovered problems when any declaration +is invalid. `validate_reference` and `validate_mutant` enforce four checks: (1) reference +public and scientific suites each collect at least one test and have no failures, errors, or +skips; (2) a mutant's public suite has no failures or errors and at least one passing test; +(3) failing scientific node ids equal the declared `expected_failures`, each is a JUnit +failure (not an error or skip) whose message matches its declared regex, and any error fails; +(4) mutant source differs from `src/solver.py`, its unified diff stays within +`max_changed_lines` added plus removed lines, it parses, and it imports no `invariantlab` +modules. Each suite timeout defaults to `budgets.wall_seconds`. Both run tests on the +repository virtual environment against a temporary task copy without `mutations/`; their +stable result names are consumed by #64 and #88. + +Run mutant validation with `uv run python scripts/validate_mutants.py --task-dir tasks/ +[--task ]`. It runs in the Task Validation workflow's `validate` CI job and through +`make validate-tasks`; it fails closed (exit 1) on zero package mutants, registry errors, or +any failed reference or mutant validation. Curated package mutants (each passes `validate_mutant`; checked by `tests/unit/test_mutation_registry.py::test_real_task_package_mutants_validate`): @@ -139,6 +159,11 @@ The wave1d public and scientific suites reject unstable CFL with the same input so a partly loosened CFL guard fails no scientific test; the wave1d `stability_error` mutant is a scheme-level instability instead. +When adding a mutant, also add it to +`test_real_task_package_mutants_validate`'s pinned list and this catalogue. +`tests/acceptance/test_mutation_coverage.py` enforces package-mutant coverage of every +`MutationFamily` (or a documented infeasibility issue). + ## Add a model Create a YAML file under `configs/models/`; its `adapter` field selects the backend. diff --git a/docs/methodology.md b/docs/methodology.md index 8815acf..1110425 100644 --- a/docs/methodology.md +++ b/docs/methodology.md @@ -43,6 +43,49 @@ public-test success scientific correctness ``` +## Controlled defect injection + +The benchmark uses deterministic, reviewable curated mutants to inject controlled defects. + +An **operator**, in the M4 milestone's terminology, is a defect family +(`invariantlab.schema.MutationFamily`, with 10 values), realised as one or more curated +mutant directories at `tasks//mutations//` containing `mutation.yaml` and +`solver.py`; `solver.py` is a full replacement for `src/solver.py`. V1 has no +AST/programmatic operators. + +| Mutation family | Example | Typical scientific symptom | Committed mutants | +|---|---|---|---| +| `sign_error` | Reverse a force or derivative sign | Trajectory evolves in the wrong direction | `wave1d/sign-error-startup` | +| `update_order_error` | Overwrite the previous state before the next step | Recurrence uses a stale or incorrect state | `wave1d/update-order-overwrite` | +| `boundary_error` | Apply a boundary condition at the wrong node | Boundary value or spatial profile is incorrect | `wave1d/dirichlet-wrong-node` | +| `discretisation_error` | Omit the square on the Courant number | Wave amplitude or propagation speed is wrong | `wave1d/courant-not-squared` | +| `stability_error` | Change the sign in a leapfrog recurrence | Numerical error grows exponentially with time | `wave1d/unstable-time-recurrence` | +| `unit_error` | Round the astronomical-unit conversion | Orbital trajectory has a systematic scale error | `kepler/unit-error-au-rounding` | +| `non_conservative_update` | Dampen a velocity update | Energy drifts over an otherwise conservative trajectory | `kepler/non-conservative-velocity-damping`, `oscillator/non-conservative-damping` | +| `hard_coded_shortcut` | Special-case public fixture sizes | Correct-looking public fixtures, incorrect general inputs | `oscillator/hard-coded-fixture-shortcut` | +| `precision_defect` | Round position updates through `float32` | Small per-step errors accumulate | `oscillator/float32-position` | +| `termination_defect` | Stop integration before the requested final step | Output ends early or repeats a stale state | `oscillator/early-termination` | + +Heat1d has no package mutants yet; adding them is tracked by +[issue #90](https://github.com/akurkar07/InvariantLab/issues/90). + +`validate_reference` and `validate_mutant` enforce four checks: + +1. **Reference:** public and scientific suites each collect at least one test and have no + failures, errors, or skips. +2. **Public profile:** a mutant's public suite has no failures or errors and at least one + passing test. +3. **Expected scientific reason:** the failing scientific node ids equal the declared + `expected_failures`; each is a JUnit failure (not an error or skip) whose message matches + its declared regex. Any error fails validation. +4. **Controlled defect:** mutant source differs from `src/solver.py`, its unified diff stays + within `max_changed_lines` added plus removed lines, it parses, and it imports no + `invariantlab` modules. + +`scripts/validate_mutants.py` runs these checks in CI. See +[Add a mutation](experiment-authoring.md#add-a-mutation) for the manifest schema and +validation command. + ## Evaluation protocol V1 measures **one-shot repair of a mutated solver**. The implemented runner is diff --git a/tests/acceptance/test_mutation_coverage.py b/tests/acceptance/test_mutation_coverage.py new file mode 100644 index 0000000..b6d66d7 --- /dev/null +++ b/tests/acceptance/test_mutation_coverage.py @@ -0,0 +1,49 @@ +"""Every MutationFamily is realised by at least one registered package mutant.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from invariantlab.mutations import discover_mutants +from invariantlab.schema import MutationFamily + +REPO_ROOT = Path(__file__).resolve().parents[2] +# Families with no feasible package mutant must cite the issue where infeasibility was found. +INFEASIBLE_FAMILIES: dict[MutationFamily, str] = {} + + +@pytest.fixture +def covered_families(monkeypatch: pytest.MonkeyPatch) -> set[MutationFamily]: + monkeypatch.chdir(REPO_ROOT) + return {mutant.definition.family for mutant in discover_mutants(Path("tasks"))} + + +@pytest.mark.parametrize("family", list(MutationFamily), ids=lambda family: family.value) +def test_every_mutation_family_has_a_mutant( + family: MutationFamily, covered_families: set[MutationFamily] +) -> None: + assert family in covered_families or family in INFEASIBLE_FAMILIES, ( + f"{family.value!r} has no package mutant; add one or cite its infeasibility issue " + "in INFEASIBLE_FAMILIES." + ) + + +def test_infeasible_families_have_no_mutants( + covered_families: set[MutationFamily], +) -> None: + overlap = covered_families.intersection(INFEASIBLE_FAMILIES) + assert not overlap, ( + f"infeasible families already have mutants: {sorted(f.value for f in overlap)}" + ) + + +def test_catalogue_lists_every_package_mutant(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(REPO_ROOT) + catalogue = (REPO_ROOT / "docs" / "experiment-authoring.md").read_text(encoding="utf-8") + for mutant in discover_mutants(Path("tasks")): + row = f"| {mutant.task_dir.name} | `{mutant.definition.id}` |" + assert row in catalogue, ( + f"catalogue is missing {mutant.task_dir.name}/{mutant.definition.id}" + ) From da2edd37c16815cbe51455b81dc780ae9b55445c Mon Sep 17 00:00:00 2001 From: Alex Kurkar Date: Tue, 6 Oct 2026 07:32:31 +0000 Subject: [PATCH 6/6] Update mutant catalogue for heat1d mutants from #178 Co-Authored-By: Devin AI --- README.md | 3 +-- docs/limitations.md | 4 ++-- docs/methodology.md | 8 ++++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index a9e0276..3a669d1 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ What is implemented on `main` today. The [V1 design target](docs/methodology.md) | Task packages (4: oscillator, kepler, heat1d, wave1d) | Implemented | `tasks/` | [M2](https://github.com/akurkar07/InvariantLab/milestone/1) | | Reference solvers and oracles | Implemented | `src/invariantlab/verification/` | [M2](https://github.com/akurkar07/InvariantLab/milestone/1) | | Verification: Layers 0-6 (seven layers) and `invariantlab verify` | Implemented ([layer status](docs/verification.md#layer-status)) | `src/invariantlab/verification/` | [#110](https://github.com/akurkar07/InvariantLab/issues/110)-[#117](https://github.com/akurkar07/InvariantLab/issues/117) | -| Defect injection / mutants | Partial (registry and validation; 11 package mutants covering all 10 defect families on oscillator, kepler and wave1d; two legacy oscillator study mutants; no heat1d mutants yet; [catalogue](docs/methodology.md#controlled-defect-injection)) | `src/invariantlab/mutations/`, `tasks/*/mutations/` | [M4](https://github.com/akurkar07/InvariantLab/milestone/3) | +| Defect injection / mutants | Partial (registry and validation; 13 package mutants covering all 10 defect families on all four tasks; two legacy oscillator study mutants; [catalogue](docs/methodology.md#controlled-defect-injection)) | `src/invariantlab/mutations/`, `tasks/*/mutations/` | [M4](https://github.com/akurkar07/InvariantLab/milestone/3) | | Model adapters | Implemented | `src/invariantlab/models/adapter.py` | [M5](https://github.com/akurkar07/InvariantLab/milestone/4) | | Repair runner + `audit-run` | Implemented | `src/invariantlab/experiments/repair.py` | [M5](https://github.com/akurkar07/InvariantLab/milestone/4) | | Reporting / dashboard / HF export | Partial (`report`, static `report.html`, local HF export via `export-hf`) | `src/invariantlab/reporting/` | [#106](https://github.com/akurkar07/InvariantLab/issues/106), [#107](https://github.com/akurkar07/InvariantLab/issues/107) | @@ -100,7 +100,6 @@ It currently provides: It does **not** provide yet: -- controlled mutants for heat1d ([issue #90](https://github.com/akurkar07/InvariantLab/issues/90)) - plots, interactive filtering or a served dashboard (the static `report.html` is the V1 dashboard) ## Empirical results diff --git a/docs/limitations.md b/docs/limitations.md index fa866de..394c0e3 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -31,8 +31,8 @@ that tracks it. The [CHANGELOG](../CHANGELOG.md) links this page from its `update-order` / `sign-error` mutants are still `interface: legacy_study` and are graded by the legacy oscillator study harness (`tasks/oscillator/candidate_runner.py` + `verifier.py`). `task_suite` is still ignored. -- **Mutant coverage.** Package mutants exist for `oscillator`, `kepler` and `wave1d`; `heat1d` - has none yet. Covered families: `sign_error`, `update_order_error`, +- **Mutant coverage.** Package mutants exist for all four tasks; `heat1d` has no + `boundary_error` mutant (any wall defect fails its public example). Covered families: `sign_error`, `update_order_error`, `non_conservative_update`, `unit_error`, `hard_coded_shortcut`, `precision_defect`, `termination_defect`, `discretisation_error`, `boundary_error` and `stability_error`. - **Reporting.** The static `report.html` is the V1 dashboard: no plots, interactive diff --git a/docs/methodology.md b/docs/methodology.md index 1110425..e992c30 100644 --- a/docs/methodology.md +++ b/docs/methodology.md @@ -58,16 +58,16 @@ AST/programmatic operators. | `sign_error` | Reverse a force or derivative sign | Trajectory evolves in the wrong direction | `wave1d/sign-error-startup` | | `update_order_error` | Overwrite the previous state before the next step | Recurrence uses a stale or incorrect state | `wave1d/update-order-overwrite` | | `boundary_error` | Apply a boundary condition at the wrong node | Boundary value or spatial profile is incorrect | `wave1d/dirichlet-wrong-node` | -| `discretisation_error` | Omit the square on the Courant number | Wave amplitude or propagation speed is wrong | `wave1d/courant-not-squared` | -| `stability_error` | Change the sign in a leapfrog recurrence | Numerical error grows exponentially with time | `wave1d/unstable-time-recurrence` | +| `discretisation_error` | Omit the square on the Courant number | Wave amplitude or propagation speed is wrong | `wave1d/courant-not-squared`, `heat1d/discretisation-grid-spacing` | +| `stability_error` | Change the sign in a leapfrog recurrence | Numerical error grows exponentially with time | `wave1d/unstable-time-recurrence`, `heat1d/sixth-order-stencil-ftcs-limit` | | `unit_error` | Round the astronomical-unit conversion | Orbital trajectory has a systematic scale error | `kepler/unit-error-au-rounding` | | `non_conservative_update` | Dampen a velocity update | Energy drifts over an otherwise conservative trajectory | `kepler/non-conservative-velocity-damping`, `oscillator/non-conservative-damping` | | `hard_coded_shortcut` | Special-case public fixture sizes | Correct-looking public fixtures, incorrect general inputs | `oscillator/hard-coded-fixture-shortcut` | | `precision_defect` | Round position updates through `float32` | Small per-step errors accumulate | `oscillator/float32-position` | | `termination_defect` | Stop integration before the requested final step | Output ends early or repeats a stale state | `oscillator/early-termination` | -Heat1d has no package mutants yet; adding them is tracked by -[issue #90](https://github.com/akurkar07/InvariantLab/issues/90). +Heat1d has no `boundary_error` mutant: any wall defect fails its public example, so that +family is covered on wave1d ([issue #90](https://github.com/akurkar07/InvariantLab/issues/90)). `validate_reference` and `validate_mutant` enforce four checks: