From 5403c50dec678c279299c1a4e1df1433ab4e3792 Mon Sep 17 00:00:00 2001 From: Donncha O'Toole Date: Wed, 28 Oct 2026 11:29:42 +0000 Subject: [PATCH] Add targeted mutation gate for scientific invariants --- CHANGELOG.md | 2 +- Makefile | 10 +- docs/scientific-reliability.md | 9 +- pyproject.toml | 2 + python/tests/test_scientific_fuzz.py | 6 +- python/tests/test_scientific_invariants.py | 38 +++++++ scripts/check_mutation_sensitivity.py | 126 +++++++++++++++++++++ 7 files changed, 184 insertions(+), 9 deletions(-) create mode 100755 scripts/check_mutation_sensitivity.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f76b220..e8a54f01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ All notable user-facing changes are tracked here. Public statements in release n - Strict typed Python surface: `py.typed`, `MatchStatus`, `Literal` policies, `TypedDict` summaries, a typed `TargetRow`, and mypy strict and pyright gates (`make typecheck`). - Example and snapshot testing flow: executable doctests, Rust `expect-test`-style snapshots, static API contract tests, and a typed quickstart (`make examples-test`). - The whole `dotmatch`/`quickdna` package now passes the strict typed gate, with Hypothesis invariants and a Python 3.9 runtime compatibility check (`make compat39`). -- Scientific reliability gate: scalar oracle equivalence across scan/indexed/exact/Hamming policies, stream count conservation, lossless integer count round-trips above float precision, gzip/plain FASTQ equivalence, seeded panel determinism, FASTQ/count/AssaySpec fuzzing, byte-deterministic assignment/panel/feature-matrix artifacts, indexed-vs-scan agreement, panel safety-vs-distance checks, unique-only feature-matrix counts, sensitivity artifact determinism, and mutation-sensitivity cases (`make scientific-tests`). See `docs/scientific-reliability.md`. +- Scientific reliability gate: scalar oracle equivalence across scan/indexed/exact/Hamming policies, stream count conservation, lossless integer count round-trips above float precision, gzip/plain FASTQ equivalence, seeded panel determinism, FASTQ/count/AssaySpec fuzzing, byte-deterministic assignment/panel/feature-matrix artifacts, indexed-vs-scan agreement, panel safety-vs-distance checks, unique-only feature-matrix counts, sensitivity artifact determinism, targeted mutation tests that must kill four known-wrong semantic changes (`make mutation-scientific`), and mutation-sensitivity cases. See `docs/scientific-reliability.md`. ### Changed - Assignment result records (`MatchResult`, `AssignmentStats`, `PosteriorAssignment`, `FastqRecord`, `StreamAssignment`) are immutable `NamedTuple`s. Field access and keyword construction are unchanged. They are now tuple-compatible, so equality with a plain tuple is true; `dataclasses.asdict`, `fields`, and `replace` no longer apply. diff --git a/Makefile b/Makefile index a21aa46c..bb0997eb 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,8 @@ TYPED_PYTHON_SOURCES ?= \ python/tests/test_scientific_invariants.py \ python/tests/test_scientific_fuzz.py \ examples/python/typed_quickstart.py \ - scripts/bench_python_api.py + scripts/bench_python_api.py \ + scripts/check_mutation_sensitivity.py TYPE_PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,$(PYTHON)) PYTHON39 ?= /usr/bin/python3 CC ?= cc @@ -46,7 +47,7 @@ DOTMATCH_SHARED_FLAGS := -shared QDALIGN_SHARED_FLAGS := -shared endif -.PHONY: all clean test cli-test coverage bench bench-batch bench-small bench-native-matrix bench-10x-control figures bench-real-report bench-gpu gpu-report gpu-evidence-gate bench-barcode-demux bench-barcode-panel-design bench-barcode-demux-competitors bench-barcode-comparison barcode-autopsy-demo barcode-panel-design-gate barcode-validation-ready barcode-comparison-report barcode-comparison-gate barcode-demux-report barcode-competitor-env fetch-barcode-demo fetch-barcode-demo-claim fetch-sanson-crispr fetch-10x-bcl-demo bench-bcl-small bench-bcl-10x bench-bcl-real bench-bcl-real-repeated bcl-figures bcl-competitor-env bcl-linux-env bcl-tiny-public-gate bcl-comparison-gate fetch-oligo-adapter-demo bench-oligo-adapter bench-oligo-adapter-public oligo-adapter-smoke-gate oligo-adapter-public-gate fetch-amplicon-panel-demo bench-amplicon-panel bench-amplicon-panel-public amplicon-panel-smoke-gate amplicon-panel-public-gate fetch-feature-barcode-demo bench-feature-barcode bench-feature-barcode-public feature-barcode-smoke-gate feature-barcode-public-gate fetch-perturb-seq-demo bench-perturb-seq bench-perturb-seq-public perturb-seq-smoke-gate perturb-seq-public-gate bench-perturb-seq-case-study-public perturb-seq-case-study-fixture-gate perturb-seq-case-study-public-gate bench-public-crispr-small bench-public-crispr bench-public-crispr-competitors bench-public-crispr-repeated bench-public-crispr-scaling bench-real-competitors bench-crispr-comparison bench-crispr-hamming-k23 crispr-comparison-report crispr-comparison-gate count-agreement count-agreement-comparison validate-public-crispr-edlib validate-crispr-comparison-edlib public-crispr-report public-crispr-evidence-gate public-crispr-smoke-gate competitor-env edlib edlib-tools bench-edlib-native benchmark-report benchmark-report-native native-exact-gate evidence-gallery evidence-gallery-ready repro repro-small asan shared python-test python-package-test typecheck lint examples-test compat39 scientific-tests quality workflow-integration-test docs-ready repository-ready release-ready pretag-ready assay-evidence-ready scientific-readiness-ready alphabet-policy-ready citation-metadata-ready native-comparator-scope-ready workflow-examples-ready workflow-adoption-status reviewer-readiness-ready distribution-record-ready bioconda-recipe-ready distribution-channels download-metrics joss-paper-ready agent-discovery-ready +.PHONY: all clean test cli-test coverage bench bench-batch bench-small bench-native-matrix bench-10x-control figures bench-real-report bench-gpu gpu-report gpu-evidence-gate bench-barcode-demux bench-barcode-panel-design bench-barcode-demux-competitors bench-barcode-comparison barcode-autopsy-demo barcode-panel-design-gate barcode-validation-ready barcode-comparison-report barcode-comparison-gate barcode-demux-report barcode-competitor-env fetch-barcode-demo fetch-barcode-demo-claim fetch-sanson-crispr fetch-10x-bcl-demo bench-bcl-small bench-bcl-10x bench-bcl-real bench-bcl-real-repeated bcl-figures bcl-competitor-env bcl-linux-env bcl-tiny-public-gate bcl-comparison-gate fetch-oligo-adapter-demo bench-oligo-adapter bench-oligo-adapter-public oligo-adapter-smoke-gate oligo-adapter-public-gate fetch-amplicon-panel-demo bench-amplicon-panel bench-amplicon-panel-public amplicon-panel-smoke-gate amplicon-panel-public-gate fetch-feature-barcode-demo bench-feature-barcode bench-feature-barcode-public feature-barcode-smoke-gate feature-barcode-public-gate fetch-perturb-seq-demo bench-perturb-seq bench-perturb-seq-public perturb-seq-smoke-gate perturb-seq-public-gate bench-perturb-seq-case-study-public perturb-seq-case-study-fixture-gate perturb-seq-case-study-public-gate bench-public-crispr-small bench-public-crispr bench-public-crispr-competitors bench-public-crispr-repeated bench-public-crispr-scaling bench-real-competitors bench-crispr-comparison bench-crispr-hamming-k23 crispr-comparison-report crispr-comparison-gate count-agreement count-agreement-comparison validate-public-crispr-edlib validate-crispr-comparison-edlib public-crispr-report public-crispr-evidence-gate public-crispr-smoke-gate competitor-env edlib edlib-tools bench-edlib-native benchmark-report benchmark-report-native native-exact-gate evidence-gallery evidence-gallery-ready repro repro-small asan shared python-test python-package-test typecheck lint examples-test compat39 scientific-tests mutation-scientific quality workflow-integration-test docs-ready repository-ready release-ready pretag-ready assay-evidence-ready scientific-readiness-ready alphabet-policy-ready citation-metadata-ready native-comparator-scope-ready workflow-examples-ready workflow-adoption-status reviewer-readiness-ready distribution-record-ready bioconda-recipe-ready distribution-channels download-metrics joss-paper-ready agent-discovery-ready all: dotmatch libdotmatch.a qda libqdalign.a @@ -502,7 +503,10 @@ compat39: shared scientific-tests: shared DOTMATCH_LIB=$(CURDIR)/libdotmatch.$(SHARED_EXT) PYTHONPATH=$(CURDIR)/python $(TYPE_PYTHON) -m pytest -m scientific python/tests -quality: typecheck lint scientific-tests python-test compat39 +mutation-scientific: shared + DOTMATCH_LIB=$(CURDIR)/libdotmatch.$(SHARED_EXT) PYTHONPATH=$(CURDIR)/python $(TYPE_PYTHON) scripts/check_mutation_sensitivity.py + +quality: typecheck lint scientific-tests mutation-scientific python-test compat39 python-package-test: python3 scripts/check_python_wheel.py diff --git a/docs/scientific-reliability.md b/docs/scientific-reliability.md index c1c03fbb..af77f3fb 100644 --- a/docs/scientific-reliability.md +++ b/docs/scientific-reliability.md @@ -44,8 +44,9 @@ protects that contract; it does not by itself prove biological correctness. ## Enforced by ```bash -make scientific-tests # oracle, conservation, lossless counts, determinism -make quality # typecheck + lint + scientific tests + full tests + 3.9 +make scientific-tests # oracle, conservation, lossless counts, determinism, fuzzing +make mutation-scientific # four known-wrong semantic mutants must fail the tests +make quality # typecheck + lint + scientific/mutation gates + full tests + 3.9 make python-test # full Python suite ``` @@ -57,6 +58,10 @@ Key files: - `python/tests/test_scientific_fuzz.py`: FASTQ/count/AssaySpec fuzzing, deterministic assignment/panel/feature-matrix artifacts, and mutation-sensitivity cases. +- `scripts/check_mutation_sensitivity.py`: applies four targeted source + mutations in throwaway package copies (radius-as-unique, disabled FASTQ + validation, float count parsing, ambiguous feature counting) and requires the + scientific tests to fail for each. - `python/tests/test_native_oracle_audit.py`: independent distance and policy oracle at length boundaries. - `python/tests/test_input_validation_boundaries.py`: FASTQ and count parsing diff --git a/pyproject.toml b/pyproject.toml index 21d6ddd2..0fb12b58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,6 +121,7 @@ files = [ "python/tests/test_scientific_fuzz.py", "examples/python/typed_quickstart.py", "scripts/bench_python_api.py", + "scripts/check_mutation_sensitivity.py", ] mypy_path = "python" strict = true @@ -144,6 +145,7 @@ include = [ "python/tests/test_scientific_fuzz.py", "examples/python/typed_quickstart.py", "scripts/bench_python_api.py", + "scripts/check_mutation_sensitivity.py", ] pythonVersion = "3.9" typeCheckingMode = "standard" diff --git a/python/tests/test_scientific_fuzz.py b/python/tests/test_scientific_fuzz.py index 7359e65c..3bdfe1bf 100644 --- a/python/tests/test_scientific_fuzz.py +++ b/python/tests/test_scientific_fuzz.py @@ -75,11 +75,11 @@ def test_corrupt_fastq_is_rejected( elif corruption == "length": quality += "I" elif corruption == "control": - sequence = sequence[:1] + "\x01" + sequence[1:] + sequence = "\x01" + sequence[1:] elif corruption == "space": - quality = quality[:1] + " " + quality[1:] + quality = " " + quality[1:] elif corruption == "non-ascii": - sequence = sequence[:1] + "\u00e9" + sequence[1:] + sequence = "\u00e9" + sequence[1:] else: plus = "+some_other_id" text = f"{header}\n{sequence}\n{plus}\n{quality}\n" diff --git a/python/tests/test_scientific_invariants.py b/python/tests/test_scientific_invariants.py index 001c622d..81862dcb 100644 --- a/python/tests/test_scientific_invariants.py +++ b/python/tests/test_scientific_invariants.py @@ -359,3 +359,41 @@ def test_sensitivity_artifacts_are_deterministic_except_wall_clock(tmp_path: Pat if name == "summary.json": continue assert (tmp_path / "first" / name).read_bytes() == (tmp_path / "second" / name).read_bytes(), name + + +def test_feature_matrix_excludes_ambiguous_counts(tmp_path: Path) -> None: + targets = tmp_path / "targets.tsv" + targets.write_text("target_id\ttarget_seq\nA\tACGT\nB\tACGA\nC\tTTTT\n", encoding="utf-8") + observations = tmp_path / "observations.tsv" + observations.write_text( + "observation_id\tcell_barcode\tfeature_seq\n" + "o1\tcell_ambiguous\tACGT\n" + "o2\tcell_unique\tTTTT\n", + encoding="utf-8", + ) + result = build_feature_matrix( + observations, + targets, + tmp_path / "matrix", + cell_column="cell_barcode", + sequence_column="feature_seq", + id_column="observation_id", + k=1, + metric="hamming", + ) + rows = list( + csv.DictReader( + (result.output_dir / "cell_feature_counts.tsv").open(encoding="utf-8"), + delimiter="\t", + ) + ) + + assert rows == [ + { + "cell_barcode": "cell_unique", + "target_id": "C", + "count": "1", + } + ] + assert result.summary["ambiguous"] == 1 + assert result.summary["assigned_unique"] == 1 diff --git a/scripts/check_mutation_sensitivity.py b/scripts/check_mutation_sensitivity.py new file mode 100755 index 00000000..14b2b8aa --- /dev/null +++ b/scripts/check_mutation_sensitivity.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Targeted mutation gate for the scientific invariants. + +Each mutant is applied to a throwaway copy of the package, then the scientific +tests must fail. A surviving mutant means the invariant tests do not distinguish +a known-wrong semantic change from correct behaviour. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +TEST_FILES = ("test_scientific_invariants.py", "test_scientific_fuzz.py") + + +@dataclass(frozen=True) +class Mutant: + name: str + relative_path: str + before: str + after: str + + +MUTANTS = ( + Mutant( + "radius-becomes-unique", + "core.py", + " MATCH_AMBIGUOUS,", + " MATCH_UNIQUE,", + ), + Mutant( + "fastq-validation-disabled", + "fastq_io.py", + " if not _valid_printable_ascii(seq):", + " if False:", + ), + Mutant( + "counts-parse-through-float", + "count_io.py", + " return int(text)", + " return int(float(text))", + ), + Mutant( + "ambiguous-features-counted", + "feature_matrix.py", + " if status == MATCH_UNIQUE and target_id:\n counts[(observation.cell_barcode, target_id)] += 1", + " if status in {MATCH_UNIQUE, MATCH_AMBIGUOUS}:\n counts[(observation.cell_barcode, target_id or f\"ambiguous_{result.target_index}\")] += 1", + ), +) + + +def _copy_source(destination: Path) -> None: + shutil.copytree(ROOT / "python" / "dotmatch", destination / "python" / "dotmatch") + shutil.copytree(ROOT / "python" / "quickdna", destination / "python" / "quickdna") + (destination / "tests").mkdir() + for name in TEST_FILES: + shutil.copy2(ROOT / "python" / "tests" / name, destination / "tests" / name) + + +def _library_path() -> Path: + for name in ("libdotmatch.dylib", "libdotmatch.so"): + candidate = ROOT / name + if candidate.is_file(): + return candidate + raise SystemExit("build the shared library first: make shared") + + +def _run_mutant(mutant: Mutant, library: Path) -> bool: + with tempfile.TemporaryDirectory(prefix="dotmatch-mutant-") as temporary: + root = Path(temporary) + _copy_source(root) + target = root / "python" / "dotmatch" / mutant.relative_path + source = target.read_text(encoding="utf-8") + if mutant.before not in source: + raise SystemExit(f"mutant {mutant.name}: source pattern not found in {mutant.relative_path}") + target.write_text(source.replace(mutant.before, mutant.after, 1), encoding="utf-8") + env = { + **os.environ, + "PYTHONPATH": str(root / "python"), + "DOTMATCH_LIB": str(library), + } + completed = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + str(root / "tests"), + "-q", + "-c", + os.devnull, + "-p", + "no:cacheprovider", + ], + cwd=root, + env=env, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if completed.returncode == 0: + print(f"survived: {mutant.name}", file=sys.stderr) + print(completed.stdout[-4000:], file=sys.stderr) + return False + print(f"killed: {mutant.name}") + return True + + +def main() -> int: + library = _library_path() + killed = [_run_mutant(mutant, library) for mutant in MUTANTS] + if not all(killed): + return 1 + print(f"all {len(MUTANTS)} targeted scientific mutants were killed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())