diff --git a/mypy/test/helpers.py b/mypy/test/helpers.py index fbd3b8f2aac32..60fea32a637fc 100644 --- a/mypy/test/helpers.py +++ b/mypy/test/helpers.py @@ -8,6 +8,7 @@ import shutil import sys import time +from collections import defaultdict from collections.abc import Callable, Iterable, Iterator from re import Pattern from typing import IO, Any @@ -106,14 +107,64 @@ def render_diff_range( output.write("\n") +def dump_original_errors(errors: list[str]) -> None: + sys.stderr.write("Original errors:\n") + for err in errors: + sys.stderr.write(err + "\n") + + +def module_order(errors: list[str]) -> list[str]: + result = [] + seen = set() + mods = [] + for e in errors: + if ":" not in e: + dump_original_errors(errors) + pytest.fail(f"Only module scoped errors are supported, got {e}") + mod, _ = e.split(":", maxsplit=1) + mods.append(mod) + for i, mod in enumerate(mods): + if i > 0: + if mod != mods[i - 1] and mod in seen: + dump_original_errors(errors) + pytest.fail(f"Each module must form a single block, {mod} appears split") + if mod not in seen: + result.append(mod) + seen.add(mod) + return result + + +def match_module_order(actual: list[str], expected_order: list[str]) -> list[str]: + actual_by_mod = defaultdict(list) + actual_order = module_order(actual) + if set(actual_order) != set(expected_order): + # Different files, give up and show actual errors. + return actual + for a in actual: + mod, _ = a.split(":", maxsplit=1) + actual_by_mod[mod].append(a) + result = [] + for mod in expected_order: + result.extend(actual_by_mod[mod]) + return result + + def assert_string_arrays_equal( - expected: list[str], actual: list[str], msg: str, *, traceback: bool = False + expected: list[str], + actual: list[str], + msg: str, + *, + traceback: bool = False, + ignore_modules_order: bool = False, ) -> None: """Assert that two string arrays are equal. Display any differences in a human-readable form. """ actual = clean_up(actual) + if ignore_modules_order: + expected_module_order = module_order(expected) + actual = match_module_order(actual, expected_module_order) if expected != actual: expected_ranges, actual_ranges = diff_ranges(expected, actual) sys.stderr.write("Expected:\n") diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index 16eb178f71f5e..f2bd21b87debf 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -88,7 +88,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: for step in range(1, num_steps + 1): idx = step - 2 ops = steps[idx] if idx < len(steps) and idx >= 0 else [] - self.run_case_once(testcase, ops, step) + self.run_case_once(testcase, ops, step, True) else: self.run_case_once(testcase) @@ -110,6 +110,7 @@ def run_case_once( testcase: DataDrivenTestCase, operations: list[FileOperation] | None = None, incremental_step: int = 0, + is_incremental: bool = False, ) -> None: if operations is None: operations = [] @@ -227,11 +228,18 @@ def run_case_once( if output != a and testcase.config.getoption("--update-data", False): update_testcase_output(testcase, a, incremental_step=incremental_step) + ignore_modules_order = False if options.num_workers > 0: + ignore_modules_order = is_incremental # TypeVarIds are not stable in parallel checking, normalize. a = remove_typevar_ids(a) output = remove_typevar_ids(output) - assert_string_arrays_equal(output, a, msg.format(testcase.file, testcase.line)) + assert_string_arrays_equal( + output, + a, + msg.format(testcase.file, testcase.line), + ignore_modules_order=ignore_modules_order, + ) if res: if options.cache_dir != os.devnull: diff --git a/test-data/unit/check-incremental.test b/test-data/unit/check-incremental.test index 5a2933c68b88e..e7d8928c2a64d 100644 --- a/test-data/unit/check-incremental.test +++ b/test-data/unit/check-incremental.test @@ -1090,9 +1090,7 @@ import foo # type: ignore [builtins fixtures/module.pyi] [stale] --- Order of processing of files is not stable in this test, skip it in parallel mode. --- Note: do not use _no_parallel unless really needed! -[case testIncrementalWithSilentImportsAndIgnore_no_parallel] +[case testIncrementalWithSilentImportsAndIgnore] # cmd: mypy -m main b # cmd2: mypy -m main c c.submodule # flags: --follow-imports=skip @@ -1163,8 +1161,7 @@ class A: [out1] main:2: error: "A" has no attribute "bar" --- Order of files unstable in parallel mode -[case testIncrementalChangedError_no_parallel] +[case testIncrementalChangedError] import m [file m.py] import n @@ -1326,8 +1323,7 @@ tmp/main.py:4: note: Revealed type is "builtins.str" [out2] tmp/main.py:4: note: Revealed type is "Any" --- Order of files unstable in parallel mode -[case testIncrementalFixedBugCausesPropagation_no_parallel] +[case testIncrementalFixedBugCausesPropagation] import mod1 [file mod1.py] @@ -1367,8 +1363,7 @@ tmp/mod1.py:3: note: Revealed type is "builtins.int" [out2] tmp/mod1.py:3: note: Revealed type is "builtins.int" --- Order of files unstable in parallel mode -[case testIncrementalIncidentalChangeWithBugCausesPropagation_no_parallel] +[case testIncrementalIncidentalChangeWithBugCausesPropagation] import mod1 [file mod1.py] @@ -1407,8 +1402,7 @@ tmp/mod1.py:3: note: Revealed type is "builtins.int" tmp/mod3.py:6: error: Incompatible types in assignment (expression has type "str", variable has type "int") tmp/mod1.py:3: note: Revealed type is "builtins.str" --- Order of files unstable in parallel mode -[case testIncrementalIncidentalChangeWithBugFixCausesPropagation_no_parallel] +[case testIncrementalIncidentalChangeWithBugFixCausesPropagation] import mod1 [file mod1.py] @@ -1976,8 +1970,7 @@ main:2: error: Name "nonexisting" is not defined [out2] main:2: error: Name "nonexisting" is not defined --- Order of files unstable in parallel mode -[case testIncrementalInnerClassAttrInMethodReveal_no_parallel] +[case testIncrementalInnerClassAttrInMethodReveal] import crash reveal_type(crash.C().a) reveal_type(crash.D().a) @@ -4089,8 +4082,7 @@ main:2: error: Argument 2 to "B" has incompatible type "str"; expected "int" [out2] [rechecked b] --- Order of files unstable in parallel mode -[case testIncrementalDataclassesThreeFiles_no_parallel] +[case testIncrementalDataclassesThreeFiles] from c import C C('foo', 5, True) @@ -5412,8 +5404,7 @@ tmp/c.py:2: note: Revealed type is "b." [out2] tmp/c.py:2: note: Revealed type is "b." --- Order of files unstable in parallel mode -[case testIsInstanceAdHocIntersectionIncrementalIntersectionToUnreachable_no_parallel] +[case testIsInstanceAdHocIntersectionIncrementalIntersectionToUnreachable] import c [file a.py] class A: @@ -5447,8 +5438,7 @@ tmp/c.py:2: note: Revealed type is "a." tmp/b.py:2: error: Cannot determine type of "y" tmp/c.py:2: note: Revealed type is "Any" --- Order of files unstable in parallel mode -[case testIsInstanceAdHocIntersectionIncrementalUnreachaableToIntersection_no_parallel] +[case testIsInstanceAdHocIntersectionIncrementalUnreachaableToIntersection] import c [file a.py] class A: