diff --git a/docs/cli.rst b/docs/cli.rst index 7cfa4f29d..f4abe491c 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -117,6 +117,11 @@ previous results and recent code changes to achieve two main goals: has a history of failing, and no relevant code changes have been made, so it is assumed to fail again and is run first. 2. Load balancing - When tests are executed in parallel using the ``-p`` option, VUnit distributes them across threads to minimize to total execution time. +With ``--parallel-compile``, as many files as ``-p`` gives are also compiled in parallel. A file is compiled once the +files it depends on are compiled. Most simulators do not lock a library while they update it, so a file is not compiled +while another file is compiled into its library, or into a library it reads. NVC locks its libraries and compiles any +files at the same time. Without ``--parallel-compile``, files are compiled one at a time. + Opening a Test Case in Simulator GUI ==================================== diff --git a/docs/news.d/1235.feature.rst b/docs/news.d/1235.feature.rst new file mode 100644 index 000000000..39e2fd906 --- /dev/null +++ b/docs/news.d/1235.feature.rst @@ -0,0 +1,3 @@ +Added ``--parallel-compile``, which compiles as many files in parallel as ``-p`` gives: a file is compiled once the files +it depends on are compiled. Unless the simulator locks its libraries, as NVC does, a file is not compiled while its +library, or a library it reads, is compiled into. diff --git a/tests/unit/test_ghdl_interface.py b/tests/unit/test_ghdl_interface.py index e72fa8967..dc37301d5 100644 --- a/tests/unit/test_ghdl_interface.py +++ b/tests/unit/test_ghdl_interface.py @@ -137,6 +137,19 @@ def test_assertion_on_unknown_backend(self, check_output): check_output.return_value = version self.assertRaises(AssertionError, GHDLInterface.determine_backend, "prefix") + @mock.patch.object(GHDLInterface, "determine_version", return_value=5.0) + def test_setup_library_mapping_creates_libraries(self, determine_version): + simif = GHDLInterface(prefix="prefix", output_path="") + project = Project() + project.add_library("lib", "lib_path") + write_file(str(Path("existing_path") / "existing-obj08.cf"), "") + project.add_library("existing", "existing_path") + simif.setup_library_mapping(project) + self.create_library.assert_called_once_with( + [str(Path("prefix") / "ghdl"), "-i", "--workdir=lib_path", "--work=lib", "--std=08"], + env=simif.get_env(), + ) + @mock.patch("vunit.sim_if.check_output", autospec=True, return_value="") # pylint: disable=no-self-use @mock.patch.object(GHDLInterface, "determine_version", return_value=6.0) def test_compile_project_2019(self, determine_version, check_output): @@ -312,6 +325,9 @@ def test_compile_project_verilog_error(self, determine_version): def setUp(self): self.output_path = str(Path(__file__).parent / "test_ghdl_interface_out") renew_path(self.output_path) + patcher = mock.patch("vunit.sim_if.ghdl.subprocess.check_call", autospec=True) + self.create_library = patcher.start() + self.addCleanup(patcher.stop) self.project = Project() self.cwd = os.getcwd() os.chdir(self.output_path) diff --git a/tests/unit/test_simulator_interface.py b/tests/unit/test_simulator_interface.py index 6ea155afd..da9c88033 100644 --- a/tests/unit/test_simulator_interface.py +++ b/tests/unit/test_simulator_interface.py @@ -12,6 +12,7 @@ from pathlib import Path from os import chdir, getcwd import subprocess +import threading from shutil import rmtree from unittest import mock from vunit.project import Project @@ -190,6 +191,89 @@ def raise_compile_error(source_file): # pylint: disable=unused-argument self.assertRaises(CompileError, simif.compile_source_files, project) self.assertEqual(project.get_files_in_compile_order(incremental=True), [source_file]) + def _compile_concurrently( + self, libraries, supports_concurrent_compiles_into_library=False, fail=None, dependencies=() + ): + """ + Compile one file per library name in libraries with 4 threads, the compile commands + waiting for each other in pairs. dependencies are (file index, index of the file it depends + on) pairs. Return the printer output and the compile commands that ran at the same time as + another. + """ + simif = create_simulator_interface() + simif.supports_concurrent_compiles_into_library = supports_concurrent_compiles_into_library + simif.compile_source_file_command.side_effect = lambda source_file: [Path(source_file.name).stem] + project, _ = create_project( + [(f"file{idx}.vhd", library) for idx, library in enumerate(libraries)], dependencies + ) + + barrier = threading.Barrier(2, timeout=0.5) + concurrent = set() + + def check_output_side_effect(command, env=None): # pylint: disable=unused-argument + try: + barrier.wait() + concurrent.add(command[0]) + except threading.BrokenBarrierError: + barrier.reset() # Ran alone, let the next compiles meet + if command[0] == fail: + raise subprocess.CalledProcessError(returncode=-1, cmd=command, output="bad stuff") + return command[0] + " output\n" + + with mock.patch("vunit.sim_if.check_output", autospec=True) as check_output: + check_output.side_effect = check_output_side_effect + printer = MockPrinter() + simif.compile_source_files(project, printer=printer, num_threads=4, continue_on_error=True) + return printer.output, concurrent + + def test_compile_source_files_of_different_libraries_concurrently(self): + _, concurrent = self._compile_concurrently(["lib1", "lib2"]) + self.assertEqual(concurrent, {"file0", "file1"}) + + def test_compile_source_files_of_a_library_one_at_a_time(self): + _, concurrent = self._compile_concurrently(["lib", "lib"]) + self.assertEqual(concurrent, set()) + + def test_compile_source_files_of_a_library_concurrently_when_supported(self): + _, concurrent = self._compile_concurrently(["lib", "lib"], supports_concurrent_compiles_into_library=True) + self.assertEqual(concurrent, {"file0", "file1"}) + + def test_compile_source_files_not_while_a_library_they_read_is_compiled_into(self): + # Once file0 is compiled, file1 is compiled into lib1, which file2 reads + _, concurrent = self._compile_concurrently(["lib1", "lib1", "lib2"], dependencies=[(1, 0), (2, 0)]) + self.assertEqual(concurrent, set()) + + def test_compile_source_files_reading_the_same_library_concurrently(self): + _, concurrent = self._compile_concurrently(["lib1", "lib2", "lib3"], dependencies=[(1, 0), (2, 0)]) + self.assertEqual(concurrent, {"file1", "file2"}) + + def test_compile_source_files_concurrently_writes_the_output_of_a_file_at_once(self): + output, _ = self._compile_concurrently(["lib1", "lib2"], fail="file1") + self.assertIn("Compiling into lib1: file0.vhd passed\nfile0 output\n", output) + self.assertIn("Compiling into lib2: file1.vhd failed\n=== Command used: ===\nfile1\n", output) + self.assertTrue(output.endswith("Compile failed\n")) + + def test_compile_source_files_waits_for_dependencies(self): + simif = create_simulator_interface() + simif.compile_source_file_command.side_effect = lambda source_file: [Path(source_file.name).stem] + project, (file1, file2, file3) = create_project( + [("file1.vhd", "lib1"), ("file2.vhd", "lib2"), ("file3.vhd", "lib1")], dependencies=[(2, 1)] + ) + + def check_output_side_effect(command, env=None): # pylint: disable=unused-argument + if command == ["file2"]: + raise subprocess.CalledProcessError(returncode=-1, cmd=command, output="bad stuff") + return "" + + with mock.patch("vunit.sim_if.check_output", autospec=True) as check_output: + check_output.side_effect = check_output_side_effect + printer = MockPrinter() + simif.compile_source_files(project, printer=printer, num_threads=4, continue_on_error=True) + self.assertIn("Compiling into lib1: file3.vhd skipped\n", printer.output) + self.assertEqual(len(check_output.mock_calls), 2) + self.assertEqual(project.get_files_in_compile_order(incremental=True), [file2, file3]) + self.assertNotIn(file1, project.get_files_in_compile_order(incremental=True)) + @mock.patch("os.environ", autospec=True) def test_find_prefix(self, environ): class MySimulatorInterface(SimulatorInterface): # pylint: disable=abstract-method @@ -319,6 +403,23 @@ def create_simulator_interface(): return simif +def create_project(files, dependencies=()): + """ + Create a project of VHDL files, (name, library) pairs. dependencies are (file index, index of + the file it depends on) pairs. Return the project and its source files. + """ + project = Project() + source_files = [] + for name, library in files: + if library not in [existing.name for existing in project.get_libraries()]: + project.add_library(library, library + "_path") + write_file(name, "") + source_files.append(project.add_source_file(name, library, file_type="vhdl")) + for idx, depends_on in dependencies: + project.add_manual_dependency(source_files[idx], depends_on=source_files[depends_on]) + return project, source_files + + class MockPrinter(object): """ Mock printer that accumulates the calls as a string diff --git a/tests/unit/test_ui.py b/tests/unit/test_ui.py index c0584581d..0c76c73f2 100644 --- a/tests/unit/test_ui.py +++ b/tests/unit/test_ui.py @@ -1180,6 +1180,23 @@ def test_can_list_files_without_simulator(self): ui = self._create_ui_real_sim("--files") self._run_main(ui, 0) + def _compile_threads(self, *args): + """ + The number of threads the files of a ui created with args are compiled with + """ + ui = self._create_ui(*args) + simulator_if = mock.Mock(spec=SimulatorInterface) + ui._compile(simulator_if) # pylint: disable=protected-access + return simulator_if.compile_project.call_args.kwargs["num_threads"] + + def test_compile_one_file_at_a_time_without_parallel_compile(self): + self.assertEqual(self._compile_threads("-p", "4"), 1) + self.assertEqual(self._compile_threads("-p", "0"), 1) + + def test_compile_in_parallel_with_parallel_compile(self): + self.assertEqual(self._compile_threads("-p", "4", "--parallel-compile"), 4) + self.assertEqual(self._compile_threads("-p", "0", "--parallel-compile"), 0) + @mock.patch("vunit.ui.LOGGER", autospec=True) def test_compile_without_simulator_fails(self, logger): with set_env(): diff --git a/vunit/sim_if/__init__.py b/vunit/sim_if/__init__.py index 53b2ea2a1..077a53574 100644 --- a/vunit/sim_if/__init__.py +++ b/vunit/sim_if/__init__.py @@ -13,6 +13,8 @@ from os import environ, listdir, pathsep import locale import subprocess +from collections import Counter +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from pathlib import Path from typing import List, Optional from ..ostools import Process, simplify_path @@ -62,6 +64,11 @@ class SimulatorInterface(object): # pylint: disable=too-many-public-methods # True if simulator supports ANSI colors in GUI mode supports_colors_in_gui = False + # True if a library can be compiled into while it is compiled into or read by another + # compile. Most compilers update the index of a library without locking it, so concurrent + # updates lose units and a compile reading it during an update fails. + supports_concurrent_compiles_into_library = False + # The path prefix the interface was built with, set by the interfaces having one _prefix: Optional[str] = None @@ -225,14 +232,18 @@ def compile_project( printer=NO_COLOR_PRINTER, continue_on_error=False, target_files=None, - ): + num_threads=1, + ): # pylint: disable=too-many-arguments """ Compile the project param: target_files: Given a list of SourceFiles only these and dependent files are compiled + param: num_threads: The number of files compiled at the same time, 0 for all logical CPUs """ self.add_simulator_specific(project) self.setup_library_mapping(project) - self.compile_source_files(project, printer, continue_on_error, target_files=target_files) + self.compile_source_files( + project, printer, continue_on_error, target_files=target_files, num_threads=num_threads + ) def simulate(self, output_path, test_suite_name, config, elaborate_only): """ @@ -281,47 +292,28 @@ def compile_source_files( printer=NO_COLOR_PRINTER, continue_on_error=False, target_files=None, - ): + num_threads=1, + ): # pylint: disable=too-many-arguments """ Use compile_source_file_command to compile all source_files param: target_files: Given a list of SourceFiles only these and dependent files are compiled + param: num_threads: The number of files compiled at the same time, 0 for all logical CPUs """ dependency_graph = project.create_dependency_graph() - failures = [] if target_files is None: source_files = project.get_files_in_compile_order(dependency_graph=dependency_graph) else: source_files = project.get_minimal_file_set_in_compile_order(target_files) - source_files_to_skip = set() - - max_library_name = 0 - max_source_file_name = 0 - if source_files: - max_library_name = max(len(source_file.library.name) for source_file in source_files) - max_source_file_name = max(len(simplify_path(source_file.name)) for source_file in source_files) - - for source_file in source_files: - printer.write( - f"Compiling into {(source_file.library.name + ':').ljust(max_library_name + 1)!s} " - f"{simplify_path(source_file.name).ljust(max_source_file_name)!s} " - ) - sys.stdout.flush() - - if source_file in source_files_to_skip: - printer.write("skipped", fg="rgi") - printer.write("\n") - continue - - if self._compile_source_file(source_file, printer): - project.update(source_file) - else: - source_files_to_skip.update(dependency_graph.get_dependent([source_file])) - failures.append(source_file) - - if not continue_on_error: - break + failures = self._compile_in_dependency_order( + project, + source_files, + dependency_graph=dependency_graph, + printer=printer, + continue_on_error=continue_on_error, + num_threads=num_threads or os.cpu_count() or 1, + ) if failures: printer.write("Compile failed\n", fg="ri") @@ -334,6 +326,97 @@ def compile_source_files( else: printer.write("Re-compile not needed\n") + def _compile_in_dependency_order( + self, project, source_files, *, dependency_graph, printer, continue_on_error, num_threads + ): # pylint: disable=too-many-arguments,too-many-locals + """ + Compile source_files, num_threads at a time, and return the files that failed. A file is + compiled once its dependencies are. Unless supports_concurrent_compiles_into_library, it is + not compiled while its library is compiled into or read, or while a library it reads, that of + any of its dependencies, is compiled into. The dependents of a failed file are skipped. + """ + max_library_name = max((len(source_file.library.name) for source_file in source_files), default=0) + max_source_file_name = max((len(simplify_path(source_file.name)) for source_file in source_files), default=0) + + def write_header(source_file, output): + output.write( + f"Compiling into {(source_file.library.name + ':').ljust(max_library_name + 1)!s} " + f"{simplify_path(source_file.name).ljust(max_source_file_name)!s} " + ) + + def compile_source_file(source_file, output): + write_header(source_file, output) + sys.stdout.flush() + return self._compile_source_file(source_file, output) + + to_compile = list(source_files) + dependencies = { + source_file: dependency_graph.get_direct_dependencies(source_file) & set(source_files) + for source_file in source_files + } + # ponytail: a bare library clause, with nothing used from the library, is not a read + libraries_read = { + source_file: {dependency.library.name for dependency in dependency_graph.get_dependencies([source_file])} + for source_file in source_files + } + compiled = {} # Whether a finished file was compiled, rather than failed or skipped + failures = [] + running = {} + libraries_written = Counter() + libraries_being_read = Counter() + + with ThreadPoolExecutor(max_workers=num_threads) as executor: + while to_compile or running: + # ponytail: scans every waiting file per completed one, fine for thousands of files + for source_file in list(to_compile): + if len(running) == num_threads or (failures and not continue_on_error): + break + + if not dependencies[source_file] <= compiled.keys(): + continue + + if not all(compiled[dependency] for dependency in dependencies[source_file]): + to_compile.remove(source_file) + compiled[source_file] = False + write_header(source_file, printer) + printer.write("skipped", fg="rgi") + printer.write("\n") + continue + + library = source_file.library.name + if not self.supports_concurrent_compiles_into_library and ( + libraries_written[library] + or libraries_being_read[library] + or any(libraries_written[read] for read in libraries_read[source_file]) + ): + continue + + to_compile.remove(source_file) + libraries_written[library] += 1 + libraries_being_read.update(libraries_read[source_file]) + # The output of a file is written once it is compiled, so that outputs do not mix + output = printer if num_threads == 1 else _BufferedPrinter() + running[executor.submit(compile_source_file, source_file, output)] = (source_file, output) + + if not running: + break + + done, _ = wait(running, return_when=FIRST_COMPLETED) + for future in done: + source_file, output = running.pop(future) + libraries_written[source_file.library.name] -= 1 + libraries_being_read.subtract(libraries_read[source_file]) + if output is not printer: + output.write_to(printer) + + compiled[source_file] = future.result() + if compiled[source_file]: + project.update(source_file) + else: + failures.append(source_file) + + return failures + def compile_source_file_command(self, source_file): # pylint: disable=unused-argument raise NotImplementedError @@ -344,6 +427,22 @@ def get_env(): """ +class _BufferedPrinter(object): + """ + Printer keeping what is written until it is written to another printer + """ + + def __init__(self): + self._writes = [] + + def write(self, text, output_file=None, fg=None, bg=None): + self._writes.append((text, output_file, fg, bg)) + + def write_to(self, printer): + for text, output_file, fg, bg in self._writes: + printer.write(text, output_file=output_file, fg=fg, bg=bg) + + def isfile(file_name): """ Case insensitive Path.is_file() diff --git a/vunit/sim_if/ghdl.py b/vunit/sim_if/ghdl.py index bb28a5c85..24e6e26f8 100644 --- a/vunit/sim_if/ghdl.py +++ b/vunit/sim_if/ghdl.py @@ -257,6 +257,21 @@ def setup_library_mapping(self, project): else: self._vhdl_standard = list(vhdl_standards)[0] + # A library clause fails on a library nothing has been compiled into, which files compiled + # in parallel may come before, so every library is created first + for library in project.get_libraries(): + if not list(Path(library.directory).glob(f"{library.name}-obj*.cf")): + subprocess.check_call( + [ + str(Path(self._prefix) / self.executable), + "-i", + f"--workdir={library.directory!s}", + f"--work={library.name!s}", + f"--std={self._std_str(self._vhdl_standard)!s}", + ], + env=self.get_env(), + ) + def compile_source_file_command(self, source_file): """ Returns the command to compile a single source_file @@ -445,6 +460,18 @@ def simulate(self, output_path, test_suite_name, config, elaborate_only): # pyl return status + def compile_source_files(self, project, *args, num_threads=1, **kwargs): # pylint: disable=arguments-differ + """ + Compile one file at a time with coverage of the gcc backend, which writes the .gcno file + of every file to the working directory + """ + if self._backend == "gcc" and any( + source_file.compile_options.get("enable_coverage", False) + for source_file in project.get_source_files_in_order() + ): + num_threads = 1 + super().compile_source_files(project, *args, num_threads=num_threads, **kwargs) + def _compile_source_file(self, source_file, printer): """ Runs parent command for compilation, and moves any .gcno files to the compilation output diff --git a/vunit/sim_if/nvc.py b/vunit/sim_if/nvc.py index cc2880384..ee9106fe3 100644 --- a/vunit/sim_if/nvc.py +++ b/vunit/sim_if/nvc.py @@ -36,6 +36,8 @@ class NVCInterface(SimulatorInterface, ViewerMixin): # pylint: disable=too-many executable = environ.get("NVC", "nvc") supports_gui_flag = True supports_colors_in_gui = True + # NVC locks the library while it updates it + supports_concurrent_compiles_into_library = True compile_options = [ ListOfStringOption("nvc.global_flags"), diff --git a/vunit/ui/__init__.py b/vunit/ui/__init__.py index 17503a6c8..346fb3467 100644 --- a/vunit/ui/__init__.py +++ b/vunit/ui/__init__.py @@ -1212,6 +1212,7 @@ def _compile(self, simulator_if: SimulatorInterface): continue_on_error=self._args.keep_compiling, printer=self._printer, target_files=target_files, + num_threads=self._args.num_threads if self._args.parallel_compile else 1, ) def _get_testbench_files(self, simulator_if: Union[None, SimulatorInterface]): diff --git a/vunit/vunit_cli.py b/vunit/vunit_cli.py index 9e8f815da..b41a05186 100644 --- a/vunit/vunit_cli.py +++ b/vunit/vunit_cli.py @@ -136,6 +136,13 @@ def _create_argument_parser(description=None, for_documentation=False): help="Continue compiling even after errors only skipping files that depend on failed files", ) + parser.add_argument( + "--parallel-compile", + action="store_true", + default=False, + help="Compile as many files in parallel as -p runs tests, rather than one file at a time", + ) + parser.add_argument( "--fail-fast", action="store_true", @@ -248,7 +255,7 @@ def _create_argument_parser(description=None, for_documentation=False): type=nonnegative_int, default=1, help=( - "Number of tests to run in parallel. " + "Number of tests to run in parallel, and of files to compile in parallel with --parallel-compile. " "Test output is not continuously written in verbose mode with p != 1. " "p = 0 uses all logical CPUs." ),