diff --git a/docs/cli.rst b/docs/cli.rst index 7cfa4f29d..8a940c9ba 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -117,6 +117,19 @@ previous results and recent code changes to achieve two main goals: has a history of failing, and no relevant code changes have been made, so it is assumed to fail again and is run first. 2. Load balancing - When tests are executed in parallel using the ``-p`` option, VUnit distributes them across threads to minimize to total execution time. +With ``--parallel-compile``, as many files as ``-p`` gives are also compiled in parallel. A file is compiled once the +files it depends on are compiled. Most simulators do not lock a library while they update it, so a file is not compiled +while another file is compiled into its library, or into a library it reads. NVC locks its libraries and compiles any +files at the same time. Without ``--parallel-compile``, files are compiled one at a time. + +With ``--batch-compile``, the VHDL files of a library that are ready to be compiled at the same time, and have the same +compile options, are compiled in one call to the compiler, rather than one call per file. This makes a clean compile +several times faster, since most of the time of compiling a file goes into starting the compiler and updating the +library. A batch that fails is compiled again one file at a time, to find the file that fails. ``--batch-compile`` is +supported by GHDL, NVC and Riviera-PRO, and can be combined with ``--parallel-compile``. NVC and Riviera-PRO read the +file names of a batch from a file. GHDL gets them in the command, so a batch ends before its command gets longer than +32000 characters, below the limit of a command line on Windows. + Opening a Test Case in Simulator GUI ==================================== diff --git a/docs/news.d/1235.feature.rst b/docs/news.d/1235.feature.rst new file mode 100644 index 000000000..39e2fd906 --- /dev/null +++ b/docs/news.d/1235.feature.rst @@ -0,0 +1,3 @@ +Added ``--parallel-compile``, which compiles as many files in parallel as ``-p`` gives: a file is compiled once the files +it depends on are compiled. Unless the simulator locks its libraries, as NVC does, a file is not compiled while its +library, or a library it reads, is compiled into. diff --git a/docs/news.d/1236.feature.rst b/docs/news.d/1236.feature.rst new file mode 100644 index 000000000..a7705d215 --- /dev/null +++ b/docs/news.d/1236.feature.rst @@ -0,0 +1,3 @@ +Added ``--batch-compile``, which compiles the VHDL files of a library that are ready to be compiled at the same time, and +have the same compile options, in one call to the compiler. A batch that fails is compiled again one file at a time. +Supported by GHDL, NVC and Riviera-PRO. diff --git a/tests/unit/test_ghdl_interface.py b/tests/unit/test_ghdl_interface.py index e72fa8967..dc37301d5 100644 --- a/tests/unit/test_ghdl_interface.py +++ b/tests/unit/test_ghdl_interface.py @@ -137,6 +137,19 @@ def test_assertion_on_unknown_backend(self, check_output): check_output.return_value = version self.assertRaises(AssertionError, GHDLInterface.determine_backend, "prefix") + @mock.patch.object(GHDLInterface, "determine_version", return_value=5.0) + def test_setup_library_mapping_creates_libraries(self, determine_version): + simif = GHDLInterface(prefix="prefix", output_path="") + project = Project() + project.add_library("lib", "lib_path") + write_file(str(Path("existing_path") / "existing-obj08.cf"), "") + project.add_library("existing", "existing_path") + simif.setup_library_mapping(project) + self.create_library.assert_called_once_with( + [str(Path("prefix") / "ghdl"), "-i", "--workdir=lib_path", "--work=lib", "--std=08"], + env=simif.get_env(), + ) + @mock.patch("vunit.sim_if.check_output", autospec=True, return_value="") # pylint: disable=no-self-use @mock.patch.object(GHDLInterface, "determine_version", return_value=6.0) def test_compile_project_2019(self, determine_version, check_output): @@ -312,6 +325,9 @@ def test_compile_project_verilog_error(self, determine_version): def setUp(self): self.output_path = str(Path(__file__).parent / "test_ghdl_interface_out") renew_path(self.output_path) + patcher = mock.patch("vunit.sim_if.ghdl.subprocess.check_call", autospec=True) + self.create_library = patcher.start() + self.addCleanup(patcher.stop) self.project = Project() self.cwd = os.getcwd() os.chdir(self.output_path) diff --git a/tests/unit/test_rivierapro_interface.py b/tests/unit/test_rivierapro_interface.py index 62a11b30d..9b836efa2 100644 --- a/tests/unit/test_rivierapro_interface.py +++ b/tests/unit/test_rivierapro_interface.py @@ -25,6 +25,40 @@ class TestRivieraProInterface(unittest.TestCase): Test the RivieraPro interface """ + @mock.patch("vunit.sim_if.check_output", autospec=True) + @mock.patch("vunit.sim_if.rivierapro.Process", autospec=True) + @mock.patch("vunit.sim_if.rivierapro.RivieraProInterface.find_prefix", return_value="prefix") + def _compile_in_a_batch(self, names, _find_prefix, _process, check_output): + """ + Compile the files names in a batch. Return the file lists and the names compiled on their own. + """ + simif = RivieraProInterface(prefix="prefix", output_path=self.output_path) + project = Project() + project.add_library("lib", "lib_path") + for name in names: + write_file(name, "") + project.add_source_file(name, "lib", file_type="vhdl", vhdl_standard=VHDL.standard("2008")) + file_lists = [] + names_on_their_own = [] + + def check_output_side_effect(command, env=None): # pylint: disable=unused-argument + if command[-2] == "-f": + file_lists.append(Path(command[-1]).read_text()) + else: + names_on_their_own.append(command[-1]) + return "" + + check_output.side_effect = check_output_side_effect + simif.compile_project(project, batch=True) + self.assertEqual(list(Path(self.output_path).glob("batch_*.txt")), []) + return file_lists, names_on_their_own + + def test_compile_project_in_a_batch_with_a_file_list(self): + file_lists, names_on_their_own = self._compile_in_a_batch(["file.vhd", "sp ace.vhd"]) + self.assertEqual(len(file_lists), 1) + self.assertEqual(sorted(file_lists[0].splitlines()), ['"file.vhd"', '"sp ace.vhd"']) + self.assertEqual(names_on_their_own, []) + @mock.patch("vunit.sim_if.check_output", autospec=True, return_value="") @mock.patch("vunit.sim_if.rivierapro.Process", autospec=True) @mock.patch("vunit.sim_if.rivierapro.RivieraProInterface.find_prefix", return_value="prefix") diff --git a/tests/unit/test_simulator_interface.py b/tests/unit/test_simulator_interface.py index 6ea155afd..e9212893f 100644 --- a/tests/unit/test_simulator_interface.py +++ b/tests/unit/test_simulator_interface.py @@ -12,6 +12,7 @@ from pathlib import Path from os import chdir, getcwd import subprocess +import threading from shutil import rmtree from unittest import mock from vunit.project import Project @@ -190,6 +191,205 @@ def raise_compile_error(source_file): # pylint: disable=unused-argument self.assertRaises(CompileError, simif.compile_source_files, project) self.assertEqual(project.get_files_in_compile_order(incremental=True), [source_file]) + def _compile_concurrently( + self, libraries, supports_concurrent_compiles_into_library=False, fail=None, dependencies=() + ): + """ + Compile one file per library name in libraries with 4 threads, the compile commands + waiting for each other in pairs. dependencies are (file index, index of the file it depends + on) pairs. Return the printer output and the compile commands that ran at the same time as + another. + """ + simif = create_simulator_interface() + simif.supports_concurrent_compiles_into_library = supports_concurrent_compiles_into_library + simif.compile_source_file_command.side_effect = lambda source_file: [Path(source_file.name).stem] + project, _ = create_project( + [(f"file{idx}.vhd", library) for idx, library in enumerate(libraries)], dependencies + ) + + barrier = threading.Barrier(2, timeout=0.5) + concurrent = set() + + def check_output_side_effect(command, env=None): # pylint: disable=unused-argument + try: + barrier.wait() + concurrent.add(command[0]) + except threading.BrokenBarrierError: + barrier.reset() # Ran alone, let the next compiles meet + if command[0] == fail: + raise subprocess.CalledProcessError(returncode=-1, cmd=command, output="bad stuff") + return command[0] + " output\n" + + with mock.patch("vunit.sim_if.check_output", autospec=True) as check_output: + check_output.side_effect = check_output_side_effect + printer = MockPrinter() + simif.compile_source_files(project, printer=printer, num_threads=4, continue_on_error=True) + return printer.output, concurrent + + def test_compile_source_files_of_different_libraries_concurrently(self): + _, concurrent = self._compile_concurrently(["lib1", "lib2"]) + self.assertEqual(concurrent, {"file0", "file1"}) + + def test_compile_source_files_of_a_library_one_at_a_time(self): + _, concurrent = self._compile_concurrently(["lib", "lib"]) + self.assertEqual(concurrent, set()) + + def test_compile_source_files_of_a_library_concurrently_when_supported(self): + _, concurrent = self._compile_concurrently(["lib", "lib"], supports_concurrent_compiles_into_library=True) + self.assertEqual(concurrent, {"file0", "file1"}) + + def test_compile_source_files_not_while_a_library_they_read_is_compiled_into(self): + # Once file0 is compiled, file1 is compiled into lib1, which file2 reads + _, concurrent = self._compile_concurrently(["lib1", "lib1", "lib2"], dependencies=[(1, 0), (2, 0)]) + self.assertEqual(concurrent, set()) + + def test_compile_source_files_reading_the_same_library_concurrently(self): + _, concurrent = self._compile_concurrently(["lib1", "lib2", "lib3"], dependencies=[(1, 0), (2, 0)]) + self.assertEqual(concurrent, {"file1", "file2"}) + + def test_compile_source_files_concurrently_writes_the_output_of_a_file_at_once(self): + output, _ = self._compile_concurrently(["lib1", "lib2"], fail="file1") + self.assertIn("Compiling into lib1: file0.vhd passed\nfile0 output\n", output) + self.assertIn("Compiling into lib2: file1.vhd failed\n=== Command used: ===\nfile1\n", output) + self.assertTrue(output.endswith("Compile failed\n")) + + def test_compile_source_files_waits_for_dependencies(self): + simif = create_simulator_interface() + simif.compile_source_file_command.side_effect = lambda source_file: [Path(source_file.name).stem] + project, (file1, file2, file3) = create_project( + [("file1.vhd", "lib1"), ("file2.vhd", "lib2"), ("file3.vhd", "lib1")], dependencies=[(2, 1)] + ) + + def check_output_side_effect(command, env=None): # pylint: disable=unused-argument + if command == ["file2"]: + raise subprocess.CalledProcessError(returncode=-1, cmd=command, output="bad stuff") + return "" + + with mock.patch("vunit.sim_if.check_output", autospec=True) as check_output: + check_output.side_effect = check_output_side_effect + printer = MockPrinter() + simif.compile_source_files(project, printer=printer, num_threads=4, continue_on_error=True) + self.assertIn("Compiling into lib1: file3.vhd skipped\n", printer.output) + self.assertEqual(len(check_output.mock_calls), 2) + self.assertEqual(project.get_files_in_compile_order(incremental=True), [file2, file3]) + self.assertNotIn(file1, project.get_files_in_compile_order(incremental=True)) + + def _compile_in_batches( + self, + files, + fail=(), + num_threads=1, + dependencies=(), + supports_concurrent_compiles_into_library=False, + batch_file_list_arguments=None, + ): # pylint: disable=too-many-arguments,too-many-locals + """ + Compile files, (name, library, option) tuples, in batches. dependencies are (file index, index + of the file it depends on) pairs, and a command naming a file in fail fails. Return the + printer output and the file names of each command. + """ + simif = create_simulator_interface() + simif.supports_batch_compile = True + simif.supports_concurrent_compiles_into_library = supports_concurrent_compiles_into_library + simif.batch_file_list_arguments = batch_file_list_arguments + simif.compile_source_file_command.side_effect = lambda source_file: [ + "compile", + source_file.library.name, + str(source_file.compile_options.get("ghdl.a_flags", [])), + source_file.name, + ] + project, source_files = create_project([(name, library) for name, library, _ in files], dependencies) + for source_file, (_, _, option) in zip(source_files, files): + if option: + source_file.set_compile_option("ghdl.a_flags", [option]) + + commands = [] + lock = threading.Lock() + + def check_output_side_effect(command, env=None): # pylint: disable=unused-argument + names = command[3:] + if names and names[0].startswith("@"): + names = Path(names[0][1:]).read_text().splitlines() + with lock: + commands.append(names) + if set(names) & set(fail): + raise subprocess.CalledProcessError(returncode=-1, cmd=command, output="bad stuff") + return "" + + with mock.patch("vunit.sim_if.check_output", autospec=True) as check_output: + check_output.side_effect = check_output_side_effect + printer = MockPrinter() + simif.compile_source_files( + project, printer=printer, continue_on_error=True, num_threads=num_threads, batch=True + ) + return printer.output, commands + + def test_compile_source_files_in_a_batch(self): + output, commands = self._compile_in_batches( + [("file1.vhd", "lib", None), ("file2.vhd", "lib", None)], dependencies=[(1, 0)] + ) + self.assertEqual(commands, [["file1.vhd", "file2.vhd"]]) + self.assertEqual( + output, + """\ +Compiling into lib: file1.vhd passed +Compiling into lib: file2.vhd passed +Compile passed +""", + ) + + def test_compile_source_files_in_batches_with_commands_of_limited_length(self): + # A command of a batch is "compile lib [] " followed by the file names + with mock.patch("vunit.sim_if._MAX_BATCH_COMMAND_LENGTH", len("compile lib [] file1.vhd file2.vhd")): + _, commands = self._compile_in_batches( + [("file1.vhd", "lib", None), ("file2.vhd", "lib", None), ("file3.vhd", "lib", None)], + dependencies=[(1, 0), (2, 1)], + ) + self.assertEqual(commands, [["file1.vhd", "file2.vhd"], ["file3.vhd"]]) + + def test_compile_source_files_in_a_batch_with_a_file_list(self): + with mock.patch("vunit.sim_if._MAX_BATCH_COMMAND_LENGTH", 1): + _, commands = self._compile_in_batches( + [("file1.vhd", "lib", None), ("file2.vhd", "lib", None), ("file3.vhd", "lib", None)], + batch_file_list_arguments=("@{}",), + ) + self.assertEqual(sorted(commands[0]), ["file1.vhd", "file2.vhd", "file3.vhd"]) + self.assertEqual(len(commands), 1) + self.assertEqual(list(Path(".").glob("batch_*.txt")), []) + + def test_compile_source_files_in_a_batch_per_library_and_compile_options(self): + _, commands = self._compile_in_batches( + [("file1.vhd", "lib1", None), ("file2.vhd", "lib1", "-x"), ("file3.vhd", "lib2", None)] + ) + self.assertEqual(sorted(commands), [["file1.vhd"], ["file2.vhd"], ["file3.vhd"]]) + + def test_compile_source_files_of_a_failed_batch_one_at_a_time(self): + output, commands = self._compile_in_batches( + [("file1.vhd", "lib", None), ("file2.vhd", "lib", None), ("file3.vhd", "lib", None)], + fail=["file2.vhd"], + dependencies=[(2, 1)], + ) + self.assertEqual(sorted(commands[0]), ["file1.vhd", "file2.vhd", "file3.vhd"]) + self.assertEqual(sorted(commands[1:]), [["file1.vhd"], ["file2.vhd"]]) + self.assertIn("Compiling 3 files into lib in one call failed, compiling them one at a time\n", output) + self.assertIn("Compiling into lib: file1.vhd passed\n", output) + self.assertIn("Compiling into lib: file2.vhd failed\n", output) + self.assertIn("Compiling into lib: file3.vhd skipped\n", output) + + def _test_compile_source_files_in_batches_in_parallel(self, supports_concurrent_compiles_into_library): + _, commands = self._compile_in_batches( + [("file1.vhd", "lib1", None), ("file2.vhd", "lib1", None), ("file3.vhd", "lib2", None)], + num_threads=4, + supports_concurrent_compiles_into_library=supports_concurrent_compiles_into_library, + ) + self.assertEqual(sorted(sorted(command) for command in commands), [["file1.vhd", "file2.vhd"], ["file3.vhd"]]) + + def test_compile_source_files_in_batches_in_parallel(self): + self._test_compile_source_files_in_batches_in_parallel(False) + + def test_compile_source_files_in_batches_in_parallel_into_a_library(self): + self._test_compile_source_files_in_batches_in_parallel(True) + @mock.patch("os.environ", autospec=True) def test_find_prefix(self, environ): class MySimulatorInterface(SimulatorInterface): # pylint: disable=abstract-method @@ -319,6 +519,23 @@ def create_simulator_interface(): return simif +def create_project(files, dependencies=()): + """ + Create a project of VHDL files, (name, library) pairs. dependencies are (file index, index of + the file it depends on) pairs. Return the project and its source files. + """ + project = Project() + source_files = [] + for name, library in files: + if library not in [existing.name for existing in project.get_libraries()]: + project.add_library(library, library + "_path") + write_file(name, "") + source_files.append(project.add_source_file(name, library, file_type="vhdl")) + for idx, depends_on in dependencies: + project.add_manual_dependency(source_files[idx], depends_on=source_files[depends_on]) + return project, source_files + + class MockPrinter(object): """ Mock printer that accumulates the calls as a string diff --git a/tests/unit/test_ui.py b/tests/unit/test_ui.py index c0584581d..df0347e13 100644 --- a/tests/unit/test_ui.py +++ b/tests/unit/test_ui.py @@ -1180,6 +1180,33 @@ def test_can_list_files_without_simulator(self): ui = self._create_ui_real_sim("--files") self._run_main(ui, 0) + def _compile_arguments(self, *args): + """ + The keyword arguments the files of a ui created with args are compiled with + """ + ui = self._create_ui(*args) + simulator_if = mock.Mock(spec=SimulatorInterface) + ui._compile(simulator_if) # pylint: disable=protected-access + return simulator_if.compile_project.call_args.kwargs + + def _compile_threads(self, *args): + """ + The number of threads the files of a ui created with args are compiled with + """ + return self._compile_arguments(*args)["num_threads"] + + def test_compile_in_batches_with_batch_compile(self): + self.assertFalse(self._compile_arguments()["batch"]) + self.assertTrue(self._compile_arguments("--batch-compile")["batch"]) + + def test_compile_one_file_at_a_time_without_parallel_compile(self): + self.assertEqual(self._compile_threads("-p", "4"), 1) + self.assertEqual(self._compile_threads("-p", "0"), 1) + + def test_compile_in_parallel_with_parallel_compile(self): + self.assertEqual(self._compile_threads("-p", "4", "--parallel-compile"), 4) + self.assertEqual(self._compile_threads("-p", "0", "--parallel-compile"), 0) + @mock.patch("vunit.ui.LOGGER", autospec=True) def test_compile_without_simulator_fails(self, logger): with set_env(): diff --git a/vunit/sim_if/__init__.py b/vunit/sim_if/__init__.py index 53b2ea2a1..2f6993e3c 100644 --- a/vunit/sim_if/__init__.py +++ b/vunit/sim_if/__init__.py @@ -13,6 +13,9 @@ from os import environ, listdir, pathsep import locale import subprocess +import tempfile +from collections import Counter +from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from pathlib import Path from typing import List, Optional from ..ostools import Process, simplify_path @@ -20,6 +23,10 @@ from ..color_printer import NO_COLOR_PRINTER +# The longest command of a batch, below the 32767 characters of a command line on Windows +_MAX_BATCH_COMMAND_LENGTH = 32000 + + class Option(object): """ A compile or sim option @@ -62,6 +69,20 @@ class SimulatorInterface(object): # pylint: disable=too-many-public-methods # True if simulator supports ANSI colors in GUI mode supports_colors_in_gui = False + # True if VHDL files of a library, with the same compile options, can be compiled in one call + supports_batch_compile = False + + # The arguments making the compiler read the file names of a batch from the file named {}, with + # batch_file_list_line naming each file. None passes them in the command, which limits the size + # of a batch. A batch whose file list a compiler cannot read is compiled one file at a time. + batch_file_list_arguments: Optional[tuple] = None + batch_file_list_line = "{}" + + # True if a library can be compiled into while it is compiled into or read by another + # compile. Most compilers update the index of a library without locking it, so concurrent + # updates lose units and a compile reading it during an update fails. + supports_concurrent_compiles_into_library = False + # The path prefix the interface was built with, set by the interfaces having one _prefix: Optional[str] = None @@ -225,14 +246,21 @@ def compile_project( printer=NO_COLOR_PRINTER, continue_on_error=False, target_files=None, - ): + num_threads=1, + *, + batch=False, + ): # pylint: disable=too-many-arguments """ Compile the project param: target_files: Given a list of SourceFiles only these and dependent files are compiled + param: num_threads: The number of files compiled at the same time, 0 for all logical CPUs + param: batch: Compile files that are ready at the same time in one call, see compile_source_files """ self.add_simulator_specific(project) self.setup_library_mapping(project) - self.compile_source_files(project, printer, continue_on_error, target_files=target_files) + self.compile_source_files( + project, printer, continue_on_error, target_files=target_files, num_threads=num_threads, batch=batch + ) def simulate(self, output_path, test_suite_name, config, elaborate_only): """ @@ -281,62 +309,244 @@ def compile_source_files( printer=NO_COLOR_PRINTER, continue_on_error=False, target_files=None, - ): + num_threads=1, + *, + batch=False, + ): # pylint: disable=too-many-arguments """ Use compile_source_file_command to compile all source_files param: target_files: Given a list of SourceFiles only these and dependent files are compiled + param: num_threads: The number of files compiled at the same time, 0 for all logical CPUs + param: batch: Compile the VHDL files of a library that are ready at the same time, and have the + same compile options, in one call if supports_batch_compile. A batch that fails is compiled + again one file at a time to find the file that fails. """ dependency_graph = project.create_dependency_graph() - failures = [] if target_files is None: source_files = project.get_files_in_compile_order(dependency_graph=dependency_graph) else: source_files = project.get_minimal_file_set_in_compile_order(target_files) - source_files_to_skip = set() + failures = self._compile_in_dependency_order( + project, + source_files, + dependency_graph=dependency_graph, + printer=printer, + continue_on_error=continue_on_error, + num_threads=num_threads or os.cpu_count() or 1, + batch=batch and self.supports_batch_compile, + ) + + if failures: + printer.write("Compile failed\n", fg="ri") + if continue_on_error: + return + raise CompileError - max_library_name = 0 - max_source_file_name = 0 if source_files: - max_library_name = max(len(source_file.library.name) for source_file in source_files) - max_source_file_name = max(len(simplify_path(source_file.name)) for source_file in source_files) + printer.write("Compile passed\n", fg="gi") + else: + printer.write("Re-compile not needed\n") - for source_file in source_files: - printer.write( + def _compile_in_dependency_order( + self, project, source_files, *, dependency_graph, printer, continue_on_error, num_threads, batch + ): # pylint: disable=too-many-arguments,too-many-locals,too-many-statements + """ + Compile source_files, num_threads at a time, and return the files that failed. A file is + compiled once its dependencies are. Unless supports_concurrent_compiles_into_library, it is + not compiled while its library is compiled into or read, or while a library it reads, that of + any of its dependencies, is compiled into. The dependents of a failed file are skipped. + """ + max_library_name = max((len(source_file.library.name) for source_file in source_files), default=0) + max_source_file_name = max((len(simplify_path(source_file.name)) for source_file in source_files), default=0) + + def write_header(source_file, output): + output.write( f"Compiling into {(source_file.library.name + ':').ljust(max_library_name + 1)!s} " f"{simplify_path(source_file.name).ljust(max_source_file_name)!s} " ) - sys.stdout.flush() - if source_file in source_files_to_skip: - printer.write("skipped", fg="rgi") - printer.write("\n") - continue + def compile_files(files, output): + if len(files) > 1: + return self._compile_batch(files, output, write_header) + write_header(files[0], output) + sys.stdout.flush() + return self._compile_source_file(files[0], output) + + batch_commands = {} + + def batch_command(source_file): + """The command compiling source_file without its file name, None if it cannot be batched""" + if source_file not in batch_commands: + batch_commands[source_file] = self._batch_command(source_file) if batch else None + return batch_commands[source_file] + + def batch_of(source_file): + """source_file and the files that are ready to be compiled with it in one call""" + files = [source_file] + if batch_command(source_file) is None: + return files + length = len(subprocess.list2cmdline(list(batch_command(source_file)) + [source_file.name])) + limit = _MAX_BATCH_COMMAND_LENGTH if self.batch_file_list_arguments is None else float("inf") + for other in to_compile: + if ( + other.library.name == source_file.library.name + and batch_command(other) == batch_command(source_file) + and all(compiled.get(dependency) or dependency in files for dependency in dependencies[other]) + and not is_busy(other) + ): + length += 1 + len(subprocess.list2cmdline([other.name])) + if length > limit: + break + files.append(other) + return files + + def is_busy(source_file): + """Whether a running compile keeps source_file from being compiled now""" + library = source_file.library.name + return not self.supports_concurrent_compiles_into_library and bool( + libraries_written[library] + or libraries_being_read[library] + or any(libraries_written[read] for read in libraries_read[source_file]) + ) - if self._compile_source_file(source_file, printer): - project.update(source_file) - else: - source_files_to_skip.update(dependency_graph.get_dependent([source_file])) - failures.append(source_file) + def finish(files, passed): + if len(files) > 1 and not passed: + # Compiled again one at a time, in compile order, to find the file that fails + for source_file in files: + batch_commands[source_file] = None + waiting = set(to_compile) | set(files) + to_compile[:] = [source_file for source_file in source_files if source_file in waiting] + return - if not continue_on_error: + for source_file in files: + compiled[source_file] = passed + if passed: + project.update(source_file) + else: + failures.append(source_file) + + to_compile = list(source_files) + dependencies = { + source_file: dependency_graph.get_direct_dependencies(source_file) & set(source_files) + for source_file in source_files + } + # ponytail: a bare library clause, with nothing used from the library, is not a read + libraries_read = { + source_file: {dependency.library.name for dependency in dependency_graph.get_dependencies([source_file])} + for source_file in source_files + } + compiled = {} # Whether a finished file was compiled, rather than failed or skipped + failures = [] + running = {} + libraries_written = Counter() + libraries_being_read = Counter() + + with ThreadPoolExecutor(max_workers=num_threads) as executor: + while to_compile or running: + # ponytail: scans every waiting file per completed one, fine for thousands of files + for source_file in list(to_compile): + if len(running) == num_threads or (failures and not continue_on_error): + break + + # Taken by a batch, or waiting for its dependencies + if source_file not in to_compile or not dependencies[source_file] <= compiled.keys(): + continue + + if not all(compiled[dependency] for dependency in dependencies[source_file]): + to_compile.remove(source_file) + compiled[source_file] = False + write_header(source_file, printer) + printer.write("skipped", fg="rgi") + printer.write("\n") + continue + + if is_busy(source_file): + continue + + to_compile.remove(source_file) + files = batch_of(source_file) + for other in files[1:]: + to_compile.remove(other) + libraries_written[source_file.library.name] += 1 + for other in files: + libraries_being_read.update(libraries_read[other]) + # The output of a file is written once it is compiled, so that outputs do not mix + output = printer if num_threads == 1 else _BufferedPrinter() + running[executor.submit(compile_files, files, output)] = (files, output) + + if not running: break - if failures: - printer.write("Compile failed\n", fg="ri") - if continue_on_error: - return - raise CompileError + done, _ = wait(running, return_when=FIRST_COMPLETED) + for future in done: + files, output = running.pop(future) + libraries_written[files[0].library.name] -= 1 + for source_file in files: + libraries_being_read.subtract(libraries_read[source_file]) + if output is not printer: + output.write_to(printer) - if source_files: - printer.write("Compile passed\n", fg="gi") - else: - printer.write("Re-compile not needed\n") + finish(files, future.result()) + + return failures def compile_source_file_command(self, source_file): # pylint: disable=unused-argument raise NotImplementedError + def _batch_command(self, source_file): + """ + The command compiling source_file without its file name, or None if source_file cannot be + compiled with other files. Files with the same command are compiled in one call. + """ + if not source_file.is_vhdl: + return None + try: + command = self.compile_source_file_command(source_file) + except CompileError: + return None + return tuple(command[:-1]) if command[-1] == source_file.name else None + + def _compile_batch(self, source_files, printer, write_header): + """ + Compile source_files in one call. The output of a batch that fails is not written, since + its files are compiled again one at a time. + """ + command = list(self._batch_command(source_files[0])) + file_list = None + if self.batch_file_list_arguments is None: + command += [source_file.name for source_file in source_files] + else: + Path(self._output_path).mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + "w", dir=self._output_path, prefix="batch_", suffix=".txt", delete=False + ) as fptr: + fptr.writelines( + self.batch_file_list_line.format(source_file.name) + "\n" for source_file in source_files + ) + file_list = fptr.name + command += [argument.format(file_list) for argument in self.batch_file_list_arguments] + try: + output = check_output(command, env=self.get_env()) + except subprocess.CalledProcessError: + printer.write( + f"Compiling {len(source_files)} files into {source_files[0].library.name} in one call failed, " + "compiling them one at a time\n" + ) + return False + + finally: + if file_list is not None: + os.remove(file_list) + + for source_file in source_files: + write_header(source_file, printer) + printer.write("passed", fg="gi") + printer.write("\n") + printer.write(output) + return True + @staticmethod def get_env(): """ @@ -344,6 +554,22 @@ def get_env(): """ +class _BufferedPrinter(object): + """ + Printer keeping what is written until it is written to another printer + """ + + def __init__(self): + self._writes = [] + + def write(self, text, output_file=None, fg=None, bg=None): + self._writes.append((text, output_file, fg, bg)) + + def write_to(self, printer): + for text, output_file, fg, bg in self._writes: + printer.write(text, output_file=output_file, fg=fg, bg=bg) + + def isfile(file_name): """ Case insensitive Path.is_file() diff --git a/vunit/sim_if/ghdl.py b/vunit/sim_if/ghdl.py index bb28a5c85..e657627f1 100644 --- a/vunit/sim_if/ghdl.py +++ b/vunit/sim_if/ghdl.py @@ -37,6 +37,7 @@ class GHDLInterface(SimulatorInterface, ViewerMixin): # pylint: disable=too-man executable = environ.get("GHDL", "ghdl") supports_gui_flag = True supports_colors_in_gui = True + supports_batch_compile = True compile_options = [ ListOfStringOption("ghdl.a_flags"), @@ -257,6 +258,21 @@ def setup_library_mapping(self, project): else: self._vhdl_standard = list(vhdl_standards)[0] + # A library clause fails on a library nothing has been compiled into, which files compiled + # in parallel may come before, so every library is created first + for library in project.get_libraries(): + if not list(Path(library.directory).glob(f"{library.name}-obj*.cf")): + subprocess.check_call( + [ + str(Path(self._prefix) / self.executable), + "-i", + f"--workdir={library.directory!s}", + f"--work={library.name!s}", + f"--std={self._std_str(self._vhdl_standard)!s}", + ], + env=self.get_env(), + ) + def compile_source_file_command(self, source_file): """ Returns the command to compile a single source_file @@ -445,6 +461,26 @@ def simulate(self, output_path, test_suite_name, config, elaborate_only): # pyl return status + def compile_source_files(self, project, *args, num_threads=1, **kwargs): # pylint: disable=arguments-differ + """ + Compile one file at a time with coverage of the gcc backend, which writes the .gcno file + of every file to the working directory + """ + if self._backend == "gcc" and any( + source_file.compile_options.get("enable_coverage", False) + for source_file in project.get_source_files_in_order() + ): + num_threads = 1 + super().compile_source_files(project, *args, num_threads=num_threads, **kwargs) + + def _batch_command(self, source_file): + """ + Files compiled with coverage are compiled one at a time, see _compile_source_file + """ + if source_file.compile_options.get("enable_coverage", False): + return None + return super()._batch_command(source_file) + def _compile_source_file(self, source_file, printer): """ Runs parent command for compilation, and moves any .gcno files to the compilation output diff --git a/vunit/sim_if/nvc.py b/vunit/sim_if/nvc.py index cc2880384..3c06498c8 100644 --- a/vunit/sim_if/nvc.py +++ b/vunit/sim_if/nvc.py @@ -36,6 +36,11 @@ class NVCInterface(SimulatorInterface, ViewerMixin): # pylint: disable=too-many executable = environ.get("NVC", "nvc") supports_gui_flag = True supports_colors_in_gui = True + # NVC locks the library while it updates it + supports_concurrent_compiles_into_library = True + supports_batch_compile = True + # One file name per line, taken as it is + batch_file_list_arguments = ("@{}",) compile_options = [ ListOfStringOption("nvc.global_flags"), diff --git a/vunit/sim_if/rivierapro.py b/vunit/sim_if/rivierapro.py index 20f4431ff..5d641f07f 100644 --- a/vunit/sim_if/rivierapro.py +++ b/vunit/sim_if/rivierapro.py @@ -28,6 +28,10 @@ class RivieraProInterface(VsimSimulatorMixin, SimulatorInterface): """ name = "rivierapro" + supports_batch_compile = True + batch_file_list_arguments = ("-f", "{}") + # Quoted, which keeps spaces, and backslashes on Windows, as they are + batch_file_list_line = '"{}"' supports_gui_flag = True package_users_depend_on_bodies = True diff --git a/vunit/ui/__init__.py b/vunit/ui/__init__.py index 17503a6c8..a0425737a 100644 --- a/vunit/ui/__init__.py +++ b/vunit/ui/__init__.py @@ -1212,6 +1212,8 @@ def _compile(self, simulator_if: SimulatorInterface): continue_on_error=self._args.keep_compiling, printer=self._printer, target_files=target_files, + num_threads=self._args.num_threads if self._args.parallel_compile else 1, + batch=self._args.batch_compile, ) def _get_testbench_files(self, simulator_if: Union[None, SimulatorInterface]): diff --git a/vunit/vunit_cli.py b/vunit/vunit_cli.py index 9e8f815da..80bcdb418 100644 --- a/vunit/vunit_cli.py +++ b/vunit/vunit_cli.py @@ -136,6 +136,23 @@ def _create_argument_parser(description=None, for_documentation=False): help="Continue compiling even after errors only skipping files that depend on failed files", ) + parser.add_argument( + "--parallel-compile", + action="store_true", + default=False, + help="Compile as many files in parallel as -p runs tests, rather than one file at a time", + ) + + parser.add_argument( + "--batch-compile", + action="store_true", + default=False, + help=( + "Compile the VHDL files of a library that are ready to be compiled at the same time, " + "and have the same compile options, in one call. Supported by GHDL, NVC and Riviera-PRO" + ), + ) + parser.add_argument( "--fail-fast", action="store_true", @@ -248,7 +265,7 @@ def _create_argument_parser(description=None, for_documentation=False): type=nonnegative_int, default=1, help=( - "Number of tests to run in parallel. " + "Number of tests to run in parallel, and of files to compile in parallel with --parallel-compile. " "Test output is not continuously written in verbose mode with p != 1. " "p = 0 uses all logical CPUs." ),