Add --parallel-compile to compile files in parallel

docs/cli.rst

@@ -117,6 +117,11 @@ previous results and recent code changes to achieve two main goals:
    has a history of failing, and no relevant code changes have been made, so it is assumed to fail again and is run first.
 2. Load balancing - When tests are executed in parallel using the ``-p`` option, VUnit distributes them across threads to minimize to total execution time.
 
+With ``--parallel-compile``, as many files as ``-p`` gives are also compiled in parallel. A file is compiled once the
+files it depends on are compiled. Most simulators do not lock a library while they update it, so a file is not compiled
+while another file is compiled into its library, or into a library it reads. NVC locks its libraries and compiles any
+files at the same time. Without ``--parallel-compile``, files are compiled one at a time.
+
 Opening a Test Case in Simulator GUI
 ====================================
 

docs/news.d/1235.feature.rst

@@ -0,0 +1,3 @@
+Added ``--parallel-compile``, which compiles as many files in parallel as ``-p`` gives: a file is compiled once the files
+it depends on are compiled. Unless the simulator locks its libraries, as NVC does, a file is not compiled while its
+library, or a library it reads, is compiled into.

tests/unit/test_ghdl_interface.py

@@ -137,6 +137,19 @@ def test_assertion_on_unknown_backend(self, check_output):
         check_output.return_value = version
         self.assertRaises(AssertionError, GHDLInterface.determine_backend, "prefix")
 
+    @mock.patch.object(GHDLInterface, "determine_version", return_value=5.0)
+    def test_setup_library_mapping_creates_libraries(self, determine_version):
+        simif = GHDLInterface(prefix="prefix", output_path="")
+        project = Project()
+        project.add_library("lib", "lib_path")
+        write_file(str(Path("existing_path") / "existing-obj08.cf"), "")
+        project.add_library("existing", "existing_path")
+        simif.setup_library_mapping(project)
+        self.create_library.assert_called_once_with(
+            [str(Path("prefix") / "ghdl"), "-i", "--workdir=lib_path", "--work=lib", "--std=08"],
+            env=simif.get_env(),
+        )
+
     @mock.patch("vunit.sim_if.check_output", autospec=True, return_value="")  # pylint: disable=no-self-use
     @mock.patch.object(GHDLInterface, "determine_version", return_value=6.0)
     def test_compile_project_2019(self, determine_version, check_output):
@@ -312,6 +325,9 @@ def test_compile_project_verilog_error(self, determine_version):
     def setUp(self):
         self.output_path = str(Path(__file__).parent / "test_ghdl_interface_out")
         renew_path(self.output_path)
+        patcher = mock.patch("vunit.sim_if.ghdl.subprocess.check_call", autospec=True)
+        self.create_library = patcher.start()
+        self.addCleanup(patcher.stop)
         self.project = Project()
         self.cwd = os.getcwd()
         os.chdir(self.output_path)

tests/unit/test_simulator_interface.py

@@ -12,6 +12,7 @@
 from pathlib import Path
 from os import chdir, getcwd
 import subprocess
+import threading
 from shutil import rmtree
 from unittest import mock
 from vunit.project import Project
@@ -190,6 +191,89 @@ def raise_compile_error(source_file):  # pylint: disable=unused-argument
             self.assertRaises(CompileError, simif.compile_source_files, project)
         self.assertEqual(project.get_files_in_compile_order(incremental=True), [source_file])
 
+    def _compile_concurrently(
+        self, libraries, supports_concurrent_compiles_into_library=False, fail=None, dependencies=()
+    ):
+        """
+        Compile one file per library name in libraries with 4 threads, the compile commands
+        waiting for each other in pairs. dependencies are (file index, index of the file it depends
+        on) pairs. Return the printer output and the compile commands that ran at the same time as
+        another.
+        """
+        simif = create_simulator_interface()
+        simif.supports_concurrent_compiles_into_library = supports_concurrent_compiles_into_library
+        simif.compile_source_file_command.side_effect = lambda source_file: [Path(source_file.name).stem]
+        project, _ = create_project(
+            [(f"file{idx}.vhd", library) for idx, library in enumerate(libraries)], dependencies
+        )
+
+        barrier = threading.Barrier(2, timeout=0.5)
+        concurrent = set()
+
+        def check_output_side_effect(command, env=None):  # pylint: disable=unused-argument
+            try:
+                barrier.wait()
+                concurrent.add(command[0])
+            except threading.BrokenBarrierError:
+                barrier.reset()  # Ran alone, let the next compiles meet
+            if command[0] == fail:
+                raise subprocess.CalledProcessError(returncode=-1, cmd=command, output="bad stuff")
+            return command[0] + " output\n"
+
+        with mock.patch("vunit.sim_if.check_output", autospec=True) as check_output:
+            check_output.side_effect = check_output_side_effect
+            printer = MockPrinter()
+            simif.compile_source_files(project, printer=printer, num_threads=4, continue_on_error=True)
+        return printer.output, concurrent
+
+    def test_compile_source_files_of_different_libraries_concurrently(self):
+        _, concurrent = self._compile_concurrently(["lib1", "lib2"])
+        self.assertEqual(concurrent, {"file0", "file1"})
+
+    def test_compile_source_files_of_a_library_one_at_a_time(self):
+        _, concurrent = self._compile_concurrently(["lib", "lib"])
+        self.assertEqual(concurrent, set())
+
+    def test_compile_source_files_of_a_library_concurrently_when_supported(self):
+        _, concurrent = self._compile_concurrently(["lib", "lib"], supports_concurrent_compiles_into_library=True)
+        self.assertEqual(concurrent, {"file0", "file1"})
+
+    def test_compile_source_files_not_while_a_library_they_read_is_compiled_into(self):
+        # Once file0 is compiled, file1 is compiled into lib1, which file2 reads
+        _, concurrent = self._compile_concurrently(["lib1", "lib1", "lib2"], dependencies=[(1, 0), (2, 0)])
+        self.assertEqual(concurrent, set())
+
+    def test_compile_source_files_reading_the_same_library_concurrently(self):
+        _, concurrent = self._compile_concurrently(["lib1", "lib2", "lib3"], dependencies=[(1, 0), (2, 0)])
+        self.assertEqual(concurrent, {"file1", "file2"})
+
+    def test_compile_source_files_concurrently_writes_the_output_of_a_file_at_once(self):
+        output, _ = self._compile_concurrently(["lib1", "lib2"], fail="file1")
+        self.assertIn("Compiling into lib1: file0.vhd passed\nfile0 output\n", output)
+        self.assertIn("Compiling into lib2: file1.vhd failed\n=== Command used: ===\nfile1\n", output)
+        self.assertTrue(output.endswith("Compile failed\n"))
+
+    def test_compile_source_files_waits_for_dependencies(self):
+        simif = create_simulator_interface()
+        simif.compile_source_file_command.side_effect = lambda source_file: [Path(source_file.name).stem]
+        project, (file1, file2, file3) = create_project(
+            [("file1.vhd", "lib1"), ("file2.vhd", "lib2"), ("file3.vhd", "lib1")], dependencies=[(2, 1)]
+        )
+
+        def check_output_side_effect(command, env=None):  # pylint: disable=unused-argument
+            if command == ["file2"]:
+                raise subprocess.CalledProcessError(returncode=-1, cmd=command, output="bad stuff")
+            return ""
+
+        with mock.patch("vunit.sim_if.check_output", autospec=True) as check_output:
+            check_output.side_effect = check_output_side_effect
+            printer = MockPrinter()
+            simif.compile_source_files(project, printer=printer, num_threads=4, continue_on_error=True)
+            self.assertIn("Compiling into lib1: file3.vhd skipped\n", printer.output)
+            self.assertEqual(len(check_output.mock_calls), 2)
+        self.assertEqual(project.get_files_in_compile_order(incremental=True), [file2, file3])
+        self.assertNotIn(file1, project.get_files_in_compile_order(incremental=True))
+
     @mock.patch("os.environ", autospec=True)
     def test_find_prefix(self, environ):
         class MySimulatorInterface(SimulatorInterface):  # pylint: disable=abstract-method
@@ -319,6 +403,23 @@ def create_simulator_interface():
     return simif
 
 
+def create_project(files, dependencies=()):
+    """
+    Create a project of VHDL files, (name, library) pairs. dependencies are (file index, index of
+    the file it depends on) pairs. Return the project and its source files.
+    """
+    project = Project()
+    source_files = []
+    for name, library in files:
+        if library not in [existing.name for existing in project.get_libraries()]:
+            project.add_library(library, library + "_path")
+        write_file(name, "")
+        source_files.append(project.add_source_file(name, library, file_type="vhdl"))
+    for idx, depends_on in dependencies:
+        project.add_manual_dependency(source_files[idx], depends_on=source_files[depends_on])
+    return project, source_files
+
+
 class MockPrinter(object):
     """
     Mock printer that accumulates the calls as a string

tests/unit/test_ui.py

@@ -1180,6 +1180,23 @@ def test_can_list_files_without_simulator(self):
             ui = self._create_ui_real_sim("--files")
             self._run_main(ui, 0)
 
+    def _compile_threads(self, *args):
+        """
+        The number of threads the files of a ui created with args are compiled with
+        """
+        ui = self._create_ui(*args)
+        simulator_if = mock.Mock(spec=SimulatorInterface)
+        ui._compile(simulator_if)  # pylint: disable=protected-access
+        return simulator_if.compile_project.call_args.kwargs["num_threads"]
+
+    def test_compile_one_file_at_a_time_without_parallel_compile(self):
+        self.assertEqual(self._compile_threads("-p", "4"), 1)
+        self.assertEqual(self._compile_threads("-p", "0"), 1)
+
+    def test_compile_in_parallel_with_parallel_compile(self):
+        self.assertEqual(self._compile_threads("-p", "4", "--parallel-compile"), 4)
+        self.assertEqual(self._compile_threads("-p", "0", "--parallel-compile"), 0)
+
     @mock.patch("vunit.ui.LOGGER", autospec=True)
     def test_compile_without_simulator_fails(self, logger):
         with set_env():