Add --batch-compile to compile files of a library in one call

docs/cli.rst

@@ -117,6 +117,19 @@ previous results and recent code changes to achieve two main goals:
    has a history of failing, and no relevant code changes have been made, so it is assumed to fail again and is run first.
 2. Load balancing - When tests are executed in parallel using the ``-p`` option, VUnit distributes them across threads to minimize to total execution time.
 
+With ``--parallel-compile``, as many files as ``-p`` gives are also compiled in parallel. A file is compiled once the
+files it depends on are compiled. Most simulators do not lock a library while they update it, so a file is not compiled
+while another file is compiled into its library, or into a library it reads. NVC locks its libraries and compiles any
+files at the same time. Without ``--parallel-compile``, files are compiled one at a time.
+
+With ``--batch-compile``, the VHDL files of a library that are ready to be compiled at the same time, and have the same
+compile options, are compiled in one call to the compiler, rather than one call per file. This makes a clean compile
+several times faster, since most of the time of compiling a file goes into starting the compiler and updating the
+library. A batch that fails is compiled again one file at a time, to find the file that fails. ``--batch-compile`` is
+supported by GHDL, NVC and Riviera-PRO, and can be combined with ``--parallel-compile``. NVC and Riviera-PRO read the
+file names of a batch from a file. GHDL gets them in the command, so a batch ends before its command gets longer than
+32000 characters, below the limit of a command line on Windows.
+
 Opening a Test Case in Simulator GUI
 ====================================
 

docs/news.d/1235.feature.rst

@@ -0,0 +1,3 @@
+Added ``--parallel-compile``, which compiles as many files in parallel as ``-p`` gives: a file is compiled once the files
+it depends on are compiled. Unless the simulator locks its libraries, as NVC does, a file is not compiled while its
+library, or a library it reads, is compiled into.

docs/news.d/1236.feature.rst

@@ -0,0 +1,3 @@
+Added ``--batch-compile``, which compiles the VHDL files of a library that are ready to be compiled at the same time, and
+have the same compile options, in one call to the compiler. A batch that fails is compiled again one file at a time.
+Supported by GHDL, NVC and Riviera-PRO.

tests/unit/test_ghdl_interface.py

@@ -137,6 +137,19 @@ def test_assertion_on_unknown_backend(self, check_output):
         check_output.return_value = version
         self.assertRaises(AssertionError, GHDLInterface.determine_backend, "prefix")
 
+    @mock.patch.object(GHDLInterface, "determine_version", return_value=5.0)
+    def test_setup_library_mapping_creates_libraries(self, determine_version):
+        simif = GHDLInterface(prefix="prefix", output_path="")
+        project = Project()
+        project.add_library("lib", "lib_path")
+        write_file(str(Path("existing_path") / "existing-obj08.cf"), "")
+        project.add_library("existing", "existing_path")
+        simif.setup_library_mapping(project)
+        self.create_library.assert_called_once_with(
+            [str(Path("prefix") / "ghdl"), "-i", "--workdir=lib_path", "--work=lib", "--std=08"],
+            env=simif.get_env(),
+        )
+
     @mock.patch("vunit.sim_if.check_output", autospec=True, return_value="")  # pylint: disable=no-self-use
     @mock.patch.object(GHDLInterface, "determine_version", return_value=6.0)
     def test_compile_project_2019(self, determine_version, check_output):
@@ -312,6 +325,9 @@ def test_compile_project_verilog_error(self, determine_version):
     def setUp(self):
         self.output_path = str(Path(__file__).parent / "test_ghdl_interface_out")
         renew_path(self.output_path)
+        patcher = mock.patch("vunit.sim_if.ghdl.subprocess.check_call", autospec=True)
+        self.create_library = patcher.start()
+        self.addCleanup(patcher.stop)
         self.project = Project()
         self.cwd = os.getcwd()
         os.chdir(self.output_path)

tests/unit/test_rivierapro_interface.py

@@ -25,6 +25,40 @@ class TestRivieraProInterface(unittest.TestCase):
     Test the RivieraPro interface
     """
 
+    @mock.patch("vunit.sim_if.check_output", autospec=True)
+    @mock.patch("vunit.sim_if.rivierapro.Process", autospec=True)
+    @mock.patch("vunit.sim_if.rivierapro.RivieraProInterface.find_prefix", return_value="prefix")
+    def _compile_in_a_batch(self, names, _find_prefix, _process, check_output):
+        """
+        Compile the files names in a batch. Return the file lists and the names compiled on their own.
+        """
+        simif = RivieraProInterface(prefix="prefix", output_path=self.output_path)
+        project = Project()
+        project.add_library("lib", "lib_path")
+        for name in names:
+            write_file(name, "")
+            project.add_source_file(name, "lib", file_type="vhdl", vhdl_standard=VHDL.standard("2008"))
+        file_lists = []
+        names_on_their_own = []
+
+        def check_output_side_effect(command, env=None):  # pylint: disable=unused-argument
+            if command[-2] == "-f":
+                file_lists.append(Path(command[-1]).read_text())
+            else:
+                names_on_their_own.append(command[-1])
+            return ""
+
+        check_output.side_effect = check_output_side_effect
+        simif.compile_project(project, batch=True)
+        self.assertEqual(list(Path(self.output_path).glob("batch_*.txt")), [])
+        return file_lists, names_on_their_own
+
+    def test_compile_project_in_a_batch_with_a_file_list(self):
+        file_lists, names_on_their_own = self._compile_in_a_batch(["file.vhd", "sp ace.vhd"])
+        self.assertEqual(len(file_lists), 1)
+        self.assertEqual(sorted(file_lists[0].splitlines()), ['"file.vhd"', '"sp ace.vhd"'])
+        self.assertEqual(names_on_their_own, [])
+
     @mock.patch("vunit.sim_if.check_output", autospec=True, return_value="")
     @mock.patch("vunit.sim_if.rivierapro.Process", autospec=True)
     @mock.patch("vunit.sim_if.rivierapro.RivieraProInterface.find_prefix", return_value="prefix")

tests/unit/test_simulator_interface.py

@@ -12,6 +12,7 @@
 from pathlib import Path
 from os import chdir, getcwd
 import subprocess
+import threading
 from shutil import rmtree
 from unittest import mock
 from vunit.project import Project
@@ -190,6 +191,205 @@ def raise_compile_error(source_file):  # pylint: disable=unused-argument
             self.assertRaises(CompileError, simif.compile_source_files, project)
         self.assertEqual(project.get_files_in_compile_order(incremental=True), [source_file])
 
+    def _compile_concurrently(
+        self, libraries, supports_concurrent_compiles_into_library=False, fail=None, dependencies=()
+    ):
+        """
+        Compile one file per library name in libraries with 4 threads, the compile commands
+        waiting for each other in pairs. dependencies are (file index, index of the file it depends
+        on) pairs. Return the printer output and the compile commands that ran at the same time as
+        another.
+        """
+        simif = create_simulator_interface()
+        simif.supports_concurrent_compiles_into_library = supports_concurrent_compiles_into_library
+        simif.compile_source_file_command.side_effect = lambda source_file: [Path(source_file.name).stem]
+        project, _ = create_project(
+            [(f"file{idx}.vhd", library) for idx, library in enumerate(libraries)], dependencies
+        )
+
+        barrier = threading.Barrier(2, timeout=0.5)
+        concurrent = set()
+
+        def check_output_side_effect(command, env=None):  # pylint: disable=unused-argument
+            try:
+                barrier.wait()
+                concurrent.add(command[0])
+            except threading.BrokenBarrierError:
+                barrier.reset()  # Ran alone, let the next compiles meet
+            if command[0] == fail:
+                raise subprocess.CalledProcessError(returncode=-1, cmd=command, output="bad stuff")
+            return command[0] + " output\n"
+
+        with mock.patch("vunit.sim_if.check_output", autospec=True) as check_output:
+            check_output.side_effect = check_output_side_effect
+            printer = MockPrinter()
+            simif.compile_source_files(project, printer=printer, num_threads=4, continue_on_error=True)
+        return printer.output, concurrent
+
+    def test_compile_source_files_of_different_libraries_concurrently(self):
+        _, concurrent = self._compile_concurrently(["lib1", "lib2"])
+        self.assertEqual(concurrent, {"file0", "file1"})
+
+    def test_compile_source_files_of_a_library_one_at_a_time(self):
+        _, concurrent = self._compile_concurrently(["lib", "lib"])
+        self.assertEqual(concurrent, set())
+
+    def test_compile_source_files_of_a_library_concurrently_when_supported(self):
+        _, concurrent = self._compile_concurrently(["lib", "lib"], supports_concurrent_compiles_into_library=True)
+        self.assertEqual(concurrent, {"file0", "file1"})
+
+    def test_compile_source_files_not_while_a_library_they_read_is_compiled_into(self):
+        # Once file0 is compiled, file1 is compiled into lib1, which file2 reads
+        _, concurrent = self._compile_concurrently(["lib1", "lib1", "lib2"], dependencies=[(1, 0), (2, 0)])
+        self.assertEqual(concurrent, set())
+
+    def test_compile_source_files_reading_the_same_library_concurrently(self):
+        _, concurrent = self._compile_concurrently(["lib1", "lib2", "lib3"], dependencies=[(1, 0), (2, 0)])
+        self.assertEqual(concurrent, {"file1", "file2"})
+
+    def test_compile_source_files_concurrently_writes_the_output_of_a_file_at_once(self):
+        output, _ = self._compile_concurrently(["lib1", "lib2"], fail="file1")
+        self.assertIn("Compiling into lib1: file0.vhd passed\nfile0 output\n", output)
+        self.assertIn("Compiling into lib2: file1.vhd failed\n=== Command used: ===\nfile1\n", output)
+        self.assertTrue(output.endswith("Compile failed\n"))
+
+    def test_compile_source_files_waits_for_dependencies(self):
+        simif = create_simulator_interface()
+        simif.compile_source_file_command.side_effect = lambda source_file: [Path(source_file.name).stem]
+        project, (file1, file2, file3) = create_project(
+            [("file1.vhd", "lib1"), ("file2.vhd", "lib2"), ("file3.vhd", "lib1")], dependencies=[(2, 1)]
+        )
+
+        def check_output_side_effect(command, env=None):  # pylint: disable=unused-argument
+            if command == ["file2"]:
+                raise subprocess.CalledProcessError(returncode=-1, cmd=command, output="bad stuff")
+            return ""
+
+        with mock.patch("vunit.sim_if.check_output", autospec=True) as check_output:
+            check_output.side_effect = check_output_side_effect
+            printer = MockPrinter()
+            simif.compile_source_files(project, printer=printer, num_threads=4, continue_on_error=True)
+            self.assertIn("Compiling into lib1: file3.vhd skipped\n", printer.output)
+            self.assertEqual(len(check_output.mock_calls), 2)
+        self.assertEqual(project.get_files_in_compile_order(incremental=True), [file2, file3])
+        self.assertNotIn(file1, project.get_files_in_compile_order(incremental=True))
+
+    def _compile_in_batches(
+        self,
+        files,
+        fail=(),
+        num_threads=1,
+        dependencies=(),
+        supports_concurrent_compiles_into_library=False,
+        batch_file_list_arguments=None,
+    ):  # pylint: disable=too-many-arguments,too-many-locals
+        """
+        Compile files, (name, library, option) tuples, in batches. dependencies are (file index, index
+        of the file it depends on) pairs, and a command naming a file in fail fails. Return the
+        printer output and the file names of each command.
+        """
+        simif = create_simulator_interface()
+        simif.supports_batch_compile = True
+        simif.supports_concurrent_compiles_into_library = supports_concurrent_compiles_into_library
+        simif.batch_file_list_arguments = batch_file_list_arguments
+        simif.compile_source_file_command.side_effect = lambda source_file: [
+            "compile",
+            source_file.library.name,
+            str(source_file.compile_options.get("ghdl.a_flags", [])),
+            source_file.name,
+        ]
+        project, source_files = create_project([(name, library) for name, library, _ in files], dependencies)
+        for source_file, (_, _, option) in zip(source_files, files):
+            if option:
+                source_file.set_compile_option("ghdl.a_flags", [option])
+
+        commands = []
+        lock = threading.Lock()
+
+        def check_output_side_effect(command, env=None):  # pylint: disable=unused-argument
+            names = command[3:]
+            if names and names[0].startswith("@"):
+                names = Path(names[0][1:]).read_text().splitlines()
+            with lock:
+                commands.append(names)
+            if set(names) & set(fail):
+                raise subprocess.CalledProcessError(returncode=-1, cmd=command, output="bad stuff")
+            return ""
+
+        with mock.patch("vunit.sim_if.check_output", autospec=True) as check_output:
+            check_output.side_effect = check_output_side_effect
+            printer = MockPrinter()
+            simif.compile_source_files(
+                project, printer=printer, continue_on_error=True, num_threads=num_threads, batch=True
+            )
+        return printer.output, commands
+
+    def test_compile_source_files_in_a_batch(self):
+        output, commands = self._compile_in_batches(
+            [("file1.vhd", "lib", None), ("file2.vhd", "lib", None)], dependencies=[(1, 0)]
+        )
+        self.assertEqual(commands, [["file1.vhd", "file2.vhd"]])
+        self.assertEqual(
+            output,
+            """\
+Compiling into lib: file1.vhd passed
+Compiling into lib: file2.vhd passed
+Compile passed
+""",
+        )
+
+    def test_compile_source_files_in_batches_with_commands_of_limited_length(self):
+        # A command of a batch is "compile lib [] " followed by the file names
+        with mock.patch("vunit.sim_if._MAX_BATCH_COMMAND_LENGTH", len("compile lib [] file1.vhd file2.vhd")):
+            _, commands = self._compile_in_batches(
+                [("file1.vhd", "lib", None), ("file2.vhd", "lib", None), ("file3.vhd", "lib", None)],
+                dependencies=[(1, 0), (2, 1)],
+            )
+        self.assertEqual(commands, [["file1.vhd", "file2.vhd"], ["file3.vhd"]])
+
+    def test_compile_source_files_in_a_batch_with_a_file_list(self):
+        with mock.patch("vunit.sim_if._MAX_BATCH_COMMAND_LENGTH", 1):
+            _, commands = self._compile_in_batches(
+                [("file1.vhd", "lib", None), ("file2.vhd", "lib", None), ("file3.vhd", "lib", None)],
+                batch_file_list_arguments=("@{}",),
+            )
+        self.assertEqual(sorted(commands[0]), ["file1.vhd", "file2.vhd", "file3.vhd"])
+        self.assertEqual(len(commands), 1)
+        self.assertEqual(list(Path(".").glob("batch_*.txt")), [])
+
+    def test_compile_source_files_in_a_batch_per_library_and_compile_options(self):
+        _, commands = self._compile_in_batches(
+            [("file1.vhd", "lib1", None), ("file2.vhd", "lib1", "-x"), ("file3.vhd", "lib2", None)]
+        )
+        self.assertEqual(sorted(commands), [["file1.vhd"], ["file2.vhd"], ["file3.vhd"]])
+
+    def test_compile_source_files_of_a_failed_batch_one_at_a_time(self):
+        output, commands = self._compile_in_batches(
+            [("file1.vhd", "lib", None), ("file2.vhd", "lib", None), ("file3.vhd", "lib", None)],
+            fail=["file2.vhd"],
+            dependencies=[(2, 1)],
+        )
+        self.assertEqual(sorted(commands[0]), ["file1.vhd", "file2.vhd", "file3.vhd"])
+        self.assertEqual(sorted(commands[1:]), [["file1.vhd"], ["file2.vhd"]])
+        self.assertIn("Compiling 3 files into lib in one call failed, compiling them one at a time\n", output)
+        self.assertIn("Compiling into lib: file1.vhd passed\n", output)
+        self.assertIn("Compiling into lib: file2.vhd failed\n", output)
+        self.assertIn("Compiling into lib: file3.vhd skipped\n", output)
+
+    def _test_compile_source_files_in_batches_in_parallel(self, supports_concurrent_compiles_into_library):
+        _, commands = self._compile_in_batches(
+            [("file1.vhd", "lib1", None), ("file2.vhd", "lib1", None), ("file3.vhd", "lib2", None)],
+            num_threads=4,
+            supports_concurrent_compiles_into_library=supports_concurrent_compiles_into_library,
+        )
+        self.assertEqual(sorted(sorted(command) for command in commands), [["file1.vhd", "file2.vhd"], ["file3.vhd"]])
+
+    def test_compile_source_files_in_batches_in_parallel(self):
+        self._test_compile_source_files_in_batches_in_parallel(False)
+
+    def test_compile_source_files_in_batches_in_parallel_into_a_library(self):
+        self._test_compile_source_files_in_batches_in_parallel(True)
+
     @mock.patch("os.environ", autospec=True)
     def test_find_prefix(self, environ):
         class MySimulatorInterface(SimulatorInterface):  # pylint: disable=abstract-method
@@ -319,6 +519,23 @@ def create_simulator_interface():
     return simif
 
 
+def create_project(files, dependencies=()):
+    """
+    Create a project of VHDL files, (name, library) pairs. dependencies are (file index, index of
+    the file it depends on) pairs. Return the project and its source files.
+    """
+    project = Project()
+    source_files = []
+    for name, library in files:
+        if library not in [existing.name for existing in project.get_libraries()]:
+            project.add_library(library, library + "_path")
+        write_file(name, "")
+        source_files.append(project.add_source_file(name, library, file_type="vhdl"))
+    for idx, depends_on in dependencies:
+        project.add_manual_dependency(source_files[idx], depends_on=source_files[depends_on])
+    return project, source_files
+
+
 class MockPrinter(object):
     """
     Mock printer that accumulates the calls as a string

tests/unit/test_ui.py

@@ -1180,6 +1180,33 @@ def test_can_list_files_without_simulator(self):
             ui = self._create_ui_real_sim("--files")
             self._run_main(ui, 0)
 
+    def _compile_arguments(self, *args):
+        """
+        The keyword arguments the files of a ui created with args are compiled with
+        """
+        ui = self._create_ui(*args)
+        simulator_if = mock.Mock(spec=SimulatorInterface)
+        ui._compile(simulator_if)  # pylint: disable=protected-access
+        return simulator_if.compile_project.call_args.kwargs
+
+    def _compile_threads(self, *args):
+        """
+        The number of threads the files of a ui created with args are compiled with
+        """
+        return self._compile_arguments(*args)["num_threads"]
+
+    def test_compile_in_batches_with_batch_compile(self):
+        self.assertFalse(self._compile_arguments()["batch"])
+        self.assertTrue(self._compile_arguments("--batch-compile")["batch"])
+
+    def test_compile_one_file_at_a_time_without_parallel_compile(self):
+        self.assertEqual(self._compile_threads("-p", "4"), 1)
+        self.assertEqual(self._compile_threads("-p", "0"), 1)
+
+    def test_compile_in_parallel_with_parallel_compile(self):
+        self.assertEqual(self._compile_threads("-p", "4", "--parallel-compile"), 4)
+        self.assertEqual(self._compile_threads("-p", "0", "--parallel-compile"), 0)
+
     @mock.patch("vunit.ui.LOGGER", autospec=True)
     def test_compile_without_simulator_fails(self, logger):
         with set_env():