Dev/dtni branch merge into main

.gitignore

@@ -17,3 +17,6 @@ docs/sphinx/_toc.yml
 
 # Build distributions
 dist/
+
+# Local sample output
+sample_reports/

cvs/cli_plugins/list_plugin.py

@@ -51,8 +51,14 @@ def discover_tests():
                 # Prune non-suite dirs in place so os.walk skips descending them.
                 dirs[:] = [d for d in dirs if d not in skip_dirs]
                 for file in files:
-                    # conftest.py holds fixtures/hooks, not a runnable suite.
-                    if file.endswith(".py") and file not in ("__init__.py", "conftest.py"):
+                    # Skip pytest infra (conftest.py) and private helpers
+                    # (e.g. _shared.py): they are not selectable suites.
+                    if (
+                        file.endswith(".py")
+                        and file != "__init__.py"
+                        and file != "conftest.py"
+                        and not file.startswith("_")
+                    ):
                         rel_path = os.path.relpath(os.path.join(root, file), tests_dir)
                         module_parts = os.path.splitext(rel_path)[0].split(os.sep)
                         # Module path: <tests_path>.<test_name>

cvs/cli_plugins/run_plugin.py

@@ -24,8 +24,12 @@ def get_parser(self, subparsers):
         )
         parser.add_argument(
             "--log-file",
-            default="/tmp/cvs/test.log",
-            help="Pytest: Path to file for logging output (default: /tmp/cvs/test.log)",
+            default=None,
+            metavar="PATH",
+            help=(
+                "Pytest: write logging output to this file (optional). "
+                "Parent directories are created automatically when set."
+            ),
         )
         parser.add_argument(
             "--log-level",

cvs/cli_plugins/unittests/test_run_plugin.py

@@ -83,6 +83,36 @@ def test_run_test_multiple_functions(self, mock_exit, mock_pytest_main):
         mock_pytest_main.assert_called_once_with(expected_args)
         mock_exit.assert_called_once_with(0)
 
+    @patch("cvs.cli_plugins.run_plugin.pytest.main")
+    @patch("cvs.cli_plugins.run_plugin.sys.exit")
+    def test_run_test_omits_log_file_when_not_set(self, mock_exit, mock_pytest_main):
+        """No --log-file is passed to pytest when the user does not request file logging."""
+        args = MagicMock()
+        args.test = "agfhc_cvs"
+        args.function = []
+        args.cluster_file = "/path/to/cluster.json"
+        args.config_file = "/path/to/config.json"
+        args.html = None
+        args.self_contained_html = False
+        args.log_file = None
+        args.log_level = None
+        args.capture = None
+        args.extra_pytest_args = []
+
+        mock_pytest_main.return_value = 0
+
+        with patch.object(self.plugin, "get_test_file", return_value="/mock/path/test.py"):
+            with patch.object(self.plugin, "_validate_json_config"):
+                self.plugin.run(args)
+
+        expected_args = [
+            "/mock/path/test.py",
+            "--cluster_file=/path/to/cluster.json",
+            "--config_file=/path/to/config.json",
+        ]
+        mock_pytest_main.assert_called_once_with(expected_args)
+        mock_exit.assert_called_once_with(0)
+
 
 class TestRunPluginJsonValidation(unittest.TestCase):
     """Tests for RunPlugin._validate_json_config pre-flight checks."""

cvs/conftest.py

@@ -7,12 +7,15 @@
 
 import importlib.metadata
 import json
+import logging
 from pathlib import Path
 
 import pytest
 
 from cvs.lib.report_plugins import HtmlReportManager, cli_option_value
 
+log = logging.getLogger(__name__)
+
 
 def _maybe_autocollect_html(config, suite_name):
     '''
@@ -79,8 +82,8 @@ def _maybe_autocollect_html(config, suite_name):
         return
 
 
-@pytest.hookimpl(tryfirst=True)
-def pytest_configure(config):
+def _sync_suite_name_from_args(config):
+    """Derive suite stem from the first ``*.py`` target in ``config.args``."""
     suite_name = "test"
     for arg in config.args:
         bare = arg.split("::")[0]
@@ -89,8 +92,97 @@ def pytest_configure(config):
             break
     config._suite_name = suite_name
     config._test_html_dir = f"{suite_name}_html"
-    _maybe_autocollect_html(config, suite_name)
+
+
+def _ensure_html_report_manager(config):
+    """Create ``HtmlReportManager`` once; safe if ``pytest_configure`` did not run."""
+    _sync_suite_name_from_args(config)
+    mgr = getattr(config, "_html_report_manager", None)
+    if mgr is not None:
+        return mgr
+
+    _maybe_autocollect_html(config, config._suite_name)
     config._html_report_manager = HtmlReportManager(config)
+    return config._html_report_manager
+
+
+def _auto_register_inference_suite_report(config):
+    from cvs.lib.report.auto_register import try_auto_register_inference_suite_report
+
+    _sync_suite_name_from_args(config)
+    return try_auto_register_inference_suite_report(config)
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_configure(config):
+    _ensure_html_report_manager(config)
+    _auto_register_inference_suite_report(config)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def _cvs_inference_suite_report_session(request):
+    """Initialize the session report store when a suite preset is registered."""
+    from cvs.lib.report.registry import clear_session_results, get_suite_report_config
+    from cvs.lib.report.types import InferenceReportConfig
+
+    if not isinstance(get_suite_report_config(request.config), InferenceReportConfig):
+        yield
+        return
+
+    clear_session_results()
+    yield
+
+
+@pytest.fixture(scope="module", autouse=True)
+def _cvs_inference_suite_report_bind_module(request, _cvs_inference_suite_report_session):
+    """Bind module-scoped suite fixtures into the session store at module teardown."""
+    from cvs.lib.report.registry import bind_session_results, get_suite_report_config
+    from cvs.lib.report.types import InferenceReportConfig
+
+    if not isinstance(get_suite_report_config(request.config), InferenceReportConfig):
+        yield
+        return
+
+    inf_res_dict = None
+    variant_config = None
+    lifecycle = None
+    try:
+        inf_res_dict = request.getfixturevalue("inf_res_dict")
+    except pytest.FixtureLookupError:
+        log.warning(
+            "Inference suite report preset registered but inf_res_dict fixture is missing; "
+            "session-end report will be skipped"
+        )
+        yield
+        return
+    try:
+        variant_config = request.getfixturevalue("variant_config")
+    except pytest.FixtureLookupError:
+        log.warning(
+            "Inference suite report preset registered but variant_config fixture is missing; "
+            "session-end report will be skipped"
+        )
+        yield
+        return
+    try:
+        lifecycle = request.getfixturevalue("lifecycle")
+    except pytest.FixtureLookupError:
+        log.warning(
+            "Inference suite report preset registered but lifecycle fixture is missing; "
+            "session-end report will be skipped"
+        )
+        yield
+        return
+
+    def _bind_at_module_end():
+        bind_session_results(
+            inf_res_dict=inf_res_dict,
+            variant_config=variant_config,
+            lifecycle=lifecycle,
+        )
+
+    request.addfinalizer(_bind_at_module_end)
+    yield
 
 
 # Add all additional cmd line arguments for the script
@@ -156,15 +248,28 @@ def pytest_metadata(metadata):
 
 # Prepare a clean per-run log directory before tests start.
 def pytest_sessionstart(session):
-    session.config._html_report_manager.setup_log_dir()
+    _auto_register_inference_suite_report(session.config)
+    _ensure_html_report_manager(session.config).setup_log_dir()
 
 
 # Capture each test report and attach a per-test external log link.
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_makereport(item, call):  # noqa: ARG001
     outcome = yield
     report = outcome.get_result()
-    report.extras = item.config._html_report_manager.write_test_log(report, item.originalname)
+    report.extras = _ensure_html_report_manager(item.config).write_test_log(report, item.originalname)
+
+    from cvs.lib.report.registry import get_suite_report_config
+    from cvs.lib.report.types import InferenceReportConfig
+
+    if isinstance(get_suite_report_config(item.config), InferenceReportConfig):
+        from cvs.lib.report.inference_wiring import (
+            attach_inference_suite_lifecycle_table,
+            attach_inference_suite_report_row_extra,
+        )
+
+        attach_inference_suite_lifecycle_table(item, report)
+        attach_inference_suite_report_row_extra(item, report)
 
 
 # Replace inline pytest-html log content with a short externalized-log message.
@@ -181,4 +286,6 @@ def pytest_html_results_summary(prefix, summary, postfix):
 @pytest.hookimpl(hookwrapper=True)
 def pytest_sessionfinish(session, exitstatus):  # noqa: ARG001
     yield  # wait for pytest-html and all other plugins to finish writing the report
-    session.config._html_report_manager.create_zip_bundle(session)
+    mgr = _ensure_html_report_manager(session.config)
+    mgr.generate_suite_reports(session)
+    mgr.create_zip_bundle(session)

cvs/core/orchestrators/baremetal.py

@@ -75,7 +75,7 @@ def __init__(self, log, config, stop_on_errors=False):
             stop_on_errors=self.stop_on_errors,
         )
 
-    def exec(self, cmd, hosts=None, timeout=None, detailed=False):
+    def exec(self, cmd, hosts=None, timeout=None, detailed=False, print_console=True):
         """
         Execute command across hosts via SSH (baremetal execution).
 
@@ -85,6 +85,8 @@ def exec(self, cmd, hosts=None, timeout=None, detailed=False):
             timeout: Command timeout
             detailed: If True, return detailed execution info including
                 exit_code (mirrors ContainerOrchestrator.exec).
+            print_console: If False, the command's output is returned but not
+                logged. Use for bulk data the caller parses itself.
 
         Returns:
             Dictionary mapping hosts to execution results
@@ -94,7 +96,7 @@ def exec(self, cmd, hosts=None, timeout=None, detailed=False):
 
         # Use appropriate handle based on target hosts
         if set(hosts) == set(self.hosts):
-            return self.all.exec(cmd, timeout=timeout, detailed=detailed)
+            return self.all.exec(cmd, timeout=timeout, detailed=detailed, print_console=print_console)
         else:
             # For arbitrary subset (including head node), create temporary handle
             pssh = Pssh(
@@ -106,7 +108,10 @@ def exec(self, cmd, hosts=None, timeout=None, detailed=False):
                 host_key_check=False,
                 stop_on_errors=self.stop_on_errors,
             )
-            return pssh.exec(cmd, timeout=timeout, detailed=detailed)
+            try:
+                return pssh.exec(cmd, timeout=timeout, detailed=detailed, print_console=print_console)
+            finally:
+                pssh.destroy_clients()
 
     def sudo_prefix(self):
         """
@@ -130,19 +135,20 @@ def sudo_prefix(self):
             self._needs_sudo = sudo_status.get(self.head_node, False)
         return 'sudo -n ' if self._needs_sudo else ''
 
-    def exec_on_head(self, cmd, timeout=None, detailed=False):
+    def exec_on_head(self, cmd, timeout=None, detailed=False, print_console=True):
         """
         Execute command on head node only via SSH.
 
         Args:
             cmd: Command to execute
             timeout: Command timeout
             detailed: See exec().
+            print_console: See exec().
 
         Returns:
             Dictionary mapping head node to execution result
         """
-        return self.head.exec(cmd, timeout=timeout, detailed=detailed)
+        return self.head.exec(cmd, timeout=timeout, detailed=detailed, print_console=print_console)
 
     def setup_env(self, hosts, env_script=None):
         """Set up environment on hosts."""
@@ -167,7 +173,10 @@ def setup_env(self, hosts, env_script=None):
                 host_key_check=False,
                 stop_on_errors=self.stop_on_errors,
             )
-            result = pssh.exec(f"bash {env_script}", timeout=60, detailed=True)
+            try:
+                result = pssh.exec(f"bash {env_script}", timeout=60, detailed=True)
+            finally:
+                pssh.destroy_clients()
 
         # Check if all hosts succeeded
         success = all(output['exit_code'] == 0 for output in result.values())

cvs/core/orchestrators/base.py

@@ -49,13 +49,14 @@ def exec(self, cmd, hosts=None, timeout=None):
         pass
 
     @abstractmethod
-    def exec_on_head(self, cmd, timeout=None):
+    def exec_on_head(self, cmd, timeout=None, detailed=False):
         """
         Execute command on head node only.
 
         Args:
             cmd: Command to execute
             timeout: Command timeout in seconds
+            detailed: If True, return detailed execution info including exit_code
 
         """
         pass