diff --git a/.gitignore b/.gitignore index 83b4dc2ec..48b7899d5 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ docs/sphinx/_toc.yml # Build distributions dist/ + +# Local sample output +sample_reports/ diff --git a/cvs/cli_plugins/list_plugin.py b/cvs/cli_plugins/list_plugin.py index 106dac53a..6d6be6938 100644 --- a/cvs/cli_plugins/list_plugin.py +++ b/cvs/cli_plugins/list_plugin.py @@ -51,8 +51,14 @@ def discover_tests(): # Prune non-suite dirs in place so os.walk skips descending them. dirs[:] = [d for d in dirs if d not in skip_dirs] for file in files: - # conftest.py holds fixtures/hooks, not a runnable suite. - if file.endswith(".py") and file not in ("__init__.py", "conftest.py"): + # Skip pytest infra (conftest.py) and private helpers + # (e.g. _shared.py): they are not selectable suites. + if ( + file.endswith(".py") + and file != "__init__.py" + and file != "conftest.py" + and not file.startswith("_") + ): rel_path = os.path.relpath(os.path.join(root, file), tests_dir) module_parts = os.path.splitext(rel_path)[0].split(os.sep) # Module path: . diff --git a/cvs/cli_plugins/run_plugin.py b/cvs/cli_plugins/run_plugin.py index ec387da88..2b89c60e2 100644 --- a/cvs/cli_plugins/run_plugin.py +++ b/cvs/cli_plugins/run_plugin.py @@ -24,8 +24,12 @@ def get_parser(self, subparsers): ) parser.add_argument( "--log-file", - default="/tmp/cvs/test.log", - help="Pytest: Path to file for logging output (default: /tmp/cvs/test.log)", + default=None, + metavar="PATH", + help=( + "Pytest: write logging output to this file (optional). " + "Parent directories are created automatically when set." + ), ) parser.add_argument( "--log-level", diff --git a/cvs/cli_plugins/unittests/test_run_plugin.py b/cvs/cli_plugins/unittests/test_run_plugin.py index 6659d80d4..22322361d 100644 --- a/cvs/cli_plugins/unittests/test_run_plugin.py +++ b/cvs/cli_plugins/unittests/test_run_plugin.py @@ -83,6 +83,36 @@ def test_run_test_multiple_functions(self, mock_exit, mock_pytest_main): mock_pytest_main.assert_called_once_with(expected_args) mock_exit.assert_called_once_with(0) + @patch("cvs.cli_plugins.run_plugin.pytest.main") + @patch("cvs.cli_plugins.run_plugin.sys.exit") + def test_run_test_omits_log_file_when_not_set(self, mock_exit, mock_pytest_main): + """No --log-file is passed to pytest when the user does not request file logging.""" + args = MagicMock() + args.test = "agfhc_cvs" + args.function = [] + args.cluster_file = "/path/to/cluster.json" + args.config_file = "/path/to/config.json" + args.html = None + args.self_contained_html = False + args.log_file = None + args.log_level = None + args.capture = None + args.extra_pytest_args = [] + + mock_pytest_main.return_value = 0 + + with patch.object(self.plugin, "get_test_file", return_value="/mock/path/test.py"): + with patch.object(self.plugin, "_validate_json_config"): + self.plugin.run(args) + + expected_args = [ + "/mock/path/test.py", + "--cluster_file=/path/to/cluster.json", + "--config_file=/path/to/config.json", + ] + mock_pytest_main.assert_called_once_with(expected_args) + mock_exit.assert_called_once_with(0) + class TestRunPluginJsonValidation(unittest.TestCase): """Tests for RunPlugin._validate_json_config pre-flight checks.""" diff --git a/cvs/conftest.py b/cvs/conftest.py index 2ec046808..fd1d4705d 100644 --- a/cvs/conftest.py +++ b/cvs/conftest.py @@ -7,12 +7,15 @@ import importlib.metadata import json +import logging from pathlib import Path import pytest from cvs.lib.report_plugins import HtmlReportManager, cli_option_value +log = logging.getLogger(__name__) + def _maybe_autocollect_html(config, suite_name): ''' @@ -79,8 +82,8 @@ def _maybe_autocollect_html(config, suite_name): return -@pytest.hookimpl(tryfirst=True) -def pytest_configure(config): +def _sync_suite_name_from_args(config): + """Derive suite stem from the first ``*.py`` target in ``config.args``.""" suite_name = "test" for arg in config.args: bare = arg.split("::")[0] @@ -89,8 +92,97 @@ def pytest_configure(config): break config._suite_name = suite_name config._test_html_dir = f"{suite_name}_html" - _maybe_autocollect_html(config, suite_name) + + +def _ensure_html_report_manager(config): + """Create ``HtmlReportManager`` once; safe if ``pytest_configure`` did not run.""" + _sync_suite_name_from_args(config) + mgr = getattr(config, "_html_report_manager", None) + if mgr is not None: + return mgr + + _maybe_autocollect_html(config, config._suite_name) config._html_report_manager = HtmlReportManager(config) + return config._html_report_manager + + +def _auto_register_inference_suite_report(config): + from cvs.lib.report.auto_register import try_auto_register_inference_suite_report + + _sync_suite_name_from_args(config) + return try_auto_register_inference_suite_report(config) + + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): + _ensure_html_report_manager(config) + _auto_register_inference_suite_report(config) + + +@pytest.fixture(scope="session", autouse=True) +def _cvs_inference_suite_report_session(request): + """Initialize the session report store when a suite preset is registered.""" + from cvs.lib.report.registry import clear_session_results, get_suite_report_config + from cvs.lib.report.types import InferenceReportConfig + + if not isinstance(get_suite_report_config(request.config), InferenceReportConfig): + yield + return + + clear_session_results() + yield + + +@pytest.fixture(scope="module", autouse=True) +def _cvs_inference_suite_report_bind_module(request, _cvs_inference_suite_report_session): + """Bind module-scoped suite fixtures into the session store at module teardown.""" + from cvs.lib.report.registry import bind_session_results, get_suite_report_config + from cvs.lib.report.types import InferenceReportConfig + + if not isinstance(get_suite_report_config(request.config), InferenceReportConfig): + yield + return + + inf_res_dict = None + variant_config = None + lifecycle = None + try: + inf_res_dict = request.getfixturevalue("inf_res_dict") + except pytest.FixtureLookupError: + log.warning( + "Inference suite report preset registered but inf_res_dict fixture is missing; " + "session-end report will be skipped" + ) + yield + return + try: + variant_config = request.getfixturevalue("variant_config") + except pytest.FixtureLookupError: + log.warning( + "Inference suite report preset registered but variant_config fixture is missing; " + "session-end report will be skipped" + ) + yield + return + try: + lifecycle = request.getfixturevalue("lifecycle") + except pytest.FixtureLookupError: + log.warning( + "Inference suite report preset registered but lifecycle fixture is missing; " + "session-end report will be skipped" + ) + yield + return + + def _bind_at_module_end(): + bind_session_results( + inf_res_dict=inf_res_dict, + variant_config=variant_config, + lifecycle=lifecycle, + ) + + request.addfinalizer(_bind_at_module_end) + yield # Add all additional cmd line arguments for the script @@ -156,7 +248,8 @@ def pytest_metadata(metadata): # Prepare a clean per-run log directory before tests start. def pytest_sessionstart(session): - session.config._html_report_manager.setup_log_dir() + _auto_register_inference_suite_report(session.config) + _ensure_html_report_manager(session.config).setup_log_dir() # Capture each test report and attach a per-test external log link. @@ -164,7 +257,19 @@ def pytest_sessionstart(session): def pytest_runtest_makereport(item, call): # noqa: ARG001 outcome = yield report = outcome.get_result() - report.extras = item.config._html_report_manager.write_test_log(report, item.originalname) + report.extras = _ensure_html_report_manager(item.config).write_test_log(report, item.originalname) + + from cvs.lib.report.registry import get_suite_report_config + from cvs.lib.report.types import InferenceReportConfig + + if isinstance(get_suite_report_config(item.config), InferenceReportConfig): + from cvs.lib.report.inference_wiring import ( + attach_inference_suite_lifecycle_table, + attach_inference_suite_report_row_extra, + ) + + attach_inference_suite_lifecycle_table(item, report) + attach_inference_suite_report_row_extra(item, report) # Replace inline pytest-html log content with a short externalized-log message. @@ -181,4 +286,6 @@ def pytest_html_results_summary(prefix, summary, postfix): @pytest.hookimpl(hookwrapper=True) def pytest_sessionfinish(session, exitstatus): # noqa: ARG001 yield # wait for pytest-html and all other plugins to finish writing the report - session.config._html_report_manager.create_zip_bundle(session) + mgr = _ensure_html_report_manager(session.config) + mgr.generate_suite_reports(session) + mgr.create_zip_bundle(session) diff --git a/cvs/core/orchestrators/baremetal.py b/cvs/core/orchestrators/baremetal.py index c1baecc12..512c3d30b 100644 --- a/cvs/core/orchestrators/baremetal.py +++ b/cvs/core/orchestrators/baremetal.py @@ -75,7 +75,7 @@ def __init__(self, log, config, stop_on_errors=False): stop_on_errors=self.stop_on_errors, ) - def exec(self, cmd, hosts=None, timeout=None, detailed=False): + def exec(self, cmd, hosts=None, timeout=None, detailed=False, print_console=True): """ Execute command across hosts via SSH (baremetal execution). @@ -85,6 +85,8 @@ def exec(self, cmd, hosts=None, timeout=None, detailed=False): timeout: Command timeout detailed: If True, return detailed execution info including exit_code (mirrors ContainerOrchestrator.exec). + print_console: If False, the command's output is returned but not + logged. Use for bulk data the caller parses itself. Returns: Dictionary mapping hosts to execution results @@ -94,7 +96,7 @@ def exec(self, cmd, hosts=None, timeout=None, detailed=False): # Use appropriate handle based on target hosts if set(hosts) == set(self.hosts): - return self.all.exec(cmd, timeout=timeout, detailed=detailed) + return self.all.exec(cmd, timeout=timeout, detailed=detailed, print_console=print_console) else: # For arbitrary subset (including head node), create temporary handle pssh = Pssh( @@ -106,7 +108,10 @@ def exec(self, cmd, hosts=None, timeout=None, detailed=False): host_key_check=False, stop_on_errors=self.stop_on_errors, ) - return pssh.exec(cmd, timeout=timeout, detailed=detailed) + try: + return pssh.exec(cmd, timeout=timeout, detailed=detailed, print_console=print_console) + finally: + pssh.destroy_clients() def sudo_prefix(self): """ @@ -130,7 +135,7 @@ def sudo_prefix(self): self._needs_sudo = sudo_status.get(self.head_node, False) return 'sudo -n ' if self._needs_sudo else '' - def exec_on_head(self, cmd, timeout=None, detailed=False): + def exec_on_head(self, cmd, timeout=None, detailed=False, print_console=True): """ Execute command on head node only via SSH. @@ -138,11 +143,12 @@ def exec_on_head(self, cmd, timeout=None, detailed=False): cmd: Command to execute timeout: Command timeout detailed: See exec(). + print_console: See exec(). Returns: Dictionary mapping head node to execution result """ - return self.head.exec(cmd, timeout=timeout, detailed=detailed) + return self.head.exec(cmd, timeout=timeout, detailed=detailed, print_console=print_console) def setup_env(self, hosts, env_script=None): """Set up environment on hosts.""" @@ -167,7 +173,10 @@ def setup_env(self, hosts, env_script=None): host_key_check=False, stop_on_errors=self.stop_on_errors, ) - result = pssh.exec(f"bash {env_script}", timeout=60, detailed=True) + try: + result = pssh.exec(f"bash {env_script}", timeout=60, detailed=True) + finally: + pssh.destroy_clients() # Check if all hosts succeeded success = all(output['exit_code'] == 0 for output in result.values()) diff --git a/cvs/core/orchestrators/base.py b/cvs/core/orchestrators/base.py index 4a923ac2a..afb2c93e1 100644 --- a/cvs/core/orchestrators/base.py +++ b/cvs/core/orchestrators/base.py @@ -49,13 +49,14 @@ def exec(self, cmd, hosts=None, timeout=None): pass @abstractmethod - def exec_on_head(self, cmd, timeout=None): + def exec_on_head(self, cmd, timeout=None, detailed=False): """ Execute command on head node only. Args: cmd: Command to execute timeout: Command timeout in seconds + detailed: If True, return detailed execution info including exit_code """ pass diff --git a/cvs/core/orchestrators/container.py b/cvs/core/orchestrators/container.py index 0e32b6c47..ed7cb22e7 100644 --- a/cvs/core/orchestrators/container.py +++ b/cvs/core/orchestrators/container.py @@ -11,6 +11,29 @@ from cvs.core.runtimes import RuntimeFactory +DEFAULT_SSHD_PORT = 2224 + + +def sshd_port_listen_probe_cmd(port: int = DEFAULT_SSHD_PORT) -> str: + """Shell probe that prints OK when TCP *port* accepts connections inside the container.""" + return ( + "bash -c '" + f"(ss -ltn 2>/dev/null | grep -q :{port}) || " + f"(netstat -ltn 2>/dev/null | grep -q :{port}) || " + f"(echo >/dev/tcp/127.0.0.1/{port}) " + "2>/dev/null && echo OK || echo NO'" + ) + + +def sshd_port_listen_ok(output) -> bool: + """Return True when a container exec result indicates the sshd port is open.""" + if isinstance(output, dict): + text = output.get("stdout") or output.get("output", "") + else: + text = output + return "OK" in (text or "") + + # Default container configuration - matches the original docker command DEFAULT_CONTAINER_ARGS = { "devices": [ @@ -504,6 +527,14 @@ def setup_sshd(self): else: self.log.info(f"SSH daemon started successfully on {hostname}") + listen_cmd = sshd_port_listen_probe_cmd(self.ssh_port) + listen_result = self.exec(listen_cmd, timeout=10, detailed=True) + for hostname, output in listen_result.items(): + if output.get("exit_code") != 0 or not sshd_port_listen_ok(output): + self.log.error(f"SSH daemon not listening on port {self.ssh_port} on {hostname}") + return False + self.log.info(f"SSH daemon listening on port {self.ssh_port} on {hostname}") + return True def verify_containers_running(self, container_name): @@ -596,7 +627,7 @@ def get_container_name(container_config, image): container_name = f"{username}_{sanitized_image}" return container_name - def exec(self, cmd, hosts=None, timeout=None, detailed=False): + def exec(self, cmd, hosts=None, timeout=None, detailed=False, print_console=True): """ Execute command in running containers. @@ -605,6 +636,9 @@ def exec(self, cmd, hosts=None, timeout=None, detailed=False): hosts: Target hosts (if None, uses all hosts) timeout: Command timeout detailed: If True, return detailed execution info including exit_code + print_console: If False, the command's output is returned but not + logged. Use for bulk data the caller parses itself — a single + unfiltered dump can otherwise reach hundreds of MB. Returns: Dictionary mapping hosts to execution results @@ -615,7 +649,7 @@ def exec(self, cmd, hosts=None, timeout=None, detailed=False): if not self.container_id: raise RuntimeError("No containers running. Call setup_containers() first.") - return self.runtime.exec(self.container_id, cmd, hosts, timeout, detailed) + return self.runtime.exec(self.container_id, cmd, hosts, timeout, detailed=detailed, print_console=print_console) def exec_cmd_list(self, cmd_list, timeout=None): """ @@ -640,18 +674,35 @@ def exec_cmd_list(self, cmd_list, timeout=None): return self.runtime.exec_cmd_list(self.container_id, cmd_list, timeout) - def exec_on_head(self, cmd, timeout=None): + def exec_on_host(self, cmd, hosts=None, timeout=None, detailed=False, print_console=True): + """Execute command on the cluster host OS (SSH), not inside the container.""" + return super().exec( + cmd, + hosts=hosts, + timeout=timeout, + detailed=detailed, + print_console=print_console, + ) + + def exec_on_head(self, cmd, timeout=None, detailed=False, print_console=True): """ Execute command directly on head node (baremetal). Args: cmd: Command to execute on head node timeout: Command timeout + detailed: If True, return detailed execution info including + exit_code. Mirrors BaremetalOrchestrator.exec_on_head, whose + build_mpi_cmd path calls this with detailed=True. + print_console: If False, the command's output is returned but not + logged. See exec(). Returns: Dictionary mapping head node to execution result """ - return self.runtime.exec_on_head(self.container_id, cmd, timeout) + return self.runtime.exec_on_head( + self.container_id, cmd, timeout, detailed=detailed, print_console=print_console + ) def distribute_using_mpi( self, diff --git a/cvs/core/orchestrators/unittests/test_baremetal.py b/cvs/core/orchestrators/unittests/test_baremetal.py index a096c5773..82608c6ff 100644 --- a/cvs/core/orchestrators/unittests/test_baremetal.py +++ b/cvs/core/orchestrators/unittests/test_baremetal.py @@ -54,7 +54,7 @@ def test_exec_delegates_to_all_when_targeting_full_set(self, _mock_pssh): orch.all = MagicMock() orch.all.exec.return_value = {"10.0.0.1": "ok", "10.0.0.2": "ok"} result = orch.exec("ls", timeout=5) - orch.all.exec.assert_called_once_with("ls", timeout=5, detailed=False) + orch.all.exec.assert_called_once_with("ls", timeout=5, detailed=False, print_console=True) self.assertEqual(result, {"10.0.0.1": "ok", "10.0.0.2": "ok"}) @patch("cvs.core.orchestrators.baremetal.Pssh") @@ -63,9 +63,49 @@ def test_exec_on_head_delegates_to_head_handle(self, _mock_pssh): orch.head = MagicMock() orch.head.exec.return_value = {"10.0.0.1": "ok"} result = orch.exec_on_head("hostname", timeout=10) - orch.head.exec.assert_called_once_with("hostname", timeout=10, detailed=False) + orch.head.exec.assert_called_once_with("hostname", timeout=10, detailed=False, print_console=True) self.assertEqual(result, {"10.0.0.1": "ok"}) + @patch("cvs.core.orchestrators.baremetal.Pssh") + def test_exec_forwards_print_console_false_to_all(self, _mock_pssh): + """print_console=False must reach the pssh handle, not be swallowed here. + + A dropped kwarg is silent -- the command still works, it just logs + hundreds of MB -- so this is pinned explicitly. + """ + orch = BaremetalOrchestrator(MagicMock(), _make_orch_config()) + orch.all = MagicMock() + orch.exec("cat /tmp/huge", print_console=False) + self.assertIs(orch.all.exec.call_args.kwargs["print_console"], False) + + @patch("cvs.core.orchestrators.baremetal.Pssh") + def test_exec_forwards_print_console_false_to_host_subset(self, mock_pssh): + """The subset branch builds its own Pssh; it must forward too. + + orch.all is stubbed with a distinct mock so that falling through to the + all-hosts branch would fail this test rather than silently satisfy it + -- both handles would otherwise be the same patched Pssh return value. + """ + orch = BaremetalOrchestrator(MagicMock(), _make_orch_config()) + orch.all = MagicMock() + mock_pssh.reset_mock() + orch.exec("cat /tmp/huge", hosts=["10.0.0.2"], print_console=False) + # A subset handle was constructed for exactly the requested host... + mock_pssh.assert_called_once() + self.assertEqual(mock_pssh.call_args.args[1], ["10.0.0.2"]) + # ...the all-hosts handle was bypassed... + orch.all.exec.assert_not_called() + # ...and the kwarg reached the subset handle. + subset_handle = mock_pssh.return_value + self.assertIs(subset_handle.exec.call_args.kwargs["print_console"], False) + + @patch("cvs.core.orchestrators.baremetal.Pssh") + def test_exec_on_head_forwards_print_console_false(self, _mock_pssh): + orch = BaremetalOrchestrator(MagicMock(), _make_orch_config()) + orch.head = MagicMock() + orch.exec_on_head("cat /tmp/huge", print_console=False) + self.assertIs(orch.head.exec.call_args.kwargs["print_console"], False) + @patch("cvs.core.orchestrators.baremetal.Pssh") def test_cleanup_returns_true(self, _mock_pssh): orch = BaremetalOrchestrator(MagicMock(), _make_orch_config()) @@ -239,5 +279,64 @@ def test_sudo_prefix_probes_at_most_once_across_multiple_calls(self, mock_pssh): pssh_instance.exec.assert_called_once_with("sudo -n true >/dev/null 2>&1; echo $?") +class TestBaremetalOrchestratorSubsetHandleCleanup(unittest.TestCase): + """The subset branch builds a throwaway Pssh; it must be destroyed. + + Left to refcounting, a call whose exec timed out keeps its SSH session + open on the target host, so a polling suite accumulates sessions until + sshd's limit is hit. + """ + + @patch("cvs.core.orchestrators.baremetal.Pssh") + def test_exec_destroys_subset_handle(self, mock_pssh): + # The timeout path is the one that leaks, so cleanup must not depend + # on a clean return. + for label, side_effect in (("returns", None), ("raises", RuntimeError("timed out"))): + with self.subTest(label): + orch = BaremetalOrchestrator(MagicMock(), _make_orch_config()) + orch.all = MagicMock() + mock_pssh.reset_mock() + mock_pssh.return_value.exec.side_effect = side_effect + + if side_effect is None: + orch.exec("hostname", hosts=["10.0.0.2"]) + else: + with self.assertRaises(RuntimeError): + orch.exec("sleep 300", hosts=["10.0.0.2"], timeout=1) + + mock_pssh.return_value.destroy_clients.assert_called_once_with() + + @patch("cvs.core.orchestrators.baremetal.Pssh") + def test_setup_env_destroys_subset_handle(self, mock_pssh): + # Same subset branch as exec(); it has no callers today, but it is an + # abstractmethod on the base class, so an implementation could reach it. + for label, side_effect in (("returns", None), ("raises", RuntimeError("timed out"))): + with self.subTest(label): + orch = BaremetalOrchestrator(MagicMock(), _make_orch_config()) + orch.all = MagicMock() + mock_pssh.reset_mock() + if side_effect is None: + mock_pssh.return_value.exec.return_value = {"10.0.0.2": {"exit_code": 0}} + mock_pssh.return_value.exec.side_effect = None + orch.setup_env(["10.0.0.2"], env_script="/tmp/env.sh") + else: + mock_pssh.return_value.exec.side_effect = side_effect + with self.assertRaises(RuntimeError): + orch.setup_env(["10.0.0.2"], env_script="/tmp/env.sh") + + mock_pssh.return_value.destroy_clients.assert_called_once_with() + + @patch("cvs.core.orchestrators.baremetal.Pssh") + def test_exec_does_not_destroy_shared_all_handle(self, _mock_pssh): + # self.all is long-lived and reused; tearing it down would break + # every later call. + orch = BaremetalOrchestrator(MagicMock(), _make_orch_config()) + orch.all = MagicMock() + + orch.exec("hostname") + + orch.all.destroy_clients.assert_not_called() + + if __name__ == "__main__": unittest.main() diff --git a/cvs/core/orchestrators/unittests/test_container.py b/cvs/core/orchestrators/unittests/test_container.py index ee9eccec2..0b7ff11cc 100644 --- a/cvs/core/orchestrators/unittests/test_container.py +++ b/cvs/core/orchestrators/unittests/test_container.py @@ -253,12 +253,22 @@ def test_setup_sshd_multinode_attempts_setup(self): orch, runtime = self._make(lifetime="per_run") orch.container_id = "cvs_iter_test" runtime.exec.return_value = { - "10.0.0.1": {"exit_code": 0}, - "10.0.0.2": {"exit_code": 0}, + "10.0.0.1": {"exit_code": 0, "stdout": "OK\n"}, + "10.0.0.2": {"exit_code": 0, "stdout": "OK\n"}, } self.assertTrue(orch.setup_sshd()) self.assertTrue(runtime.exec.called) + def test_sshd_port_listen_probe_falls_back_to_dev_tcp(self): + cmd = __import__( + "cvs.core.orchestrators.container", fromlist=["sshd_port_listen_probe_cmd"] + ).sshd_port_listen_probe_cmd(2224) + self.assertIn("/dev/tcp/127.0.0.1/2224", cmd) + ok = __import__("cvs.core.orchestrators.container", fromlist=["sshd_port_listen_ok"]).sshd_port_listen_ok + self.assertTrue(ok({"stdout": "OK\n"})) + self.assertTrue(ok({"output": "OK\n"})) + self.assertFalse(ok({"stdout": "NO\n"})) + # ------------------------------------------------------------------ # teardown_containers lifetime branching # ------------------------------------------------------------------ @@ -296,6 +306,70 @@ def test_teardown_containers_short_circuits_when_no_container_id(self): runtime.teardown_containers.assert_not_called() +class TestContainerOrchestratorExecForwarding(unittest.TestCase): + """print_console / detailed must survive the orchestrator -> runtime hop. + + Mirrors the forwarding pins in test_baremetal.py, but for the container + path -- which is the one the vLLM suite actually runs on, and the one that + previously dropped both kwargs. A dropped kwarg here is silent: the command + still succeeds, it just logs hundreds of MB again. + + Asserted by keyword rather than positionally so the pin keeps holding if + the runtime signature gains a parameter. + """ + + def setUp(self): + p_pssh = patch("cvs.core.orchestrators.baremetal.Pssh") + p_rf = patch("cvs.core.orchestrators.container.RuntimeFactory") + self.mock_pssh = p_pssh.start() + self.mock_rf = p_rf.start() + self.addCleanup(p_pssh.stop) + self.addCleanup(p_rf.stop) + self.runtime = MagicMock(name="docker_runtime") + self.mock_rf.create.return_value = self.runtime + self.orch = ContainerOrchestrator(MagicMock(), _make_orch_config()) + # exec()/exec_on_head() raise unless a container is registered. + self.orch.container_id = "cvs_iter_test" + + def _kwarg(self, call, name, position): + """Read an argument passed either by keyword or positionally. + + Fails the test (rather than raising IndexError) when the argument was + not forwarded at all, so a dropped kwarg reads as the assertion it is. + """ + if name in call.kwargs: + return call.kwargs[name] + if position >= len(call.args): + self.fail(f"{name!r} was not forwarded to the runtime (call was {call})") + return call.args[position] + + def test_exec_forwards_print_console_false_to_runtime(self): + self.orch.exec("cat /tmp/huge", print_console=False) + self.assertIs(self._kwarg(self.runtime.exec.call_args, "print_console", 5), False) + + def test_exec_defaults_print_console_true(self): + self.orch.exec("ls") + self.assertIs(self._kwarg(self.runtime.exec.call_args, "print_console", 5), True) + + def test_exec_forwards_detailed_to_runtime(self): + self.orch.exec("ls", detailed=True) + self.assertIs(self._kwarg(self.runtime.exec.call_args, "detailed", 4), True) + + def test_exec_on_head_forwards_print_console_false_to_runtime(self): + self.orch.exec_on_head("cat /tmp/huge", print_console=False) + self.assertIs(self._kwarg(self.runtime.exec_on_head.call_args, "print_console", 4), False) + + def test_exec_on_head_defaults_print_console_true(self): + self.orch.exec_on_head("hostname") + self.assertIs(self._kwarg(self.runtime.exec_on_head.call_args, "print_console", 4), True) + + def test_exec_on_head_forwards_detailed_to_runtime(self): + # build_mpi_cmd calls exec_on_head(detailed=True); pinned so the + # container path cannot regress it independently of baremetal. + self.orch.exec_on_head("hostname", detailed=True) + self.assertIs(self._kwarg(self.runtime.exec_on_head.call_args, "detailed", 3), True) + + class TestResolveContainerLifetime(unittest.TestCase): """One assertion per row of the lifetime resolution table.""" diff --git a/cvs/core/runtimes/base.py b/cvs/core/runtimes/base.py index 9df3f99c5..f370c7ee2 100644 --- a/cvs/core/runtimes/base.py +++ b/cvs/core/runtimes/base.py @@ -29,12 +29,17 @@ def is_running(self, container_name): """ ... - def exec(self, container_name, cmd, hosts=None, timeout=None): - """Execute command in running containers.""" + def exec(self, container_name, cmd, hosts=None, timeout=None, detailed=False, print_console=True): + """Execute command in running containers. + + print_console=False returns the output without logging it; use for bulk + data the caller parses itself. + """ ... - def exec_on_head(self, container_name, cmd, timeout=None): - """Execute command directly on head node (baremetal).""" + def exec_on_head(self, container_name, cmd, timeout=None, detailed=False, print_console=True): + """Execute command directly on head node (baremetal). See exec() for + the detailed and print_console semantics.""" ... def load_image(self, tar_path, timeout=None): diff --git a/cvs/core/runtimes/docker.py b/cvs/core/runtimes/docker.py index 2fe7fce78..0ac73c468 100644 --- a/cvs/core/runtimes/docker.py +++ b/cvs/core/runtimes/docker.py @@ -162,6 +162,14 @@ def setup_containers( if not self.registry_login(runtime_args_config['registry']): return False + if not self.check_image_exists(image): + self.log.info(f"Image {image} not present on all hosts; pulling before start") + pull_result = self.pull_image(image, timeout=600) + failed_pull = [host for host, res in pull_result.items() if res.get('exit_code') != 0] + if failed_pull: + self.log.error(f"Failed to pull image on hosts: {failed_pull}") + return False + cmd = f"{self.orchestrator.sudo_prefix()}docker run -d --name {container_name} {all_args_str} {image} sleep infinity" self.log.info(f"Starting long-running containers on {len(self.orchestrator.hosts)} nodes: {container_name}") @@ -171,7 +179,7 @@ def setup_containers( remove_cmd = f"{self.orchestrator.sudo_prefix()}docker rm -f {container_name} || true" self.orchestrator.all.exec(remove_cmd, timeout=30, print_console=False) - result = self.orchestrator.all.exec(cmd, timeout=60, detailed=True) + result = self.orchestrator.all.exec(cmd, timeout=120, detailed=True) # Check if all hosts started successfully success = all(output['exit_code'] == 0 for output in result.values()) @@ -227,12 +235,15 @@ def teardown_containers(self, container_name): return success - def exec(self, container_name, cmd, hosts=None, timeout=None, detailed=False): + def exec(self, container_name, cmd, hosts=None, timeout=None, detailed=False, print_console=True): """Execute command in running Docker containers. cmd is wrapped in `bash -c` so shell features (cd, ;, &&, |, globs, redirects) run inside the container -- docker exec uses execve with no implicit shell. + + print_console=False returns the output without logging it; use for bulk + data the caller parses itself. """ exec_cmd = f"{self.orchestrator.sudo_prefix()}docker exec {container_name} bash -c {shlex.quote(cmd)}" if hosts: @@ -249,9 +260,12 @@ def exec(self, container_name, cmd, hosts=None, timeout=None, detailed=False): host_key_check=False, stop_on_errors=self.orchestrator.stop_on_errors, ) - return pssh.exec(exec_cmd, timeout=timeout, detailed=detailed) + try: + return pssh.exec(exec_cmd, timeout=timeout, detailed=detailed, print_console=print_console) + finally: + pssh.destroy_clients() - return self.orchestrator.all.exec(exec_cmd, timeout=timeout, detailed=detailed) + return self.orchestrator.all.exec(exec_cmd, timeout=timeout, detailed=detailed, print_console=print_console) def exec_cmd_list(self, container_name, cmd_list, timeout=None): """Execute different commands on different hosts inside the container. @@ -272,11 +286,11 @@ def exec_cmd_list(self, container_name, cmd_list, timeout=None): exec_cmd_list = [f"{sudo_prefix}docker exec {container_name} bash -c {shlex.quote(cmd)}" for cmd in cmd_list] return self.orchestrator.all.exec_cmd_list(exec_cmd_list, timeout=timeout) - def exec_on_head(self, container_name, cmd, timeout=None): + def exec_on_head(self, container_name, cmd, timeout=None, detailed=False, print_console=True): """Execute command directly on head node (container). See exec() for - the bash -c wrap rationale.""" + the bash -c wrap rationale and the print_console semantics.""" exec_cmd = f"{self.orchestrator.sudo_prefix()}docker exec {container_name} bash -c {shlex.quote(cmd)}" - return self.orchestrator.head.exec(exec_cmd, timeout=timeout) + return self.orchestrator.head.exec(exec_cmd, timeout=timeout, detailed=detailed, print_console=print_console) @staticmethod def _build_runtime_args(runtime_args_config): @@ -337,6 +351,21 @@ def _build_runtime_args(runtime_args_config): return args + def pull_image(self, image_name, timeout=None): + """Pull container image on all hosts. + + Uses sudo_prefix() like every other docker call in this class. A + hardcoded `sudo` would pull as root while registry_login() -- which + does honor sudo_prefix() -- authenticated as the SSH user, so a private + image would fail with "pull access denied" on any cluster without + passwordless sudo. Bare `sudo` also drops the `-n`, letting a password + prompt block until the timeout instead of failing fast. + """ + timeout = timeout or 600 + cmd = f"{self.orchestrator.sudo_prefix()}docker pull {shlex.quote(image_name)}" + self.log.info(f"Pulling image on all hosts: {image_name}") + return self.orchestrator.all.exec(cmd, timeout=timeout, detailed=True) + def load_image(self, tar_path, timeout=None): """Load container image from tar file on all hosts.""" cmd = f"{self.orchestrator.sudo_prefix()}docker load < {tar_path}" diff --git a/cvs/core/runtimes/enroot.py b/cvs/core/runtimes/enroot.py index 9d62bf4af..61cf11166 100644 --- a/cvs/core/runtimes/enroot.py +++ b/cvs/core/runtimes/enroot.py @@ -28,12 +28,12 @@ def is_running(self, container_name): self.log.error("Enroot runtime not yet implemented") return {} - def exec(self, container_name, cmd, hosts=None, timeout=None): + def exec(self, container_name, cmd, hosts=None, timeout=None, detailed=False, print_console=True): """Execute in Enroot containers - not yet implemented.""" self.log.error("Enroot runtime not yet implemented") return {} - def exec_on_head(self, container_name, cmd, timeout=None): + def exec_on_head(self, container_name, cmd, timeout=None, detailed=False, print_console=True): """Execute on head in Enroot containers - not yet implemented.""" self.log.error("Enroot runtime not yet implemented") return {} diff --git a/cvs/core/runtimes/unittests/test_docker.py b/cvs/core/runtimes/unittests/test_docker.py index 9cc365487..416753379 100644 --- a/cvs/core/runtimes/unittests/test_docker.py +++ b/cvs/core/runtimes/unittests/test_docker.py @@ -20,6 +20,7 @@ # retry, which double-runs the caller's payload whenever it fails for any # reason (not just permission-denied). +import shlex import unittest from unittest.mock import MagicMock, patch @@ -137,6 +138,58 @@ def test_cmd_never_contains_gpus_all(self): f"[{label}] '--gpus all' must never appear in docker cmd:\n{captured[0]}", ) + def _run_missing_image_setup(self, sudo_prefix): + """setup_containers with the image absent, returning every rendered cmd.""" + calls = [] + + def _fake_exec(cmd, timeout=None, detailed=False, print_console=True): + calls.append(cmd) + if "docker images" in cmd and "grep" in cmd: + return {"host1": {"output": "", "exit_code": 1}} + return {"host1": {"output": "", "exit_code": 0}} + + orchestrator = MagicMock() + orchestrator.hosts = ["host1"] + orchestrator.all.exec.side_effect = _fake_exec + orchestrator.sudo_prefix.return_value = sudo_prefix + rt = DockerRuntime(MagicMock(), orchestrator) + + result = rt.setup_containers( + container_config=_container_config(), + container_name="cvs_iter_test", + volumes=["/home/u:/workspace"], + ) + return result, calls + + def test_pulls_image_when_missing_before_run(self): + result, calls = self._run_missing_image_setup("sudo -n ") + + self.assertTrue(result) + self.assertTrue(any("docker pull" in c for c in calls)) + self.assertTrue(any(c.startswith("sudo -n docker run") for c in calls)) + + def test_pull_uses_sudo_prefix_not_hardcoded_sudo(self): + """The pull must carry the same prefix as every other docker call. + + A hardcoded `sudo docker pull` pulls as root while registry_login -- + which honors sudo_prefix() -- authenticated as the SSH user, so root + reads an empty /root/.docker/config.json and a private image fails with + "pull access denied" on any cluster without passwordless sudo. Bare + `sudo` also loses the `-n`, so a password prompt blocks until timeout. + """ + for sudo_prefix in ("", "sudo -n "): + with self.subTest(sudo_prefix=sudo_prefix or ""): + _, calls = self._run_missing_image_setup(sudo_prefix) + + pulls = [c for c in calls if "docker pull" in c] + self.assertEqual(len(pulls), 1, f"expected exactly one pull, got: {pulls}") + self.assertEqual(pulls[0], f"{sudo_prefix}docker pull {shlex.quote('img:test')}") + self.assertNotIn( + "sudo docker pull", + pulls[0], + "pull must use sudo_prefix(), never a hardcoded unconditional `sudo`", + ) + class TestDockerRuntimeRegistryLogin(unittest.TestCase): def test_registry_login_requires_username_and_password_file(self): @@ -361,6 +414,77 @@ def test_exec_on_head_uses_sudo_prefix(self): self.assertTrue(rendered.startswith("docker exec cvs_iter_test bash -c ")) +class TestDockerRuntimeExecPrintConsole(unittest.TestCase): + """print_console must survive the runtime layer. + + DockerRuntime sits between ContainerOrchestrator and Pssh. It previously + dropped print_console on all three exec paths, so a caller asking for a + quiet bulk read still got every line logged -- 486 MB of amd-smi JSON in + one observed vLLM run. A dropped kwarg fails silently, hence these pins. + """ + + def test_exec_forwards_print_console_false(self): + orchestrator = MagicMock() + orchestrator.all.exec.return_value = {"host1": ""} + orchestrator.sudo_prefix.return_value = "" + rt = DockerRuntime(MagicMock(), orchestrator) + + rt.exec("cvs_iter_test", "cat /tmp/huge", print_console=False) + + self.assertIs(orchestrator.all.exec.call_args.kwargs["print_console"], False) + + def test_exec_with_hosts_subset_forwards_print_console_false(self): + orchestrator = MagicMock() + orchestrator.sudo_prefix.return_value = "" + rt = DockerRuntime(MagicMock(), orchestrator) + + with patch("cvs.lib.parallel_ssh_lib.Pssh") as mock_pssh_cls: + mock_pssh = MagicMock() + mock_pssh.exec.return_value = {"host1": ""} + mock_pssh_cls.return_value = mock_pssh + + rt.exec("cvs_iter_test", "cat /tmp/huge", hosts=["host1"], print_console=False) + + self.assertIs(mock_pssh.exec.call_args.kwargs["print_console"], False) + + def test_exec_on_head_forwards_print_console_false(self): + orchestrator = MagicMock() + orchestrator.head.exec.return_value = {"host1": ""} + orchestrator.sudo_prefix.return_value = "" + rt = DockerRuntime(MagicMock(), orchestrator) + + rt.exec_on_head("cvs_iter_test", "cat /tmp/huge", print_console=False) + + self.assertIs(orchestrator.head.exec.call_args.kwargs["print_console"], False) + + def test_exec_on_head_accepts_and_forwards_detailed(self): + """BaremetalOrchestrator.build_mpi_cmd calls exec_on_head(detailed=True). + + The container path lacked the parameter entirely, so that call raised + TypeError for any container-orchestrated MPI job. Pinned here because + distribute_using_mpi has no in-tree caller to catch it. + """ + orchestrator = MagicMock() + orchestrator.head.exec.return_value = {"host1": {"output": "", "exit_code": 0}} + orchestrator.sudo_prefix.return_value = "" + rt = DockerRuntime(MagicMock(), orchestrator) + + rt.exec_on_head("cvs_iter_test", "echo hi", detailed=True) + + self.assertIs(orchestrator.head.exec.call_args.kwargs["detailed"], True) + + def test_default_stays_verbose(self): + """Omitting the kwarg must keep the historical logging behavior.""" + orchestrator = MagicMock() + orchestrator.all.exec.return_value = {"host1": ""} + orchestrator.sudo_prefix.return_value = "" + rt = DockerRuntime(MagicMock(), orchestrator) + + rt.exec("cvs_iter_test", "echo hi") + + self.assertIs(orchestrator.all.exec.call_args.kwargs["print_console"], True) + + class TestDockerRuntimeSudoProbeCachedAcrossCalls(unittest.TestCase): """Regression test for the bug being fixed: with a REAL BaremetalOrchestrator (not a bare MagicMock) as DockerRuntime's orchestrator, the underlying @@ -394,5 +518,43 @@ def test_sudo_probe_fires_once_across_exec_and_exec_on_head(self, mock_pssh): self.assertEqual(len(probe_calls), 1, f"probe must fire once total, calls: {pssh_instance.exec.call_args_list}") +class TestDockerRuntimeExecSubsetHandleCleanup(unittest.TestCase): + """The host-subset branch builds a throwaway Pssh; it must be destroyed. + + Mirrors BaremetalOrchestrator.exec: without an explicit teardown a + timed-out exec leaves its sshd session open on the target host. + """ + + def _make_runtime(self): + orchestrator = MagicMock() + orchestrator.hosts = ["host1"] + orchestrator.log = MagicMock() + orchestrator.user = "u" + orchestrator.password = None + orchestrator.pkey = None + orchestrator.stop_on_errors = False + orchestrator.sudo_prefix.return_value = "" + return DockerRuntime(MagicMock(), orchestrator), orchestrator + + def test_exec_destroys_subset_handle(self): + # The timeout path is the one that leaks, so cleanup must not depend + # on a clean return. + for label, side_effect in (("returns", None), ("raises", RuntimeError("timed out"))): + with self.subTest(label): + rt, _ = self._make_runtime() + + with patch("cvs.lib.parallel_ssh_lib.Pssh") as mock_pssh_cls: + mock_pssh_cls.return_value.exec.side_effect = side_effect + mock_pssh_cls.return_value.exec.return_value = {"host1": {"output": "", "exit_code": 0}} + + if side_effect is None: + rt.exec("cvs_iter_test", "echo hi", hosts=["host1"]) + else: + with self.assertRaises(RuntimeError): + rt.exec("cvs_iter_test", "sleep 300", hosts=["host1"], timeout=1) + + mock_pssh_cls.return_value.destroy_clients.assert_called_once_with() + + if __name__ == "__main__": unittest.main() diff --git a/cvs/input/cluster_file/atom_cluster.json b/cvs/input/cluster_file/atom_cluster.json new file mode 100644 index 000000000..3ed0d9f4d --- /dev/null +++ b/cvs/input/cluster_file/atom_cluster.json @@ -0,0 +1,35 @@ +{ + "_comment": "ATOM cluster template (container backend). Copy to ~/input/cluster_file/atom_cluster.json and edit placeholders. head_node_dict.mgmt_ip MUST be the rank-0 GPU node VPC IP (same as variant params.master_addr for nnodes>1) — not the pytest jumphost. Trim node_dict to one host for single-node variants. Variant config overrides container.image/name/volumes; cluster container block is the fallback default.", + "orchestrator": "container", + "username": "{user-id}", + "priv_key_file": "/home/{user-id}/.ssh/cluster_id_ed25519", + "head_node_dict": { + "mgmt_ip": "{head-node-ip}" + }, + "env_vars": {}, + "_env_vars_comment": "Optional host env exported on each GPU node before container setup (e.g. ROCm install on host). Usually empty when the workload image carries ROCm/vLLM.", + "node_dict": { + "{head-node-ip}": { + "bmc_ip": "NA", + "vpc_ip": "{head-node-ip}" + }, + "{worker-node-ip}": { + "bmc_ip": "NA", + "vpc_ip": "{worker-node-ip}" + } + }, + "container": { + "lifetime": "per_run", + "image": "rocm/atom-dev:latest", + "name": "atom", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G" + } + } + } +} diff --git a/cvs/input/config_file/inference/atom/README.md b/cvs/input/config_file/inference/atom/README.md new file mode 100644 index 000000000..46c72a7e7 --- /dev/null +++ b/cvs/input/config_file/inference/atom/README.md @@ -0,0 +1,291 @@ +# ATOM Inference — Config and Threshold Files + +This folder holds the input files for the `atom` suite (see +`cvs/tests/inference/atom/README.md` for how to run it). Each **config** file +has a sibling **threshold** file (referenced by its `threshold_json` field). +One config = one GPU arch + topology + driver mode (single-node, multinode PP, +baseline matrix, …). + +W1 workloads target **DeepSeek R1 FP8** on 8× GPU per node (ISL/OSL sweeps, +TP8 unless noted). + +## File inventory + +In the CVS repo, variants are flat sibling pairs in **this directory**: + +```text +{gpu}_atom_{model}_{precision}[_{mode}].json +{gpu}_atom_{model}_{precision}[_{mode}]_threshold.json +``` + +| Config | Threshold | GPU | Driver | Notes | +|---|---|---|---|---| +| `mi300x_atom_deepseek-r1_fp8_single` | `…_single_threshold.json` | MI300X | `atom` | W1 single-node; portable min-SLO thresholds; server reuse | +| `mi300x_atom_deepseek-r1_fp8_baseline_sweep` | `…_baseline_sweep_threshold.json` | MI300X | `atom` | DTNI baseline: 1K/1K + 8K/1K × C=4–256 (14 cells) | +| `mi300x_atom_deepseek-r1_fp8_baseline_sweep_distributed` | `…_baseline_sweep_distributed_threshold.json` | MI300X | `vllm_atom` | 2-node DTNI baseline (14 cells); PP=2, scaling gates | +| `mi300x_atom_deepseek-r1_fp8_distributed` | `…_distributed_threshold.json` | MI300X | `vllm_atom` | W1 2-node PP=2; lab-calibrated thresholds | +| `mi300x_atom_deepseek-r1_fp8_mtp3` | `…_mtp3_threshold.json` | MI300X | `atom` | W1 FP8 + MTP3 | +| `mi355x_atom_deepseek-r1_fp8_single` | `…_single_threshold.json` | MI355X | `atom` | W1 single-node; CI seeds, `enforce_thresholds: false` | +| `mi355x_atom_deepseek-r1_fp8_baseline_sweep` | `…_baseline_sweep_threshold.json` | MI355X | `atom` | DTNI baseline matrix; record-only | +| `mi355x_atom_deepseek-r1_fp8_distributed` | `…_distributed_threshold.json` | MI355X | `vllm_atom` | W1 2-node PP=2; record-only until lab confirm | +| `mi355x_atom_deepseek-r1_fp8_mtp3` | `…_mtp3_threshold.json` | MI355X | `atom` | W1 FP8 + MTP3 | + +Add analogous config + threshold pairs for other archs or models as needed. + +Keys prefixed with `_` (e.g. `_comment`) are inline comments and are ignored by +the loader. + +## What you MUST change for your cluster / setup + +Start from the config closest to your target GPU / topology / driver and edit +these: + +| Where | Variable | Change to | +|---|---|---| +| `container.image` | container image | Your ATOM ROCm image on the nodes | +| `container.name` | container name | Any unique name (optional) | +| `paths.shared_fs` | base path | A path reachable from all nodes; `{user-id}` resolves to the cluster/OS user | +| `paths.models_dir` | model cache | Host path to the staged model (shipped configs use `/home/models`) | +| `paths.log_dir` | benchmark logs | Usually `{shared_fs}/LOGS` | +| `paths.hf_token_file` | HF token path | Location of your Hugging Face token file | +| `model.id` | model repo id | The model under test (W1: `deepseek-ai/DeepSeek-R1-0528`) | +| `model.remote` | fetch mode | `0` = already cached on nodes; `1` = not implemented | +| `params.driver` | execution stack | `atom` (single-node) or `vllm_atom` (multinode PP); see below | +| `params.nnodes` | node count | `1` single-node; `2` for shipped multinode PP variants | +| `params.master_addr` | PP coordinator | Head node VPC IP (replace `{head-node-ip}` on multinode stems) | +| `params.master_port` | PP coordinator port | Usually `29501` | +| `params.pipeline_parallel_size` | PP size | `2` on shipped multinode stems | +| `params.scaling_baseline_output_throughput` | 1-node baseline | Measured single-node output tok/s for `scaling.efficiency_pct` (multinode) | +| `roles.server.atom_args` | ATOM server CLI | Tokens after `--model` / `--server-port` when `driver=atom` | +| `roles.server.serve_args` | multinode serve flags | Dict merged into the multinode server argv when `driver=vllm_atom` | +| `roles.server.ib_hca_devices` | RDMA HCAs | `"auto"` (default) or explicit list; probed in `test_discover_topology` | +| `roles.server.ib_netdev` | socket netdev | `"auto"` (default on distributed) or explicit name; **not** `mlx5_*` | +| `roles.server.env` | server env | ATOM / multinode env (e.g. mmap, AITER flags) | +| `.json` gated values | thresholds | Calibrated PASS/FAIL bounds for your hardware | +| cluster file `node_dict` | node IPs | Your node IPs; **host count must equal `params.nnodes`** | +| cluster template | `atom_cluster.json` | Copy from `cvs/input/cluster_file/atom_cluster.json` | + +Also set `enforce_thresholds` to `true` for real PASS/FAIL or `false` for +record-only (MI355X stems ship record-only until lab calibration). + +### Lab directory layout + +On your lab machine (`~/input/config_file/inference/atom/`), copy each variant +into its **own subdirectory** so threshold discovery is unambiguous: + +```text +~/input/.../atom/single/ # single-node config + threshold only +~/input/.../atom/distributed/ # multinode PP=2 (driver=vllm_atom) +~/input/.../atom/baseline_sweep/ # DTNI single-node matrix +``` + +`substitute_config` globs the config's parent directory; multiple `*threshold.json` +files in one folder raises `ValueError: multiple *threshold.json files … (ambiguous)`. + +Each shipped config sets `"threshold_json"` to the sibling threshold filename +(relative to the config directory). You may also use an absolute path. + +## Placeholder substitution + +Configs use placeholders resolved at load time: + +- `{user-id}` — the cluster username (or the local OS user as fallback). +- `{shared_fs}` — self-reference within the `paths` block. +- `{paths.models_dir}` (and other `{paths.*}`) — cross-referenced anywhere. +- `{head-node-ip}` — replace manually in copied multinode configs (not auto-resolved). + +`threshold_json` is a literal filename resolved next to the config; no +placeholder substitution is applied to it. + +## Config structure + +Top-level (framework-agnostic) fields: + +| Field | Meaning | +|---|---| +| `schema_version` | Always `1` | +| `framework` | `atom` | +| `gpu_arch` | `mi300x` / `mi355x` (labels the run) | +| `enforce_thresholds` | `true` = metrics gate PASS/FAIL; `false` = record-only | +| `threshold_json` | Sibling threshold filename | +| `run_card` | Optional metadata (`atom_image_pin`, `upstream_run_url`, `notes`) logged at session start | +| `paths` | `shared_fs`, `models_dir`, `log_dir`, `hf_token_file` | +| `model` | `id`, `remote` (0 = already cached), `precision` (label) | +| `container` | `lifetime`, `name`, `image`, `runtime` (docker `args`: network/ipc/privileged/shm-size/volumes/devices) | + +### `params` block + +| Field | Meaning | +|---|---| +| `driver` | `atom` (single-node) or `vllm_atom` (multinode PP) | +| `tensor_parallelism` | TP size (W1: `8`) | +| `pipeline_parallel_size` | PP size (`1` single-node; `2` on multinode stems) | +| `nnodes` | Node count (`1` or `2` on shipped variants) | +| `master_addr` / `master_port` | Multinode PP coordinator address/port | +| `port_no` | Server HTTP port (default `8000`) | +| `num_prompts` | Benchmark prompt count per cell | +| `max_model_length` | Server MML; must cover `(ISL+OSL) × (1+random_range_ratio)` | +| `random_range_ratio` | Random workload ratio passed to bench client | +| `metric_percentiles` | Tail percentiles requested (e.g. `95,99`) | +| `reuse_server_across_sweep` | `true` = keep server warm across cells with matching session key | +| `scaling_baseline_output_throughput` | Single-node output tok/s baseline for `scaling.efficiency_pct` | +| `bench_extra_args` | Extra bench client tokens (MTP3 variants) | +| `server_*` / `client_*` poll waits | Timeouts for server ready and client completion | + +### `roles.server` block + +| Field | Meaning | +|---|---| +| `atom_args` | Extra CLI tokens for `python -m atom.entrypoints.openai_server` (`driver=atom`) | +| `serve_args` | Multinode server flags when `driver=vllm_atom` | +| `env` | Exported in `/tmp/server_env_script.sh` (orchestrator-managed NCCL keys are stripped) | +| `ib_hca_devices` | `"auto"` or explicit HCA list for `NCCL_IB_HCA` | +| `ib_netdev` | `"auto"` or explicit socket interface for `NCCL/GLOO/TP_SOCKET_IFNAME` | + +### Execution drivers (`params.driver`) + +Standalone ATOM has **no native pipeline parallel**. Single-node variants use +the native ATOM server; shipped multinode PP stems use `vllm_atom` (same ATOM +bench client and ROCm env, with a PP coordinator for 2-node runs): + +| Driver | When to use | Server entrypoint | Multinode PP | +|---|---|---|---| +| `atom` | Single-node W1, baseline sweep, MTP3 | `atom.entrypoints.openai_server` | No | +| `vllm_atom` | Shipped 2-node PP stems | Multinode serve path + ATOM ROCm env | Yes | + +Multinode fabric is probed once per run in `test_discover_topology` (on the +**cluster host OS**, not inside the container). Probes can be skipped on +single-node (`nnodes=1`). Lazy resolution also runs on first `build_server_cmd` +if topology discovery is omitted via a smoke `-k` filter. + +### `sweep` block + +| Field | Meaning | +|---|---| +| `sequence_combinations` | Named `(isl, osl)` shapes, e.g. `{name, isl, osl}` | +| `runs` | `{combo, concurrency}` pairs — one benchmark cell each | + +Each run produces a **threshold cell key** via `cell_key()`: + +- Single-node: `ISL=1024,OSL=1024,TP=8,CONC=128` +- Multinode PP: `ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128` + +The key must match a top-level entry in the threshold file. Parametrize IDs in +pytest look like `w1_1k_1k-conc128` or `w1_1k_1k-conc128-throughput` (metric +tier suffix on gate rows). + +## Threshold files + +A threshold file maps each **cell key** to `{metric: spec}`. Metrics use the +`client.*` namespace (plus `scaling.efficiency_pct` on multinode). A metric is +gated only when `enforce_thresholds: true`, the metric belongs to the tier under +test, and a spec exists; otherwise it is recorded. Metrics missing from the +benchmark artifact are skipped (ATOM may omit tail percentiles). + +Threshold kinds: + +| kind | Passes when | Notes | +|---|---|---| +| `min` | `actual >= value` | lower bound (e.g. success_rate) | +| `max` | `actual <= value` | upper bound (e.g. failed count) | +| `max_ms` | `actual <= value` | upper bound, `ms` in the message | +| `min_tok_s` | `actual >= value` | lower bound, `tok/s` in the message | +| `within` | `value +/- tolerance_pct%` | needs `tolerance_pct` | +| `min_ratio` | `actual / actuals[reference] >= value` | needs `reference` | +| `info` | always | record-only; retains a default `value` to calibrate later | + +Example single-node cell: + +```json +"ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": { "kind": "min_tok_s", "value": 3000 }, + "client.output_throughput": { "kind": "min_tok_s", "value": 1500 }, + "client.per_gpu_throughput": { "kind": "min_tok_s", "value": 375 }, + "client.p99_ttft_ms": { "kind": "max_ms", "value": 1000000 }, + "client.success_rate": { "kind": "min", "value": 1 }, + "client.failed": { "kind": "max", "value": 0 } +} +``` + +Example multinode cell (adds scaling): + +```json +"ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.output_throughput": { "kind": "min_tok_s", "value": 2500 }, + "client.p99_ttft_ms": { "kind": "max_ms", "value": 5000 }, + "scaling.efficiency_pct": { "kind": "min", "value": 80 } +} +``` + +To start gating a metric currently marked `info`: replace `"kind": "info"` with +`min`/`max`/etc. and set a calibrated `value`. The threshold cell key must match +the sweep cell exactly (including `PP` and `NNODES` on multinode), or the metric +falls back to record-only. + +## Cluster file + +Template: `cvs/input/cluster_file/atom_cluster.json`. Copy to +`~/input/cluster_file/atom_cluster.json` and edit IPs, `username`, and +`priv_key_file`. + +| Variant type | `params.nnodes` | `node_dict` | +|---|---|---| +| Single-node (`*_single`, baseline sweep, MTP3) | `1` | Head node only | +| Multinode PP (`*_distributed`, `*_baseline_sweep_distributed`) | `2` | Head + worker | + +`test_setup_sshd` runs when `len(node_dict) > 1`. + +## Running on a lab machine + +**Launcher vs GPU node:** CVS pytest runs on the launcher; +`ContainerOrchestrator` SSHes to cluster nodes and runs `sudo docker` there. + +| Item | Launcher | GPU node | +|---|---|---| +| `cvs run`, venv, `~/input/`, `~/cvs_results/` | Yes | No | +| `priv_key_file`, HF token file | Yes | No | +| `/home/models` (when `model.remote: 0`) | No | Yes | +| Container image, `sudo docker` | No | Yes | +| `~/LOGS/` (via volume mount) | No | Yes | + +After `git pull`, run **`make install` first**, then **`source .cvs_venv/bin/activate`** +(do not activate the venv before `make install`). + +Typical workflow: + +```bash +cd ~/cvs +make install +source .cvs_venv/bin/activate + +SINGLE_DIR=~/input/config_file/inference/atom/single +mkdir -p "$SINGLE_DIR" + +cvs copy-config inference/atom/mi300x_atom_deepseek-r1_fp8_single.json \ + --output "$SINGLE_DIR/mi300x_atom_deepseek-r1_fp8_single.json" +cvs copy-config inference/atom/mi300x_atom_deepseek-r1_fp8_single_threshold.json \ + --output "$SINGLE_DIR/mi300x_atom_deepseek-r1_fp8_single_threshold.json" +cvs copy-config atom_cluster.json --output ~/input/cluster_file/atom_cluster.json + +# Edit cluster IPs; trim node_dict to one host for single-node variants. + +TS=$(date +%Y%m%d_%H%M%S) +cvs run atom \ + --cluster_file ~/input/cluster_file/atom_cluster.json \ + --config_file "$SINGLE_DIR/mi300x_atom_deepseek-r1_fp8_single.json" \ + --html=~/cvs_results/${TS}_atom-w1-single_mi300x.html \ + --self-contained-html \ + --log-file=~/cvs_results/${TS}_atom-w1-single_mi300x.log \ + -vvv -s +``` + +For multinode PP variants, use a two-host cluster file, copy the matching +`*_distributed*` config pair into its own subdirectory, set `container.image`, +`params.master_addr`, and verify fabric discovery (or set `roles.server.ib_netdev` +explicitly). + +Smoke a single cell with `-k`, for example `-k "w1_1k_1k-conc128"`. + +When `--html` is set, the **ATOM Run Deck** is generated at session end and +bundled into the pytest zip (render-only; does not affect gates). See +`cvs/lib/report/README.md`. diff --git a/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_baseline_sweep.json b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_baseline_sweep.json new file mode 100644 index 000000000..5da5c6fe5 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_baseline_sweep.json @@ -0,0 +1,143 @@ +{ + "_comment": "DTNI baseline sweep: 1K/1K + 8K/1K \u00d7 C=4-256 (14 cells). max_model_length=10240 fits 8192+1024 with headroom. Runs grouped by shape for server reuse.", + "schema_version": 1, + "framework": "atom", + "gpu_arch": "mi300x", + "enforce_thresholds": true, + "threshold_json": "mi300x_atom_deepseek-r1_fp8_baseline_sweep_threshold.json", + "run_card": { + "atom_image_pin": "rocm/atom-dev:latest", + "notes": "DTNI baseline matrix; portable threshold floors until lab calibration" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "atom_mi300x", + "image": "rocm/atom-dev:latest", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "atom_args": [ + "-tp", + "8", + "--kv_cache_dtype", + "fp8", + "--trust-remote-code" + ], + "env": { + "ATOM_DISABLE_MMAP": "true" + } + } + }, + "params": { + "driver": "atom", + "port_no": "8000", + "tensor_parallelism": "8", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "10240", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "baseline_1k_1k", + "isl": "1024", + "osl": "1024" + }, + { + "name": "baseline_8k_1k", + "isl": "8192", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "baseline_1k_1k", + "concurrency": 4 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 8 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 16 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 32 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 64 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 128 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 256 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 4 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 8 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 16 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 32 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 64 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 128 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 256 + } + ] + } +} diff --git a/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_baseline_sweep_distributed.json b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_baseline_sweep_distributed.json new file mode 100644 index 000000000..379579934 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_baseline_sweep_distributed.json @@ -0,0 +1,152 @@ +{ + "_comment": "DTNI baseline sweep multinode (2x8 GPU, PP=2): 1K/1K + 8K/1K x C=4-256 (14 cells). vLLM-ATOM pipeline parallel; align master_addr and ib_netdev with cluster head.", + "schema_version": 1, + "framework": "atom", + "gpu_arch": "mi300x", + "enforce_thresholds": true, + "threshold_json": "mi300x_atom_deepseek-r1_fp8_baseline_sweep_distributed_threshold.json", + "run_card": { + "atom_image_pin": "rocm/atom-dev:latest", + "notes": "MI300X 2-node DTNI baseline matrix (PP=2 vLLM-ATOM); recalibrate thresholds after true PP lab run" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "atom_mi300x_multi", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "kv-cache-dtype": "fp8", + "trust-remote-code": true, + "enforce-eager": true, + "gpu-memory-utilization": "0.95", + "block-size": 64, + "no-enable-prefix-caching": true + }, + "ib_hca_devices": "auto", + "ib_netdev": "auto", + "env": { + "VLLM_ROCM_USE_AITER": "1", + "AMDGCN_USE_BUFFER_OPS": "0" + } + } + }, + "params": { + "driver": "vllm_atom", + "port_no": "8000", + "tensor_parallelism": "8", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "10240", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "client_poll_count": "80", + "client_poll_wait_time": "60", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "{head-node-ip}", + "master_port": "29501", + "scaling_baseline_output_throughput": "1500" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "baseline_1k_1k", + "isl": "1024", + "osl": "1024" + }, + { + "name": "baseline_8k_1k", + "isl": "8192", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "baseline_1k_1k", + "concurrency": 4 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 8 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 16 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 32 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 64 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 128 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 256 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 4 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 8 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 16 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 32 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 64 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 128 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 256 + } + ] + } +} diff --git a/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_baseline_sweep_distributed_threshold.json b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_baseline_sweep_distributed_threshold.json new file mode 100644 index 000000000..641190785 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_baseline_sweep_distributed_threshold.json @@ -0,0 +1,1487 @@ +{ + "_comment": "MI300X 2-node baseline sweep thresholds (PP=2 vLLM-ATOM); recalibrate after true pipeline-parallel lab run.", + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=4": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 12.5 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 6 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 9.0 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=8": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 12 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 15.5 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 25 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 25.0 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=32": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 50 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 40.0 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1600 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 100 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 40.0 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 375 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 188 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 40.0 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=256": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 40.0 + } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,NNODES=2,CONC=4": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 6 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 3 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 7.9 + } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,NNODES=2,CONC=8": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 12.5 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 6 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 13.0 + } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 12 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 19.2 + } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,NNODES=2,CONC=32": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 25 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 27.6 + } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 50 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 34.7 + } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 187 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 40.0 + } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,NNODES=2,CONC=256": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1250 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 312 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 156 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 40.0 + } + } +} diff --git a/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_baseline_sweep_threshold.json b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_baseline_sweep_threshold.json new file mode 100644 index 000000000..63d578729 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_baseline_sweep_threshold.json @@ -0,0 +1,1431 @@ +{ + "_comment": "MI300X baseline sweep portable mins (1K/1K + 8K/1K, C=4-256). Throughput floors are conservative; latency gates loose. Health: success_rate=1, failed=0. Recalibrate after first lab run.", + "ISL=1024,OSL=1024,TP=8,CONC=4": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 12.5 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 6 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=8": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 12 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 25 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=32": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 50 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1600 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 100 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 375 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 188 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=4": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 6 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 3 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=8": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 12.5 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 6 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 12 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=32": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 25 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 50 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 187 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1250 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 312 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 156 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + } + } +} diff --git a/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_distributed.json b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_distributed.json new file mode 100644 index 000000000..d9c3e4980 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_distributed.json @@ -0,0 +1,175 @@ +{ + "_comment": "W1 DeepSeek R1 FP8 multinode (2\u00c3\u20148 GPU, PP=2). vLLM coordinates pipeline parallel; ATOM accelerates local kernels via vllm_atom driver. max_model_length=8192 fits W1 sweep (2k+2k \u00d7 random_range_ratio 0.8). Set roles.server.ib_netdev and container.image before lab run.", + "schema_version": 1, + "framework": "atom", + "gpu_arch": "mi300x", + "enforce_thresholds": true, + "threshold_json": "mi300x_atom_deepseek-r1_fp8_distributed_threshold.json", + "run_card": { + "atom_image_pin": "rocm/atom-dev:latest", + "notes": "MI300X 2-node PP=2 via vLLM-ATOM (M5); align master_addr and ib_netdev with cluster" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "atom_mi300x_multi", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "kv-cache-dtype": "fp8", + "trust-remote-code": true, + "enforce-eager": true, + "gpu-memory-utilization": "0.95", + "block-size": 64, + "no-enable-prefix-caching": true + }, + "ib_hca_devices": "auto", + "ib_netdev": "auto", + "env": { + "VLLM_ROCM_USE_AITER": "1", + "AMDGCN_USE_BUFFER_OPS": "0" + } + } + }, + "params": { + "driver": "vllm_atom", + "port_no": "8000", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "{head-node-ip}", + "master_port": "29501", + "scaling_baseline_output_throughput": "1500", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "8192", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "server_precheck_wait_s": "30", + "server_warmup_wait_s": "330", + "server_poll_count": "120", + "server_poll_wait_time": "60", + "client_poll_count": "150", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_512_512", + "isl": "512", + "osl": "512" + }, + { + "name": "w1_1k_1k", + "isl": "1024", + "osl": "1024" + }, + { + "name": "w1_2k_1k", + "isl": "2048", + "osl": "1024" + }, + { + "name": "w1_1k_2k", + "isl": "1024", + "osl": "2048" + }, + { + "name": "w1_2k_2k", + "isl": "2048", + "osl": "2048" + } + ], + "runs": [ + { + "combo": "w1_512_512", + "concurrency": 16 + }, + { + "combo": "w1_512_512", + "concurrency": 64 + }, + { + "combo": "w1_512_512", + "concurrency": 128 + }, + { + "combo": "w1_1k_1k", + "concurrency": 16 + }, + { + "combo": "w1_1k_1k", + "concurrency": 64 + }, + { + "combo": "w1_1k_1k", + "concurrency": 128 + }, + { + "combo": "w1_2k_1k", + "concurrency": 16 + }, + { + "combo": "w1_2k_1k", + "concurrency": 64 + }, + { + "combo": "w1_2k_1k", + "concurrency": 128 + }, + { + "combo": "w1_1k_2k", + "concurrency": 16 + }, + { + "combo": "w1_1k_2k", + "concurrency": 64 + }, + { + "combo": "w1_1k_2k", + "concurrency": 128 + }, + { + "combo": "w1_2k_2k", + "concurrency": 16 + }, + { + "combo": "w1_2k_2k", + "concurrency": 64 + }, + { + "combo": "w1_2k_2k", + "concurrency": 128 + } + ] + } +} diff --git a/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_distributed_threshold.json b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_distributed_threshold.json new file mode 100644 index 000000000..47378dfb9 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_distributed_threshold.json @@ -0,0 +1,1593 @@ +{ + "_comment": "MI300X W1 multinode thresholds calibrated from lab run 20260728 (10.32.80.112/113, vLLM-ATOM PP=2 full 15-cell sweep). Throughput/scaling mins at 80% of measured; ISL=512/OSL=512 CONC=16 ttft/tpot from 20260724 run at 110%.", + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 671 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 334 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 84 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 42 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 402 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 15441 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 42 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 43 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 11 + } + }, + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 2374 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1180 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 296 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 147 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 39 + } + }, + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3743 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1861 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 467 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 232 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 678 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 337 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 84 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 42 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 11 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 2403 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1194 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 300 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 149 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 39 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3734 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1856 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 466 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 232 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 997 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 329 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 124 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 41 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 10 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1111 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 138 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 37 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1793 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 224 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 511 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 339 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 63 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 42 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 11 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1868 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1240 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 233 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 155 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3245 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 405 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 679 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 337 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 84 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 42 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 11 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 2400 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1193 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 300 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 149 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 39 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + } +} diff --git a/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_mtp3.json b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_mtp3.json new file mode 100644 index 000000000..1ee818ef7 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_mtp3.json @@ -0,0 +1,94 @@ +{ + "_comment": "W1 DeepSeek R1 FP8+MTP3 on 8x MI300X. Recipe: dsr1-fp8-mi300x-atom-mtp3. Thresholds: plan Section 4.2.", + "schema_version": 1, + "framework": "atom", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_atom_deepseek-r1_fp8_mtp3_threshold.json", + "run_card": { + "atom_image_pin": "rocm/atom-dev:latest", + "notes": "MI300X MTP3 lab reference Section 4.2" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "atom_mi300x", + "image": "rocm/atom-dev:latest", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "atom_args": [ + "-tp", + "8", + "--kv_cache_dtype", + "fp8", + "--trust-remote-code", + "--method", + "mtp", + "--num-speculative-tokens", + "3" + ], + "env": { + "ATOM_DISABLE_MMAP": "true" + } + } + }, + "params": { + "driver": "atom", + "port_no": "8000", + "tensor_parallelism": "8", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "4096", + "metric_percentiles": "95,99", + "bench_extra_args": "--use-chat-template", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_1k_1k", + "isl": "1024", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "w1_1k_1k", + "concurrency": 128 + }, + { + "combo": "w1_1k_1k", + "concurrency": 256 + } + ] + } +} diff --git a/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_mtp3_threshold.json b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_mtp3_threshold.json new file mode 100644 index 000000000..7d047749a --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_mtp3_threshold.json @@ -0,0 +1,57 @@ +{ + "_comment": "MI300X W1 FP8+MTP3 seeds from plan Section 4.2 (record-only).", + "ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 12470.4}, + "client.output_throughput": {"kind": "min_tok_s", "value": 6913}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 1558.8}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 777.71}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 17.5}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 20.13}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 0}, + "client.failed": {"kind": "max", "value": 1000000000} + }, + "ISL=1024,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 13124.7}, + "client.output_throughput": {"kind": "min_tok_s", "value": 7284}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 1640.59}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 819.45}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 33.0}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 37.95}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 0}, + "client.failed": {"kind": "max", "value": 1000000000} + } +} diff --git a/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_sglang_distributed.json b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_sglang_distributed.json new file mode 100644 index 000000000..7a11eff35 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_sglang_distributed.json @@ -0,0 +1,172 @@ +{ + "_comment": "W1 DeepSeek R1 FP8 multinode (2x8 GPU, PP=2) via SGLang pipeline parallel. Set ib_netdev and SGLang-capable container.image before lab run.", + "schema_version": 1, + "framework": "atom", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_atom_deepseek-r1_fp8_sglang_distributed_threshold.json", + "run_card": { + "atom_image_pin": "rocm/atom-dev:latest", + "notes": "MI300X 2-node PP=2 via SGLang; enforce_thresholds false until lab confirm" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "atom_mi300x_multi_sglang", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "sglang_args": [ + "--kv-cache-dtype", + "fp8", + "--trust-remote-code", + "--disable-cuda-graph", + "--mem-fraction-static", + "0.9", + "--attention-backend", + "aiter" + ], + "ib_hca_devices": "auto", + "ib_netdev": "auto", + "env": { + "SGLANG_USE_AITER": "1" + } + } + }, + "params": { + "driver": "sglang", + "port_no": "8000", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "{head-node-ip}", + "master_port": "29501", + "scaling_baseline_output_throughput": "1500", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "4096", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_512_512", + "isl": "512", + "osl": "512" + }, + { + "name": "w1_1k_1k", + "isl": "1024", + "osl": "1024" + }, + { + "name": "w1_2k_1k", + "isl": "2048", + "osl": "1024" + }, + { + "name": "w1_1k_2k", + "isl": "1024", + "osl": "2048" + }, + { + "name": "w1_2k_2k", + "isl": "2048", + "osl": "2048" + } + ], + "runs": [ + { + "combo": "w1_512_512", + "concurrency": 16 + }, + { + "combo": "w1_512_512", + "concurrency": 64 + }, + { + "combo": "w1_512_512", + "concurrency": 128 + }, + { + "combo": "w1_1k_1k", + "concurrency": 16 + }, + { + "combo": "w1_1k_1k", + "concurrency": 64 + }, + { + "combo": "w1_1k_1k", + "concurrency": 128 + }, + { + "combo": "w1_2k_1k", + "concurrency": 16 + }, + { + "combo": "w1_2k_1k", + "concurrency": 64 + }, + { + "combo": "w1_2k_1k", + "concurrency": 128 + }, + { + "combo": "w1_1k_2k", + "concurrency": 16 + }, + { + "combo": "w1_1k_2k", + "concurrency": 64 + }, + { + "combo": "w1_1k_2k", + "concurrency": 128 + }, + { + "combo": "w1_2k_2k", + "concurrency": 16 + }, + { + "combo": "w1_2k_2k", + "concurrency": 64 + }, + { + "combo": "w1_2k_2k", + "concurrency": 128 + } + ] + } +} diff --git a/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_sglang_distributed_threshold.json b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_sglang_distributed_threshold.json new file mode 100644 index 000000000..a5de4dd92 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_sglang_distributed_threshold.json @@ -0,0 +1,1593 @@ +{ + "_comment": "MI300X W1 multinode SGLang PP=2 seed thresholds (copy of vLLM-ATOM keys); recalibrate after lab run.", + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 188 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 22 + } + }, + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 188 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 23 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 188 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 22 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 735 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 138 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 92 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 24 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 188 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 24 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 438 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 219 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + } +} diff --git a/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_single.json b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_single.json new file mode 100644 index 000000000..834f65e5c --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_single.json @@ -0,0 +1,90 @@ +{ + "_comment": "W1 DeepSeek R1 FP8 on 8x MI300X. Recipe: dsr1-fp8-mi300x-atom. Thresholds: portable minimum SLOs (throughput + health), not per-node calibration.", + "schema_version": 1, + "framework": "atom", + "gpu_arch": "mi300x", + "enforce_thresholds": true, + "threshold_json": "mi300x_atom_deepseek-r1_fp8_single_threshold.json", + "run_card": { + "atom_image_pin": "rocm/atom-dev:latest", + "notes": "MI300X lab reference Section 4.1" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "atom_mi300x", + "image": "rocm/atom-dev:latest", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "atom_args": [ + "-tp", + "8", + "--kv_cache_dtype", + "fp8", + "--trust-remote-code" + ], + "env": { + "ATOM_DISABLE_MMAP": "true" + } + } + }, + "params": { + "driver": "atom", + "port_no": "8000", + "tensor_parallelism": "8", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "4096", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_1k_1k", + "isl": "1024", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "w1_1k_1k", + "concurrency": 128 + }, + { + "combo": "w1_1k_1k", + "concurrency": 256 + } + ] + } +} diff --git a/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_single_threshold.json b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_single_threshold.json new file mode 100644 index 000000000..19c1cd417 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_single_threshold.json @@ -0,0 +1,57 @@ +{ + "_comment": "MI300X W1 portable minimum SLOs (any healthy lab node). Throughput mins are conservative floors (~50% below typical W1 runs), not per-node calibration. Latency tier gates (ttft/tpot) are loose (1e6 ms) so enforcement focuses on throughput + health. Health: success_rate=1, failed=0.", + "ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 3000}, + "client.output_throughput": {"kind": "min_tok_s", "value": 1500}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 375}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 188}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 1}, + "client.failed": {"kind": "max", "value": 0} + }, + "ISL=1024,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 5000}, + "client.output_throughput": {"kind": "min_tok_s", "value": 2500}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 625}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 313}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 1}, + "client.failed": {"kind": "max", "value": 0} + } +} diff --git a/cvs/input/config_file/inference/atom/mi300x_atom_gpt-oss-120b_bf16.json b/cvs/input/config_file/inference/atom/mi300x_atom_gpt-oss-120b_bf16.json new file mode 100644 index 000000000..6ca5fdd07 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi300x_atom_gpt-oss-120b_bf16.json @@ -0,0 +1,91 @@ +{ + "_comment": "Single-node ATOM (schema_version 1). Set container.image and container.name. ISL+OSL must fit params.max_model_length. Volume bind home only; ContainerOrchestrator adds /home/:/workspace.", + "schema_version": 1, + "framework": "atom", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_atom_gpt-oss-120b_bf16_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "openai/gpt-oss-120b", + "remote": 0, + "precision": "bf16" + }, + "container": { + "lifetime": "per_run", + "name": "", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "enforce-eager": true, + "gpu-memory-utilization": "0.95", + "block-size": 64, + "no-enable-prefix-caching": true + }, + "env": { + "AMDGCN_USE_BUFFER_OPS": "0", + "VLLM_ROCM_USE_AITER": "1", + "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8000", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.8", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "99", + "num_prompts": "1000", + "max_model_length": "8192", + "client_poll_count": "50", + "client_poll_wait_time": "60", + "bench_max_failed_requests": "0" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "legacy_profile", + "isl": "7168", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "legacy_profile", + "concurrency": 64 + } + ] + } +} diff --git a/cvs/input/config_file/inference/atom/mi300x_atom_gpt-oss-120b_bf16_threshold.json b/cvs/input/config_file/inference/atom/mi300x_atom_gpt-oss-120b_bf16_threshold.json new file mode 100644 index 000000000..bffa3c01d --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi300x_atom_gpt-oss-120b_bf16_threshold.json @@ -0,0 +1,105 @@ +{ + "_comment": "client.* thresholds for ISL=7168,OSL=1024,TP=8,CONC=64. Every GATED_METRICS member has a spec so the loader coverage check passes. config.json sets enforce_thresholds=false until calibrated; output_throughput/mean_ttft_ms/mean_tpot_ms carry legacy verify_inference_results targets.", + "ISL=7168,OSL=1024,TP=8,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 4200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 525 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 500 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 15 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + } +} diff --git a/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_baseline_sweep.json b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_baseline_sweep.json new file mode 100644 index 000000000..f8f32281c --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_baseline_sweep.json @@ -0,0 +1,144 @@ +{ + "_comment": "DTNI baseline sweep: 1K/1K + 8K/1K \u00d7 C=4-256 (14 cells). max_model_length=10240 fits 8192+1024 with headroom. enforce_thresholds false until MI355X lab calibration.", + "schema_version": 1, + "framework": "atom", + "gpu_arch": "mi355x", + "enforce_thresholds": false, + "threshold_json": "mi355x_atom_deepseek-r1_fp8_baseline_sweep_threshold.json", + "run_card": { + "upstream_run_url": "https://github.com/ROCm/ATOM/actions/runs/27912164002", + "atom_image_pin": "rocm/atom-dev:nightly_202606211542", + "notes": "DTNI baseline matrix; threshold seeds only until lab confirm" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "atom_mi355x", + "image": "rocm/atom-dev:nightly_202606211542", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "atom_args": [ + "-tp", + "8", + "--kv_cache_dtype", + "fp8", + "--trust-remote-code" + ], + "env": { + "ATOM_DISABLE_MMAP": "true" + } + } + }, + "params": { + "driver": "atom", + "port_no": "8000", + "tensor_parallelism": "8", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "10240", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "baseline_1k_1k", + "isl": "1024", + "osl": "1024" + }, + { + "name": "baseline_8k_1k", + "isl": "8192", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "baseline_1k_1k", + "concurrency": 4 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 8 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 16 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 32 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 64 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 128 + }, + { + "combo": "baseline_1k_1k", + "concurrency": 256 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 4 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 8 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 16 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 32 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 64 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 128 + }, + { + "combo": "baseline_8k_1k", + "concurrency": 256 + } + ] + } +} diff --git a/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_baseline_sweep_threshold.json b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_baseline_sweep_threshold.json new file mode 100644 index 000000000..613369e16 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_baseline_sweep_threshold.json @@ -0,0 +1,1431 @@ +{ + "_comment": "MI355X baseline sweep seeds (record-only until lab). Placeholder throughput mins; flip enforce_thresholds after calibration.", + "ISL=1024,OSL=1024,TP=8,CONC=4": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 12.5 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 6 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=8": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 12 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 25 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=32": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 50 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1600 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 100 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 3000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 375 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 188 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=1024,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 5000 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 625 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 313 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=4": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 6 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 3 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=8": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 12.5 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 6 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 25 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 12 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=32": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 50 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 25 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 800 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 400 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 100 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 50 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 1500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 750 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 187 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 94 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + }, + "ISL=8192,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 2500 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 1250 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 312 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 156 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + } +} diff --git a/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_distributed.json b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_distributed.json new file mode 100644 index 000000000..82386d3e9 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_distributed.json @@ -0,0 +1,172 @@ +{ + "_comment": "W1 DeepSeek R1 FP8 multinode (2\u00c3\u20148 GPU MI355X, PP=2). vLLM-ATOM pipeline parallel; set ib_netdev and container.image before lab run.", + "schema_version": 1, + "framework": "atom", + "gpu_arch": "mi355x", + "enforce_thresholds": false, + "threshold_json": "mi355x_atom_deepseek-r1_fp8_distributed_threshold.json", + "run_card": { + "upstream_run_url": "https://github.com/ROCm/ATOM/actions/runs/27912164002", + "atom_image_pin": "rocm/atom-dev:nightly_202606211542", + "notes": "MI355X 2-node PP=2 via vLLM-ATOM (M5); lab re-run required before enforce_thresholds: true" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "atom_mi355x_multi", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "kv-cache-dtype": "fp8", + "trust-remote-code": true, + "enforce-eager": true, + "gpu-memory-utilization": "0.95", + "block-size": 64, + "no-enable-prefix-caching": true + }, + "ib_hca_devices": "auto", + "ib_netdev": "auto", + "env": { + "VLLM_ROCM_USE_AITER": "1", + "AMDGCN_USE_BUFFER_OPS": "0" + } + } + }, + "params": { + "driver": "vllm_atom", + "port_no": "8000", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "{head-node-ip}", + "master_port": "29501", + "scaling_baseline_output_throughput": "4000", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "4096", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_512_512", + "isl": "512", + "osl": "512" + }, + { + "name": "w1_1k_1k", + "isl": "1024", + "osl": "1024" + }, + { + "name": "w1_2k_1k", + "isl": "2048", + "osl": "1024" + }, + { + "name": "w1_1k_2k", + "isl": "1024", + "osl": "2048" + }, + { + "name": "w1_2k_2k", + "isl": "2048", + "osl": "2048" + } + ], + "runs": [ + { + "combo": "w1_512_512", + "concurrency": 16 + }, + { + "combo": "w1_512_512", + "concurrency": 64 + }, + { + "combo": "w1_512_512", + "concurrency": 128 + }, + { + "combo": "w1_1k_1k", + "concurrency": 16 + }, + { + "combo": "w1_1k_1k", + "concurrency": 64 + }, + { + "combo": "w1_1k_1k", + "concurrency": 128 + }, + { + "combo": "w1_2k_1k", + "concurrency": 16 + }, + { + "combo": "w1_2k_1k", + "concurrency": 64 + }, + { + "combo": "w1_2k_1k", + "concurrency": 128 + }, + { + "combo": "w1_1k_2k", + "concurrency": 16 + }, + { + "combo": "w1_1k_2k", + "concurrency": 64 + }, + { + "combo": "w1_1k_2k", + "concurrency": 128 + }, + { + "combo": "w1_2k_2k", + "concurrency": 16 + }, + { + "combo": "w1_2k_2k", + "concurrency": 64 + }, + { + "combo": "w1_2k_2k", + "concurrency": 128 + } + ] + } +} diff --git a/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_distributed_threshold.json b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_distributed_threshold.json new file mode 100644 index 000000000..071b1ba74 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_distributed_threshold.json @@ -0,0 +1,1593 @@ +{ + "_comment": "MI355X W1 multinode seeded thresholds (scaling.efficiency_pct min 50% floor). Throughput mins scaled from MI300X multinode scaffold \u00d7 single-node MI355X/MI300X ratio. Lab re-run required before enforce_thresholds: true.", + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 8009.32 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 4004.66 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 1003.83 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 501.92 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 18688.41 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 9344.21 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 2338.72 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1169.36 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 26697.73 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 13348.87 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 3337.22 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1671.28 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 8009.32 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 4004.66 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 1003.83 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 501.92 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 18688.41 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 9344.21 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 2338.72 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1169.36 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 26697.73 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 13348.87 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 3337.22 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1671.28 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 8009.32 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 4004.66 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 1003.83 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 501.92 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 18688.41 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 9344.21 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 2338.72 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1169.36 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 26697.73 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 13348.87 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 3337.22 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1671.28 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 8009.32 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 4004.66 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 1003.83 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 501.92 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 18688.41 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 9344.21 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 2338.72 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1169.36 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=1024,OSL=2048,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 26697.73 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 13348.87 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 3337.22 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1671.28 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 8009.32 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 4004.66 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 1003.83 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 501.92 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 18688.41 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 9344.21 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 2338.72 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1169.36 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + }, + "ISL=2048,OSL=2048,TP=8,PP=2,NNODES=2,CONC=128": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 26697.73 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 13348.87 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 3337.22 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 1671.28 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 1 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "scaling.efficiency_pct": { + "kind": "min", + "value": 50 + } + } +} diff --git a/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_mtp3.json b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_mtp3.json new file mode 100644 index 000000000..e9925cd38 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_mtp3.json @@ -0,0 +1,95 @@ +{ + "_comment": "W1 DeepSeek R1 FP8+MTP3 on 8x MI355X. Recipe: dsr1-fp8-mi355x-atom-mtp3. Thresholds: plan Section 4.3.2.", + "schema_version": 1, + "framework": "atom", + "gpu_arch": "mi355x", + "enforce_thresholds": false, + "threshold_json": "mi355x_atom_deepseek-r1_fp8_mtp3_threshold.json", + "run_card": { + "upstream_run_url": "https://github.com/ROCm/ATOM/actions/runs/27912164002", + "atom_image_pin": "rocm/atom-dev:nightly_202606211542", + "notes": "MTP3 cells from ATOM CI run 27912164002" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "atom_mi355x", + "image": "rocm/atom-dev:nightly_202606211542", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "atom_args": [ + "-tp", + "8", + "--kv_cache_dtype", + "fp8", + "--trust-remote-code", + "--method", + "mtp", + "--num-speculative-tokens", + "3" + ], + "env": { + "ATOM_DISABLE_MMAP": "true" + } + } + }, + "params": { + "driver": "atom", + "port_no": "8000", + "tensor_parallelism": "8", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "4096", + "metric_percentiles": "95,99", + "bench_extra_args": "--use-chat-template", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_1k_1k", + "isl": "1024", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "w1_1k_1k", + "concurrency": 128 + }, + { + "combo": "w1_1k_1k", + "concurrency": 256 + } + ] + } +} diff --git a/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_mtp3_threshold.json b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_mtp3_threshold.json new file mode 100644 index 000000000..e2f74248e --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_mtp3_threshold.json @@ -0,0 +1,57 @@ +{ + "_comment": "MI355X W1 FP8+MTP3 calibrated from ROCm/ATOM run 27912164002 (Section 4.3.2). Throughput mins = ATOM CI × 0.9; latency maxes = ATOM CI × 1.1. Lab re-run required before enforce_thresholds: true.", + "ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 0}, + "client.output_throughput": {"kind": "min_tok_s", "value": 4591.79}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 0}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 573.97}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 627.46}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 26.15}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 0}, + "client.failed": {"kind": "max", "value": 1000000000} + }, + "ISL=1024,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 0}, + "client.output_throughput": {"kind": "min_tok_s", "value": 6451.59}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 0}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 806.45}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 667.34}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 37.64}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 0}, + "client.failed": {"kind": "max", "value": 1000000000} + } +} diff --git a/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_single.json b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_single.json new file mode 100644 index 000000000..8c8e268dd --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_single.json @@ -0,0 +1,91 @@ +{ + "_comment": "W1 DeepSeek R1 FP8 on 8x MI355X. Recipe: dsr1-fp8-mi355x-atom. Thresholds: plan Section 4.3 (ATOM run 27912164002).", + "schema_version": 1, + "framework": "atom", + "gpu_arch": "mi355x", + "enforce_thresholds": false, + "threshold_json": "mi355x_atom_deepseek-r1_fp8_single_threshold.json", + "run_card": { + "upstream_run_url": "https://github.com/ROCm/ATOM/actions/runs/27912164002", + "atom_image_pin": "rocm/atom-dev:nightly_202606211542", + "notes": "ATOM commit ea08015; re-pin on image bumps" + }, + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "deepseek-ai/DeepSeek-R1-0528", + "remote": 0, + "precision": "fp8" + }, + "container": { + "lifetime": "per_run", + "name": "atom_mi355x", + "image": "rocm/atom-dev:nightly_202606211542", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "atom_args": [ + "-tp", + "8", + "--kv_cache_dtype", + "fp8", + "--trust-remote-code" + ], + "env": { + "ATOM_DISABLE_MMAP": "true" + } + } + }, + "params": { + "driver": "atom", + "port_no": "8000", + "tensor_parallelism": "8", + "random_range_ratio": "0.8", + "num_prompts": "1000", + "max_model_length": "4096", + "metric_percentiles": "95,99", + "reuse_server_across_sweep": "true", + "client_poll_count": "80", + "client_poll_wait_time": "60" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_1k_1k", + "isl": "1024", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "w1_1k_1k", + "concurrency": 128 + }, + { + "combo": "w1_1k_1k", + "concurrency": 256 + } + ] + } +} diff --git a/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_single_threshold.json b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_single_threshold.json new file mode 100644 index 000000000..3c66d23c2 --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_single_threshold.json @@ -0,0 +1,57 @@ +{ + "_comment": "MI355X W1 calibrated from ROCm/ATOM run 27912164002 (Section 4.3.1). Throughput mins = ATOM CI × 0.9; latency maxes = ATOM CI × 1.1. Tail gates seeded from mean × 2.5 / × 1.15. Lab re-run required before enforce_thresholds: true.", + "ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 8018.11}, + "client.output_throughput": {"kind": "min_tok_s", "value": 4004.66}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 1002.26}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 500.58}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 362.18}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 905.45}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 30.40}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 34.96}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 0}, + "client.failed": {"kind": "max", "value": 1000000000} + }, + "ISL=1024,OSL=1024,TP=8,CONC=256": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 11244.09}, + "client.output_throughput": {"kind": "min_tok_s", "value": 5624.76}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 1405.51}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": 703.10}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 606.83}, + "client.median_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_ttft_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1517.08}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": 43.41}, + "client.median_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 49.92}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_itl_ms": {"kind": "max_ms", "value": 1000000}, + "client.mean_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.median_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p90_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p95_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.p99_e2el_ms": {"kind": "max_ms", "value": 1000000}, + "client.success_rate": {"kind": "min", "value": 0}, + "client.failed": {"kind": "max", "value": 1000000000} + } +} diff --git a/cvs/input/config_file/inference/atom/mi355x_atom_gpt-oss-120b_bf16.json b/cvs/input/config_file/inference/atom/mi355x_atom_gpt-oss-120b_bf16.json new file mode 100644 index 000000000..ad53aef9c --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi355x_atom_gpt-oss-120b_bf16.json @@ -0,0 +1,87 @@ +{ + "_comment": "Single-node ATOM sample (schema_version 1). Set container.image and container.name. Verify serve_args against your ATOM revision.", + "schema_version": 1, + "framework": "atom", + "gpu_arch": "mi355x", + "enforce_thresholds": false, + "threshold_json": "mi355x_atom_gpt-oss-120b_bf16_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/home/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "openai/gpt-oss-120b", + "remote": 0, + "precision": "bf16" + }, + "container": { + "lifetime": "per_run", + "name": "", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/home/models:/home/models" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "enforce-eager": true, + "gpu-memory-utilization": "0.92", + "block-size": 64, + "no-enable-prefix-caching": true + }, + "env": {} + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8000", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.8", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "99", + "num_prompts": "1000", + "max_model_length": "8192", + "client_poll_count": "50", + "client_poll_wait_time": "60", + "bench_max_failed_requests": "0" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "legacy_profile", + "isl": "7168", + "osl": "1024" + } + ], + "runs": [ + { + "combo": "legacy_profile", + "concurrency": 64 + } + ] + } +} diff --git a/cvs/input/config_file/inference/atom/mi355x_atom_gpt-oss-120b_bf16_threshold.json b/cvs/input/config_file/inference/atom/mi355x_atom_gpt-oss-120b_bf16_threshold.json new file mode 100644 index 000000000..1b21327dd --- /dev/null +++ b/cvs/input/config_file/inference/atom/mi355x_atom_gpt-oss-120b_bf16_threshold.json @@ -0,0 +1,105 @@ +{ + "_comment": "client.* thresholds for ISL=7168,OSL=1024,TP=8,CONC=64. Placeholder values until MI355x is calibrated; enforce_thresholds=false in config.json.", + "ISL=7168,OSL=1024,TP=8,CONC=64": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 4200 + }, + "client.per_gpu_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_tput_per_gpu": { + "kind": "min_tok_s", + "value": 525 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 500 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 15 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 1000000 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 1000000000 + } + } +} diff --git a/cvs/input/config_file/inference/inferencemax/mi300x_inferencemax_gpt_oss_120b_single.json b/cvs/input/config_file/inference/inferencemax/mi300x_inferencemax_gpt_oss_120b_single.json deleted file mode 100644 index 18a4d5a8b..000000000 --- a/cvs/input/config_file/inference/inferencemax/mi300x_inferencemax_gpt_oss_120b_single.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "config": { - "container_image": "rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1", - "container_name": "inference_max_rocm", - "_example_nnodes": "4", - "nnodes": "4", - "inferencemax_repo": "https://github.com/SemiAnalysisAI/InferenceX.git", - "benchmark_script_repo": "https://github.com/kimbochen/bench_serving.git", - "hf_token_file": "/home/{user-id}/.hf_token", - "shm_size": "128G", - "log_dir": "/home/{user-id}/LOGS", - "container_config": { - "device_list": [ - "/dev/dri", - "/dev/kfd" - ], - "volume_dict": { - "/home/{user-id}": "/home/{user-id}" - }, - "env_dict": {} - } - }, - "benchmark_params": { - "gpt-oss-120b": { - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8000", - "_example_dataset_name": "sharegpt|hf|random|sonnet|burstgpt", - "dataset_name": "random", - "max_concurrency": "64", - "model": "openai/gpt-oss-120b", - "num_prompts": "1000", - "input_sequence_length": "8192", - "output_sequence_length": "1024", - "burstiness": "1.0", - "seed": "0", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "random_prefix_len": "0", - "tensor_parallelism": "8", - "_example_tokenizer_mode": "auto|slow|mistral|custom", - "tokenizer_mode": "auto", - "percentiles_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "server_script": "gptoss_fp4_mi300x.sh", - "bench_serv_script": "benchmark_serving.py", - "result_dict": { - "output_throughput_per_sec": "4200", - "mean_ttft_ms": "500", - "mean_tpot_ms": "15" - } - } - } -} \ No newline at end of file diff --git a/cvs/input/config_file/inference/inferencemax/mi355x_inferencemax_gpt_oss_120b_single.json b/cvs/input/config_file/inference/inferencemax/mi355x_inferencemax_gpt_oss_120b_single.json deleted file mode 100644 index 516efca2b..000000000 --- a/cvs/input/config_file/inference/inferencemax/mi355x_inferencemax_gpt_oss_120b_single.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "config": { - "container_image": "rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1", - "container_name": "inference_max_rocm", - "_example_nnodes": "4", - "nnodes": "4", - "inferencemax_repo": "https://github.com/SemiAnalysisAI/InferenceX.git", - "benchmark_script_repo": "https://github.com/kimbochen/bench_serving.git", - "hf_token_file": "/home/{user-id}/.hf_token", - "shm_size": "128G", - "log_dir": "/home/{user-id}/LOGS", - "container_config": { - "device_list": [ - "/dev/dri", - "/dev/kfd" - ], - "volume_dict": { - "/home/{user-id}": "/home/{user-id}" - }, - "env_dict": {} - } - }, - "benchmark_params": { - "gpt-oss-120b": { - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8000", - "_example_dataset_name": "sharegpt|hf|random|sonnet|burstgpt", - "dataset_name": "random", - "max_concurrency": "64", - "model": "openai/gpt-oss-120b", - "num_prompts": "1000", - "input_sequence_length": "8192", - "output_sequence_length": "1024", - "burstiness": "1.0", - "seed": "0", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "random_prefix_len": "0", - "tensor_parallelism": "8", - "_example_tokenizer_mode": "auto|slow|mistral|custom", - "tokenizer_mode": "auto", - "percentiles_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "server_script": "gptoss_fp4_mi355x.sh", - "bench_serv_script": "benchmark_serving.py", - "result_dict": { - "output_throughput_per_sec": "4200", - "mean_ttft_ms": "500", - "mean_tpot_ms": "15" - } - } - } -} \ No newline at end of file diff --git a/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_disaggregated.json b/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_disaggregated.json new file mode 100644 index 000000000..5ebbbac12 --- /dev/null +++ b/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_disaggregated.json @@ -0,0 +1,135 @@ +{ + "config": + { + "container_image": "rocm/sgl-dev:v0.5.12.post1-rocm720-mi30x-20260603", + "_container_image": "rocm/sgl-dev:v0.5.12.post1-rocm720-mi30x-20260601", + "container_name": "sglang_container", + "_example_nnodes": "4", + "nnodes": "2", + "hf_token_file": "/home/{user-id}/.hf_token", + "shm_size": "128G", + "_log_dir_comments": "Provide some common file system that is accessible from any node", + "log_dir": "/home/{user-id}/LOGS/sglang", + "log_level": "info", + "nic_type": "thor2", + "_example_nccl_ib_hca_list": "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7", + "nccl_ib_hca_list": "", + "_example_nccl_ib_hca": "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7", + "nccl_ib_hca": "", + "hca_id_prefix": "", + "mount_vol": "/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so", + "_example_nccl_socket_ifname": "eno0", + "nccl_socket_ifname": "", + "_example_gloo_socket_ifname": "eno0", + "gloo_socket_ifname": "", + "_example_gloo_tcp_ifname": "eno0", + "gloo_tcp_ifname": "", + "nccl_ib_gid_index": "3", + "nccl_debug": "ERROR", + "prefill_node_list": ["", ""], + "decode_node_list": ["", ""], + "proxy_router_node": "", + "benchmark_serv_node": "", + "prefill_serv_port": "30001", + "decode_serv_port": "30002", + "proxy_router_port": "8000", + "_prefill_coordinator_addr": "This is the master address for co-ordination among all prefill nodes", + "prefill_coordinator_addr": "", + "_decode_coordinator_addr": "This is the master address for co-ordination among all decode nodes", + "decode_coordinator_addr": "", + "prefill_coordinator_port": "40001", + "decode_coordinator_port": "40002", + "proxy_router_serv_port": "8000", + "container_config": + { + "device_list": [ "/dev/dri", "/dev/kfd", "/dev/infiniband/rdma_cm" ], + "volume_dict": + { + "/home/{user-id}": "/home/{user-id}", + "/mnt/dtni/models": "/root/models", + "/dev/infiniband": "/dev/infiniband", + "/usr/local/lib/libbnxt_re-rdmav34.so": "/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so.host", + "/lib/libibverbs.d": "/lib/libibverbs.d" + }, + "env_dict": + { + } + } + + }, + "active_benchmark": "deepseek-r1", + "benchmark_params": + { + "deepseek-r1": + { + "backend": "sglang", + "threshold_file": "cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_threshold.json", + "max_concurrency": "256", + "_comments_model": "If the model is local, specify the full path of the model", + "model": "/root/models/DeepSeek-R1-0528", + "prefill_policy": "cache_aware", + "decode_policy": "cache_aware", + "tensor_parallelism": "8", + "pipeline_parallelism": "2", + "memory_fraction": "0.7", + "tokenizer_mode": "auto", + "inference_poll_iterations": "16", + "context_length": "205000", + "add_export_env": ["SGLANG_USE_AITER=1", "AMDGCN_USE_BUFFER_OPS=1", "ROCM_QUICK_REDUCE_QUANTIZATION=INT8", "GPU_ARCHS=gfx942"], + "add_flags": ["--attention-backend aiter"], + "inference_tests": + { + "bench_serv_random": + { + "backend": "sglang", + "data_set_name": "random", + "num_prompts": "25", + "random_range_ratio": "0.5", + "model_num_params": "671000000000", + "peak_gpu_tflops": "2615" + }, + "bench_serv_generated_shared_prefix": + { + "backend": "sglang", + "gsp_num_groups": "1", + "gsp_prompts_per_group": "16", + "gsp_system_prompt_len": "0", + "gsp_question_len": "1024", + "gsp_output_len": "1024" + }, + "long_ctx_niah": { + "num_prompts": "6", + "seed": "42", + "request_timeout_sec": "7200", + "exec_timeout_sec": "21600", + "tolerance_frac": "0.05" + }, + "lm_eval_hellaswag": + { + "backend": "sglang", + "lm_eval_model": "local-completions", + "tasks": "hellaswag", + "num_fewshot": "0", + "batch_size": "auto", + "limit": "100", + "num_concurrent": "1", + "exec_timeout_sec": "7200", + "extra_model_args": "tokenizer_backend=huggingface" + }, + "lm_eval_gsm8k": + { + "backend": "sglang", + "lm_eval_model": "local-completions", + "tasks": "gsm8k", + "num_fewshot": "5", + "batch_size": "auto", + "limit": "100", + "num_concurrent": "4", + "exec_timeout_sec": "7200", + "extra_model_args": "tokenizer_backend=huggingface" + } + + } + } + } +} \ No newline at end of file diff --git a/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_distributed.json b/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_distributed.json new file mode 100644 index 000000000..ef284e76e --- /dev/null +++ b/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_distributed.json @@ -0,0 +1,118 @@ +{ + "config": + { + "container_image": "rocm/sgl-dev:v0.5.12.post1-rocm720-mi30x-20260603", + "_container_image": "rocm/sgl-dev:v0.5.12.post1-rocm720-mi30x-20260601", + "container_name": "sglang_container", + "nnodes": "2", + "hf_token_file": "/home/{user-id}/.hf_token", + "shm_size": "128G", + "_log_dir_comments": "Provide some common file system that is accessible from any node", + "log_dir": "/home/{user-id}/LOGS/sglang", + "log_level": "info", + "nic_type": "thor2", + "_example_nccl_ib_hca": "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7", + "nccl_ib_hca": "", + "hca_id_prefix": "", + "mount_vol": "/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so", + "_example_nccl_socket_ifname": "eno0", + "nccl_socket_ifname": "", + "_example_gloo_socket_ifname": "eno0", + "gloo_socket_ifname": "", + "nccl_ib_gid_index": "3", + "nccl_debug": "ERROR", + "_example_server_node_list": "Two MI30X nodes; TP=8 per node, PP=2 across nodes", + "server_node_list": ["", ""], + "_dist_init_comments": "dist-init binds to rank-0 (first server_node_list entry) on dist_init_port unless dist_init_addr is set", + "dist_init_port": "40001", + "benchmark_serv_node": "", + "proxy_router_serv_port": "8000", + "container_config": + { + "device_list": [ "/dev/dri", "/dev/kfd", "/dev/infiniband/rdma_cm" ], + "volume_dict": + { + "/home/{user-id}": "/home/{user-id}", + "/mnt/dtni/models": "/root/models", + "/dev/infiniband": "/dev/infiniband", + "/usr/local/lib/libbnxt_re-rdmav34.so": "/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so.host", + "/lib/libibverbs.d": "/lib/libibverbs.d" + }, + "env_dict": + { + } + } + }, + "active_benchmark": "deepseek-r1", + "benchmark_params": + { + "deepseek-r1": + { + "backend": "sglang", + "threshold_file": "cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_threshold.json", + "max_concurrency": "256", + "_comments_model": "If the model is local, specify the full path of the model", + "model": "/root/models/DeepSeek-R1-0528", + "tensor_parallelism": "8", + "pipeline_parallelism": "2", + "memory_fraction": "0.7", + "tokenizer_mode": "auto", + "inference_poll_iterations": "16", + "context_length": "205000", + "add_export_env": ["SGLANG_USE_AITER=1", "AMDGCN_USE_BUFFER_OPS=1", "ROCM_QUICK_REDUCE_QUANTIZATION=INT8", "GPU_ARCHS=gfx942"], + "add_flags": ["--attention-backend aiter"], + "inference_tests": + { + "bench_serv_random": + { + "backend": "sglang", + "data_set_name": "random", + "num_prompts": "25", + "random_range_ratio": "0.5", + "model_num_params": "671000000000", + "peak_gpu_tflops": "2615" + }, + "bench_serv_generated_shared_prefix": + { + "backend": "sglang", + "gsp_num_groups": "1", + "gsp_prompts_per_group": "16", + "gsp_system_prompt_len": "0", + "gsp_question_len": "1024", + "gsp_output_len": "1024" + }, + "long_ctx_niah": { + "num_prompts": "6", + "seed": "42", + "request_timeout_sec": "7200", + "exec_timeout_sec": "21600", + "tolerance_frac": "0.05" + }, + "lm_eval_hellaswag": + { + "backend": "sglang", + "lm_eval_model": "local-completions", + "tasks": "hellaswag", + "num_fewshot": "0", + "batch_size": "auto", + "limit": "100", + "num_concurrent": "1", + "exec_timeout_sec": "7200", + "extra_model_args": "tokenizer_backend=huggingface" + }, + "lm_eval_gsm8k": + { + "backend": "sglang", + "lm_eval_model": "local-completions", + "tasks": "gsm8k", + "num_fewshot": "5", + "batch_size": "auto", + "limit": "100", + "num_concurrent": "4", + "exec_timeout_sec": "7200", + "extra_model_args": "tokenizer_backend=huggingface" + } + } + } + } +} diff --git a/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_single.json b/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_single.json new file mode 100644 index 000000000..ad429b334 --- /dev/null +++ b/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_single.json @@ -0,0 +1,103 @@ +{ + "config": + { + "container_image": "rocm/sgl-dev:v0.5.12.post1-rocm720-mi30x-20260603", + "_container_image": "rocm/sgl-dev:v0.5.12.post1-rocm720-mi30x-20260601", + "container_name": "sglang_container", + "nnodes": "1", + "hf_token_file": "/home/{user-id}/.hf_token", + "shm_size": "128G", + "_log_dir_comments": "Provide some common file system that is accessible from any node", + "log_dir": "/home/{user-id}/LOGS/sglang", + "log_level": "info", + "nccl_debug": "ERROR", + "benchmark_serv_node": "", + "proxy_router_serv_port": "8000", + "container_config": + { + "device_list": [ "/dev/dri", "/dev/kfd", "/dev/infiniband/rdma_cm" ], + "volume_dict": + { + "/home/{user-id}": "/home/{user-id}", + "/mnt/dtni/models": "/root/models", + "/dev/infiniband": "/dev/infiniband", + "/usr/local/lib/libbnxt_re-rdmav34.so": "/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so.host", + "/lib/libibverbs.d": "/lib/libibverbs.d" + }, + "env_dict": + { + } + } + }, + "active_benchmark": "deepseek-r1", + "benchmark_params": + { + "deepseek-r1": + { + "backend": "sglang", + "threshold_file": "cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_threshold.json", + "max_concurrency": "256", + "_comments_model": "If the model is local, specify the full path of the model", + "model": "/root/models/DeepSeek-R1-0528", + "tensor_parallelism": "8", + "pipeline_parallelism": "1", + "memory_fraction": "0.7", + "tokenizer_mode": "auto", + "inference_poll_iterations": "16", + "add_export_env": ["SGLANG_USE_AITER=1", "AMDGCN_USE_BUFFER_OPS=1", "ROCM_QUICK_REDUCE_QUANTIZATION=INT8", "GPU_ARCHS=gfx942"], + "add_flags": ["--attention-backend aiter"], + "inference_tests": + { + "bench_serv_random": + { + "backend": "sglang", + "data_set_name": "random", + "num_prompts": "25", + "random_range_ratio": "0.5", + "model_num_params": "671000000000", + "peak_gpu_tflops": "2615" + }, + "bench_serv_generated_shared_prefix": + { + "backend": "sglang", + "gsp_num_groups": "1", + "gsp_prompts_per_group": "16", + "gsp_system_prompt_len": "0", + "gsp_question_len": "1024", + "gsp_output_len": "1024" + }, + "long_ctx_niah": { + "num_prompts": "6", + "seed": "42", + "request_timeout_sec": "7200", + "exec_timeout_sec": "21600", + "tolerance_frac": "0.05" + }, + "lm_eval_hellaswag": + { + "backend": "sglang", + "lm_eval_model": "local-completions", + "tasks": "hellaswag", + "num_fewshot": "0", + "batch_size": "auto", + "limit": "100", + "num_concurrent": "1", + "exec_timeout_sec": "7200", + "extra_model_args": "tokenizer_backend=huggingface" + }, + "lm_eval_gsm8k": + { + "backend": "sglang", + "lm_eval_model": "local-completions", + "tasks": "gsm8k", + "num_fewshot": "5", + "batch_size": "auto", + "limit": "100", + "num_concurrent": "4", + "exec_timeout_sec": "7200", + "extra_model_args": "tokenizer_backend=huggingface" + } + } + } + } +} diff --git a/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_threshold.json b/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_threshold.json new file mode 100644 index 000000000..25b32b251 --- /dev/null +++ b/cvs/input/config_file/inference/sglang/mi30x_sglang_deepseek_r1_0528_threshold.json @@ -0,0 +1,79 @@ +{ + "_comment": "DeepSeek-R1-0528 thresholds for MI30X SGLang disaggregated. ISL=1024 OSL=1024 concurrency sweep: 4,8,16,32,64,128,256. Other ISL/OSL at CONC=64.", + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=4": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 75 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.007 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=8": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 75 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.007 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=16": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 115 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.01 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=32": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 195 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.01 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=64": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 205 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.01 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=128": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 195 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.01 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=256": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 205 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.02 } + }, + "ISL=8192,OSL=1024,TP=8,PP=1,CONC=64": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 125 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.05 } + }, + "ACC_ISL=131072,OSL=1024": { + "pass_rate": { "kind": "min", "value": 0.95 } + }, + "ACC_ISL=261120,OSL=1024": { + "pass_rate": { "kind": "min", "value": 0.90 } + }, + "BENCH=lm_eval_hellaswag": { + "acc_norm,none": { "kind": "min", "value": 0.23 } + }, + "BENCH=lm_eval_gsm8k": { + "exact_match,flexible-extract": { "kind": "min", "value": 0.95 } + } +} \ No newline at end of file diff --git a/cvs/input/config_file/inference/sglang/mi30x_sglang_distributed.json b/cvs/input/config_file/inference/sglang/mi30x_sglang_distributed.json index d2f8e108a..0c102c5a6 100644 --- a/cvs/input/config_file/inference/sglang/mi30x_sglang_distributed.json +++ b/cvs/input/config_file/inference/sglang/mi30x_sglang_distributed.json @@ -185,4 +185,4 @@ } -} +} \ No newline at end of file diff --git a/cvs/input/config_file/inference/sglang/mi30x_sglang_glm_52_fp8_threshold.json b/cvs/input/config_file/inference/sglang/mi30x_sglang_glm_52_fp8_threshold.json new file mode 100644 index 000000000..b5cb9276d --- /dev/null +++ b/cvs/input/config_file/inference/sglang/mi30x_sglang_glm_52_fp8_threshold.json @@ -0,0 +1,79 @@ +{ + "_comment": "GLM-5.2-FP8 thresholds for MI30X SGLang disaggregated. ISL=1024 OSL=1024 concurrency sweep: 4,8,16,32,64,128,256. Other ISL/OSL at CONC=64.", + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=4": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=8": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=32": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=64": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=128": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=256": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,CONC=64": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ACC_ISL=131072,OSL=1024": { + "pass_rate": { "kind": "min", "value": 0.95 } + }, + "ACC_ISL=261120,OSL=1024": { + "pass_rate": { "kind": "min", "value": 0.90 } + }, + "BENCH=lm_eval_hellaswag": { + "acc_norm,none": { "kind": "min", "value": 0.23 } + }, + "BENCH=lm_eval_gsm8k": { + "exact_match,flexible-extract": { "kind": "min", "value": 0.95 } + } + } \ No newline at end of file diff --git a/cvs/input/config_file/inference/sglang/mi30x_sglang_gpt_oss_120b_threshold.json b/cvs/input/config_file/inference/sglang/mi30x_sglang_gpt_oss_120b_threshold.json new file mode 100644 index 000000000..463e57744 --- /dev/null +++ b/cvs/input/config_file/inference/sglang/mi30x_sglang_gpt_oss_120b_threshold.json @@ -0,0 +1,79 @@ +{ + "_comment": "GPT-OSS-120B thresholds for MI30X SGLang disaggregated. ISL=1024 OSL=1024 concurrency sweep: 4,8,16,32,64,128,256. Other ISL/OSL at CONC=64.", + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=4": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 900 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=8": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 900 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=16": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 900 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=32": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 900 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=64": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 900 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=128": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 900 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=256": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 900 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=8192,OSL=1024,TP=8,PP=1,CONC=64": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 900 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ACC_ISL=131072,OSL=1024": { + "pass_rate": { "kind": "min", "value": 0.95 } + }, + "ACC_ISL=261120,OSL=1024": { + "pass_rate": { "kind": "min", "value": 0.90 } + }, + "BENCH=lm_eval_hellaswag": { + "acc_norm,none": { "kind": "min", "value": 0.23 } + }, + "BENCH=lm_eval_gsm8k": { + "exact_match,flexible-extract": { "kind": "min", "value": 0.95 } + } +} \ No newline at end of file diff --git a/cvs/input/config_file/inference/sglang/mi30x_sglang_kimi_k26_threshold.json b/cvs/input/config_file/inference/sglang/mi30x_sglang_kimi_k26_threshold.json new file mode 100644 index 000000000..bd87317f6 --- /dev/null +++ b/cvs/input/config_file/inference/sglang/mi30x_sglang_kimi_k26_threshold.json @@ -0,0 +1,79 @@ +{ + "_comment": "Kimi-K2.6 thresholds for MI30X SGLang disaggregated. ISL=1024 OSL=1024 concurrency sweep: 4,8,16,32,64,128,256. Other ISL/OSL at CONC=64.", + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=4": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=8": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=32": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=64": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=128": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=256": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ISL=8192,OSL=1024,TP=8,PP=2,CONC=64": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 340 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 250 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.25 } + }, + "ACC_ISL=131072,OSL=1024": { + "pass_rate": { "kind": "min", "value": 0.95 } + }, + "ACC_ISL=261120,OSL=1024": { + "pass_rate": { "kind": "min", "value": 0.90 } + }, + "BENCH=lm_eval_hellaswag": { + "acc_norm,none": { "kind": "min", "value": 0.23 } + }, + "BENCH=lm_eval_gsm8k": { + "exact_match,flexible-extract": { "kind": "min", "value": 0.95 } + } + } \ No newline at end of file diff --git a/cvs/input/config_file/inference/sglang/mi30x_sglang_llama_70b_threshold.json b/cvs/input/config_file/inference/sglang/mi30x_sglang_llama_70b_threshold.json new file mode 100644 index 000000000..f018d97e5 --- /dev/null +++ b/cvs/input/config_file/inference/sglang/mi30x_sglang_llama_70b_threshold.json @@ -0,0 +1,79 @@ +{ + "_comment": "Llama 3.1 70B thresholds for MI30X SGLang disaggregated. ISL=1024 OSL=1024 concurrency sweep: 4,8,16,32,64,128,256. Other ISL/OSL at CONC=64.", + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=4": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 150 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.004 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=8": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 245 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.004 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=16": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 460 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.009 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=32": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 800 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.01 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=64": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 900 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.02 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=128": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 900 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.02 } + }, + "ISL=1024,OSL=1024,TP=8,PP=1,CONC=256": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 900 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.02 } + }, + "ISL=8192,OSL=1024,TP=8,PP=1,CONC=64": { + "output_throughput_per_sec": { "kind": "min_tok_s", "value": 900 }, + "mean_ttft_ms": { "kind": "max_ms", "value": 60000 }, + "mean_tpot_ms": { "kind": "max_ms", "value": 150 }, + "mean_e2e_latency_ms": { "kind": "max_ms", "value": 120000 }, + "goodput": { "kind": "min", "value": 0.99 }, + "mfu": { "kind": "min", "value": 0.02 } + }, + "ACC_ISL=131072,OSL=1024": { + "pass_rate": { "kind": "min", "value": 0.95 } + }, + "ACC_ISL=261120,OSL=1024": { + "pass_rate": { "kind": "min", "value": 0.90 } + }, + "BENCH=lm_eval_hellaswag": { + "acc_norm,none": { "kind": "min", "value": 0.23 } + }, + "BENCH=lm_eval_gsm8k": { + "exact_match,flexible-extract": { "kind": "min", "value": 0.95 } + } +} \ No newline at end of file diff --git a/cvs/input/config_file/inference/vllm/mi300x_vllm_llama31-70b_fp8_distributed.json b/cvs/input/config_file/inference/vllm/mi300x_vllm_llama31-70b_fp8_distributed.json new file mode 100644 index 000000000..97fc35911 --- /dev/null +++ b/cvs/input/config_file/inference/vllm/mi300x_vllm_llama31-70b_fp8_distributed.json @@ -0,0 +1,87 @@ +{ + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "", + "paths": { + "shared_fs": "/mnt/dtni/{user-id}", + "models_dir": "{shared_fs}/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.cache/huggingface/token" + }, + "model": { + "id": "amd/Llama-3.1-70B-Instruct-FP8-KV", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "w2_llama31_70b_fp8kv_dist_rocm", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/mnt/dtni:/mnt/dtni", + "{paths.models_dir}:/models" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "kv-cache-dtype": "fp8", + "enforce-eager": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.8", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "", + "master_port": "29501", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "3200", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w2_isl=1000_osl=1000", + "isl": "1000", + "osl": "1000", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { "combo": "w2_isl=1000_osl=1000", "concurrency": 16 } + ] + } +} diff --git a/cvs/input/config_file/inference/vllm/mi300x_vllm_llama31-70b_fp8_single.json b/cvs/input/config_file/inference/vllm/mi300x_vllm_llama31-70b_fp8_single.json new file mode 100644 index 000000000..5265dce98 --- /dev/null +++ b/cvs/input/config_file/inference/vllm/mi300x_vllm_llama31-70b_fp8_single.json @@ -0,0 +1,83 @@ +{ + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "", + "paths": { + "shared_fs": "/mnt/dtni/{user-id}", + "models_dir": "{shared_fs}/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.cache/huggingface/token" + }, + "model": { + "id": "amd/Llama-3.1-70B-Instruct-FP8-KV", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "w1_llama31_70b_fp8kv_perf_inference_rocm", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/mnt/dtni:/mnt/dtni", + "{paths.models_dir}:/models" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "kv-cache-dtype": "fp8", + "enforce-eager": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.8", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "3200", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w1_isl=1000_osl=1000", + "isl": "1000", + "osl": "1000", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { "combo": "w1_isl=1000_osl=1000", "concurrency": 16 } + ] + } +} diff --git a/cvs/input/config_file/inference/vllm/mi355x_vllm_single.json b/cvs/input/config_file/inference/vllm/mi355x_vllm_single.json deleted file mode 100644 index 801acb6bf..000000000 --- a/cvs/input/config_file/inference/vllm/mi355x_vllm_single.json +++ /dev/null @@ -1,371 +0,0 @@ -{ - "config": { - "container_image": "rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1", - "container_name": "vllm_inference_rocm", - "nnodes": "1", - "benchmark_server_script_path": "/home/{user-id}/benchmark_server_scripts/", - "benchmark_script_repo": "https://github.com/kimbochen/bench_serving.git", - "hf_token_file": "/home/{user-id}/.hf_token", - "shm_size": "16G", - "log_dir": "/home/{user-id}/LOGS", - "data_cache_dir": "/it-share/models/", - "container_config": { - "device_list": [ - "/dev/dri", - "/dev/kfd", - "/dev/mem" - ], - "volume_dict": { - "/home/{user-id}": "/home/{user-id}", - "/it-share/models/": "/models" - }, - "env_dict": { - "HF_HUB_CACHE": "/models/huggingface-cache" - } - } - }, - "benchmark_params": { - "gpt-oss-120b": { - "container_image": "rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1", - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8888", - "_example_dataset_name": "sharegpt|hf|random|sonnet|burstgpt", - "dataset_name": "random", - "concurrency_levels": [ - 16, - 32, - 64 - ], - "model": "openai/gpt-oss-120b", - "num_prompts": "3200", - "sequence_combinations": [ - { - "isl": "1024", - "osl": "1024", - "name": "balanced" - }, - { - "isl": "1024", - "osl": "8192", - "name": "long_generation" - }, - { - "isl": "8192", - "osl": "1024", - "name": "long_context" - } - ], - "burstiness": "1.0", - "seed": "0", - "request_rate": "inf", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "random_prefix_len": "0", - "tensor_parallelism": "1", - "_example_tokenizer_mode": "auto|slow|mistral|custom", - "tokenizer_mode": "auto", - "percentile_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "server_script": "gpt-oss-120b_fp4_mi355x_vllm_docker.sh", - "bench_serv_script": "benchmark_serving.py", - "result_dict": { - "ISL=1024,OSL=1024,TP=1,CONC=16": { - "total_throughput_per_sec": "4651", - "mean_ttft_ms": "70", - "mean_tpot_ms": "8" - }, - "ISL=1024,OSL=1024,TP=1,CONC=32": { - "total_throughput_per_sec": "7043", - "mean_ttft_ms": "180", - "mean_tpot_ms": "9" - }, - "ISL=1024,OSL=1024,TP=1,CONC=64": { - "total_throughput_per_sec": "10677", - "mean_ttft_ms": "76", - "mean_tpot_ms": "13" - }, - "ISL=1024,OSL=8192,TP=1,CONC=16": { - "total_throughput_per_sec": "2735", - "mean_ttft_ms": "57", - "mean_tpot_ms": "7" - }, - "ISL=1024,OSL=8192,TP=1,CONC=32": { - "total_throughput_per_sec": "4038", - "mean_ttft_ms": "67", - "mean_tpot_ms": "10" - }, - "ISL=1024,OSL=8192,TP=1,CONC=64": { - "total_throughput_per_sec": "6140", - "mean_ttft_ms": "93", - "mean_tpot_ms": "13" - }, - "ISL=8192,OSL=1024,TP=1,CONC=16": { - "total_throughput_per_sec": "16509", - "mean_ttft_ms": "335", - "mean_tpot_ms": "24" - }, - "ISL=8192,OSL=1024,TP=1,CONC=32": { - "total_throughput_per_sec": "22072", - "mean_ttft_ms": "320", - "mean_tpot_ms": "19" - }, - "ISL=8192,OSL=1024,TP=1,CONC=64": { - "total_throughput_per_sec": "28863", - "mean_ttft_ms": "280", - "mean_tpot_ms": "22" - } - } - }, - "qwen3-235b": { - "container_image": "amdsiloai/vllm:2025111-0.11.1rc2-qwen3", - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8888", - "dataset_name": "random", - "concurrency_levels": [ - 16, - 32, - 64 - ], - "model": "Qwen/Qwen3-235B-A22B-Instruct-2507", - "num_prompts": "3200", - "sequence_combinations": [ - { - "isl": "1024", - "osl": "1024", - "name": "balanced" - }, - { - "isl": "1024", - "osl": "8192", - "name": "long_generation" - }, - { - "isl": "8192", - "osl": "1024", - "name": "long_context" - } - ], - "burstiness": "1.0", - "seed": "0", - "request_rate": "inf", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "random_prefix_len": "0", - "tensor_parallelism": "8", - "tokenizer_mode": "auto", - "percentile_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "server_script": "qwen3-235b-bf16_mi355x_vllm_docker.sh", - "bench_serv_script": "benchmark_serving.py", - "result_dict": { - "ISL=1024,OSL=1024,TP=8,CONC=16": { - "total_throughput_per_sec": "2000", - "mean_ttft_ms": "850", - "mean_tpot_ms": "18" - }, - "ISL=1024,OSL=1024,TP=8,CONC=32": { - "total_throughput_per_sec": "3435", - "mean_ttft_ms": "80", - "mean_tpot_ms": "10" - }, - "ISL=1024,OSL=1024,TP=8,CONC=64": { - "total_throughput_per_sec": "5840", - "mean_ttft_ms": "260", - "mean_tpot_ms": "10" - }, - "ISL=1024,OSL=8192,TP=8,CONC=16": { - "total_throughput_per_sec": "1119", - "mean_ttft_ms": "415", - "mean_tpot_ms": "25" - }, - "ISL=1024,OSL=8192,TP=8,CONC=32": { - "total_throughput_per_sec": "1876", - "mean_ttft_ms": "70", - "mean_tpot_ms": "10" - }, - "ISL=1024,OSL=8192,TP=8,CONC=64": { - "total_throughput_per_sec": "3139", - "mean_ttft_ms": "310", - "mean_tpot_ms": "14" - }, - "ISL=8192,OSL=1024,TP=8,CONC=16": { - "total_throughput_per_sec": "7476", - "mean_ttft_ms": "300", - "mean_tpot_ms": "21" - }, - "ISL=8192,OSL=1024,TP=8,CONC=32": { - "total_throughput_per_sec": "11312", - "mean_ttft_ms": "355", - "mean_tpot_ms": "27" - }, - "ISL=8192,OSL=1024,TP=8,CONC=64": { - "total_throughput_per_sec": "16082", - "mean_ttft_ms": "450", - "mean_tpot_ms": "39" - } - } - }, - "qwen3-80b": { - "container_image": "rocm/vllm-dev:nightly", - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8888", - "dataset_name": "random", - "concurrency_levels": [ - 16, - 32, - 64 - ], - "model": "Qwen/Qwen3-Next-80B-A3B-Instruct", - "num_prompts": "3200", - "sequence_combinations": [ - { - "isl": "1024", - "osl": "1024", - "name": "balanced" - }, - { - "isl": "1024", - "osl": "8192", - "name": "long_generation" - }, - { - "isl": "8192", - "osl": "1024", - "name": "long_context" - } - ], - "burstiness": "1.0", - "seed": "0", - "request_rate": "inf", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "random_prefix_len": "0", - "tensor_parallelism": "1", - "tokenizer_mode": "auto", - "percentile_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "server_script": "qwen3-80b-bf16_mi355x_vllm_docker.sh", - "bench_serv_script": "benchmark_serving.py", - "result_dict": { - "ISL=1024,OSL=1024,TP=1,CONC=16": { - "total_throughput_per_sec": "2003", - "mean_ttft_ms": "69", - "mean_tpot_ms": "9" - }, - "ISL=1024,OSL=1024,TP=1,CONC=32": { - "total_throughput_per_sec": "3155", - "mean_ttft_ms": "69", - "mean_tpot_ms": "12" - }, - "ISL=1024,OSL=1024,TP=1,CONC=64": { - "total_throughput_per_sec": "4570", - "mean_ttft_ms": "375", - "mean_tpot_ms": "23" - }, - "ISL=1024,OSL=8192,TP=1,CONC=16": { - "total_throughput_per_sec": "1200", - "mean_ttft_ms": "84", - "mean_tpot_ms": "12" - }, - "ISL=1024,OSL=8192,TP=1,CONC=32": { - "total_throughput_per_sec": "1800", - "mean_ttft_ms": "200", - "mean_tpot_ms": "12" - }, - "ISL=1024,OSL=8192,TP=1,CONC=64": { - "total_throughput_per_sec": "2600", - "mean_ttft_ms": "768", - "mean_tpot_ms": "21" - }, - "ISL=8192,OSL=1024,TP=1,CONC=16": { - "total_throughput_per_sec": "7500", - "mean_ttft_ms": "495", - "mean_tpot_ms": "16" - }, - "ISL=8192,OSL=1024,TP=1,CONC=32": { - "total_throughput_per_sec": "11300", - "mean_ttft_ms": "280", - "mean_tpot_ms": "17" - }, - "ISL=8192,OSL=1024,TP=1,CONC=64": { - "total_throughput_per_sec": "16000", - "mean_ttft_ms": "91", - "mean_tpot_ms": "18" - } - } - }, - "deepseek-v31": { - "container_image": "rocm/7.x-preview:rocm7.2_preview_ubuntu_22.04_vlm_0.10.1_instinct_20251029", - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8888", - "dataset_name": "random", - "concurrency_levels": [ - 16, - 32, - 64 - ], - "model": "deepseek-ai/DeepSeek-V3.1", - "num_prompts": "3200", - "sequence_combinations": [ - { - "isl": "1024", - "osl": "1024", - "name": "balanced" - }, - { - "isl": "1024", - "osl": "8192", - "name": "long_generation" - } - ], - "burstiness": "1.0", - "seed": "0", - "request_rate": "inf", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "random_prefix_len": "0", - "tensor_parallelism": "8", - "tokenizer_mode": "auto", - "percentile_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "server_script": "dsr1_fp8_mi355x_vllm_docker.sh", - "bench_serv_script": "benchmark_serving.py", - "result_dict": { - "ISL=1024,OSL=1024,TP=8,CONC=16": { - "total_throughput_per_sec": "1944", - "mean_ttft_ms": "84", - "mean_tpot_ms": "12" - }, - "ISL=1024,OSL=1024,TP=8,CONC=32": { - "total_throughput_per_sec": "2939", - "mean_ttft_ms": "302", - "mean_tpot_ms": "21" - }, - "ISL=1024,OSL=1024,TP=8,CONC=64": { - "total_throughput_per_sec": "4834", - "mean_ttft_ms": "250", - "mean_tpot_ms": "11" - }, - "ISL=1024,OSL=8192,TP=8,CONC=16": { - "total_throughput_per_sec": "1109", - "mean_ttft_ms": "253", - "mean_tpot_ms": "19" - }, - "ISL=1024,OSL=8192,TP=8,CONC=32": { - "total_throughput_per_sec": "1676", - "mean_ttft_ms": "305", - "mean_tpot_ms": "21" - }, - "ISL=1024,OSL=8192,TP=8,CONC=64": { - "total_throughput_per_sec": "2716", - "mean_ttft_ms": "280", - "mean_tpot_ms": "13" - } - } - } - } -} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/README.md b/cvs/input/config_file/inference/vllm_mi300x_workloads/README.md new file mode 100644 index 000000000..658d2e5a8 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/README.md @@ -0,0 +1,182 @@ +# vllm MI300X workload configs + +14 inference workloads for the `vllm` suite on MI300X, each shipped as a +`single` / `distributed` pair — 28 configs, 28 sibling thresholds. + +## Layout + +Flat sibling pairs, same convention as `inferencex_atom_single`: + +```text +mi300x_vllm_{model}_{precision}_{topology}_config.json +mi300x_vllm_{model}_{precision}_{topology}_threshold.json +``` + +`topology` is `single` or `distributed`. Each config points `threshold_json` at +its sibling filename, so **copy one variant at a time into its own directory** +on the lab machine — `substitute_config` globs the config's parent for +`*threshold.json` and raises on more than one match only when `threshold_json` +is absent, but keeping one pair per directory avoids the trap entirely. + +## Topology + +| | PP | nnodes | +|---|---|---| +| `single` | 1 | 1 | +| `distributed` | 2 | 2 | + +TP is **per model**, following the source workload list: TP=4 for +`deepseek-v4-flash`, `kimi-k26`, `kimi-k25` and `gpt-oss-20b`; TP=8 for +everything else. A TP=4 distributed variant still spans 2 nodes via PP=2, +using 4 GPUs per node. + +## Sweep + +Every config carries all three concurrency-16 shapes: + +| combo suffix | ISL | OSL | +|---|---|---| +| `1k1k` | 1024 | 1024 | +| `1k8k` | 1024 | 8192 | +| `8k1k` | 8192 | 1024 | + +Only `1k1k` is referenced by `sweep.runs`, so that is the single cell a run +executes. To run another shape, add it to `sweep.runs` **and** add the matching +cell key to the threshold file — the coverage check compares the two. + +Cell keys follow the loader's format: + +```text +single: ISL=1024,OSL=1024,TP=,CONC=16 +distributed: ISL=1024,OSL=1024,TP=,PP=2,CONC=16 +``` + +`` is the config's own `params.tensor_parallelism` (4 or 8). + +`random_range_ratio` is `0.0` so ISL/OSL are exact rather than jittered ±80%. + +`num_prompts` is **320**, not the `3200` schema default used by the configs in +`cvs/input/config_file/inference/vllm/`. That makes each cell a characterization +pass — enough to shake out topology, AITER and kv-cache settings on new +hardware, at roughly a tenth the wall-clock. Raise it to `3200` before quoting +numbers that need to line up with the shipped examples. + +## Thresholds + +`enforce_thresholds` is **false** on every config, so nothing gates and metrics +are only recorded. Each threshold file carries a placeholder for every metric +the suite can gate on — 32 per file. That full grid is a convenience for later +calibration, **not** a loader requirement: the vLLM loader checks cell coverage +only, and an absent metric spec means "don't gate this metric". A threshold file +may gate just the handful of metrics you care about. + +| Family | Count | Source of the list | +|---|---|---| +| `client.*` | 23 | the suite's gated-metric set | +| `gpu.*` | 5 | `cvs.lib.utils.gpu.GPU_METRICS` | +| `prom.*` | 4 | `vllm_server_metrics.PROM_METRICS` | + +**Every value is `0`**, meaning *not yet measured* — not a real bound. On a +`max`/`max_ms` kind, `0` is an impossible bound, so enabling enforcement before +calibrating fails loudly rather than passing silently. Replace them with +measured values from a calibration run before flipping `enforce_thresholds`. + +### Accuracy + +Accuracy is split across the two files, unlike the three families above: + +- **`config.json` → `accuracy.tasks`** selects *which* lm-eval tasks run. + Shipped empty, so no accuracy stage runs and the pytest node is auto-skipped. +- **`threshold.json` → `accuracy`** holds the gating values, keyed by task id + then by lm-eval metric key. Shipped as `{}`. + +Because the threshold keys are derived from the task ids you choose, they +cannot be pre-enumerated the way `client.*`/`gpu.*`/`prom.*` can — the two +blocks must be filled in together: + +```jsonc +// config.json +"accuracy": {"tasks": [{"id": "gsm8k", "task": "gsm8k", "num_fewshot": 5}]} + +// threshold.json +"accuracy": {"gsm8k": {"gsm8k.exact_match__strict-match": {"kind": "min", "value": 0}}} +``` + +The metric key is the lm-eval `results.json` key with commas replaced by `__`. +The `accuracy` block is exempt from the sweep-cell coverage check via +`NON_SWEEP_THRESHOLD_KEYS`, so it does not need a cell key. + +## Before running — fill in the `` fields + +Every environment-specific value is redacted. Per config: + +| Field | What to set | +|---|---| +| `model.id` | Local model path (e.g. `/models/GLM-5.1-FP8`) or an HF repo id | +| `container.image` | The vLLM/ROCm image tag under test | +| `container.runtime.args.volumes[1]` | Replace `` with the host models directory | +| `roles.server.ib_netdev` | *(distributed only)* socket interface name for `NCCL_SOCKET_IFNAME` / `GLOO_SOCKET_IFNAME` / `TP_SOCKET_IFNAME`. Must be **UP and hold a routable IPv4 reaching the other node** — check `ip -o -4 addr show`, not just `ip -o link show`. An interface that exists but is DOWN/addressless fails engine init with gloo `Unable to find address for: ` | +| `params.master_addr` | *(distributed only)* head node IP | + +`paths.models_dir` is `/models`, the in-container mount point — it is exported +as `HF_HUB_CACHE`. When `model.id` is an absolute path under `/models`, vLLM +loads straight from the mount and no download occurs. + +## Workload set + +| Config stem | Model | Notes | +|---|---|---| +| `llama33-70b_fp8` | Llama 3.3 70B FP8 | `kv-cache-dtype: fp8` | +| `glm-51_fp8` | GLM 5.1 FP8 | | +| `glm-52_fp8` | GLM 5.2 FP8 | | +| `deepseek-v4-pro_fp8` | DeepSeek V4 Pro FP8 | | +| `deepseek-v4-flash_fp8` | DeepSeek V4 Flash FP8 | | +| `kimi-k26_mxfp4` | Kimi K2.6 MXFP4 | | +| `kimi-k27-code_mxfp4` | Kimi K2.7 Code MXFP4 | | +| `kimi-k25_w4a8` | Kimi K2.5 W4A8 | | +| `qwen35-397b-a17b_bf16` | Qwen3.5 397B A17B BF16 | | +| `minimax-m3_bf16` | MiniMax M3 BF16 | | +| `mimo-v25-pro_fp8` | MiMo V2.5 Pro FP8 | | +| `mistral-large-3_bf16` | Mistral Large 3 BF16 | Mistral-native format: `tokenizer-mode`/`config-format`/`load-format` all `mistral` | +| `deepseek-r1-0528_fp8` | DeepSeek R1 0528 FP8 PTPC | | +| `gpt-oss-20b_fp8` | GPT-OSS 20B FP8 | | + +Models with a custom tokenizer or modelling code set `trust-remote-code: true`; +the suite mirrors that flag onto the bench client so it can load the same +tokenizer. + +## Running + +```bash +cd ~/cvs && source .cvs_venv/bin/activate + +VAR=mi300x_vllm_glm-51_fp8_single +DIR=~/input/config_file/inference/vllm_mi300x_workloads/$VAR +mkdir -p "$DIR" + +cvs copy-config inference/vllm_mi300x_workloads/${VAR}_config.json \ + --output "$DIR/${VAR}_config.json" +cvs copy-config inference/vllm_mi300x_workloads/${VAR}_threshold.json \ + --output "$DIR/${VAR}_threshold.json" +# then edit "$DIR/${VAR}_config.json" and fill in every + +TS=$(date +%Y%m%d_%H%M%S) +cvs run vllm \ + --cluster_file ~/input/cluster_file/.json \ + --config_file "$DIR/${VAR}_config.json" \ + --html=~/cvs_results/${TS}_${VAR}.html \ + --self-contained-html \ + --log-file=~/cvs_results/${TS}_${VAR}.log \ + -vvv -s +``` + +Do not pass pytest function names — let the suite's default tests run. + +## See also + +This file covers only what is specific to this workload set. For the vLLM suite +itself — threshold kinds, cell-key format, multinode prerequisites, accuracy +metric keys — see the suite reference and how-to: + +- `docs/reference/configuration-files/vllm.rst` +- `docs/how-to/run-vllm-benchmarks.rst` diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-r1-0528_fp8_distributed_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-r1-0528_fp8_distributed_config.json new file mode 100644 index 000000000..9b4666818 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-r1-0528_fp8_distributed_config.json @@ -0,0 +1,117 @@ +{ + "_comment": "vllm distributed workload: deepseek-r1-0528_fp8 on MI300X, TP=8, PP=2 across 2 nodes. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_deepseek-r1-0528_fp8_distributed_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_deepseek-r1-0528_fp8_distributed_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90", + "master_addr": "", + "master_port": "29501" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_deepseek-r1-0528_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_deepseek-r1-0528_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_deepseek-r1-0528_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_deepseek-r1-0528_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-r1-0528_fp8_distributed_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-r1-0528_fp8_distributed_threshold.json new file mode 100644 index 000000000..1171b7f1c --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-r1-0528_fp8_distributed_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for deepseek-r1-0528_fp8 (distributed, TP=8, PP=2, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-r1-0528_fp8_single_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-r1-0528_fp8_single_config.json new file mode 100644 index 000000000..bbba9baf1 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-r1-0528_fp8_single_config.json @@ -0,0 +1,113 @@ +{ + "_comment": "vllm single workload: deepseek-r1-0528_fp8 on MI300X, TP=8 single node. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_deepseek-r1-0528_fp8_single_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_deepseek-r1-0528_fp8_single_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_deepseek-r1-0528_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_deepseek-r1-0528_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_deepseek-r1-0528_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_deepseek-r1-0528_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-r1-0528_fp8_single_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-r1-0528_fp8_single_threshold.json new file mode 100644 index 000000000..4859734f5 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-r1-0528_fp8_single_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for deepseek-r1-0528_fp8 (single, TP=8, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-flash_fp8_distributed_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-flash_fp8_distributed_config.json new file mode 100644 index 000000000..79749427c --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-flash_fp8_distributed_config.json @@ -0,0 +1,120 @@ +{ + "_comment": "vllm distributed workload: deepseek-v4-flash_fp8 on MI300X, TP=4, PP=2 across 2 nodes. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_deepseek-v4-flash_fp8_distributed_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_deepseek-v4-flash_fp8_distributed_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true, + "kv-cache-dtype": "fp8" + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "VLLM_ROCM_USE_AITER": "1", + "GPU_ARCHS": "gfx942" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "4", + "pipeline_parallel_size": "2", + "nnodes": "2", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90", + "master_addr": "", + "master_port": "29501" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_deepseek-v4-flash_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_deepseek-v4-flash_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_deepseek-v4-flash_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_deepseek-v4-flash_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-flash_fp8_distributed_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-flash_fp8_distributed_threshold.json new file mode 100644 index 000000000..9550d2872 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-flash_fp8_distributed_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for deepseek-v4-flash_fp8 (distributed, TP=4, PP=2, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=4,PP=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-flash_fp8_single_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-flash_fp8_single_config.json new file mode 100644 index 000000000..7c0cb68a3 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-flash_fp8_single_config.json @@ -0,0 +1,116 @@ +{ + "_comment": "vllm single workload: deepseek-v4-flash_fp8 on MI300X, TP=4 single node. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_deepseek-v4-flash_fp8_single_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_deepseek-v4-flash_fp8_single_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true, + "kv-cache-dtype": "fp8" + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "VLLM_ROCM_USE_AITER": "1", + "GPU_ARCHS": "gfx942" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "4", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_deepseek-v4-flash_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_deepseek-v4-flash_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_deepseek-v4-flash_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_deepseek-v4-flash_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-flash_fp8_single_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-flash_fp8_single_threshold.json new file mode 100644 index 000000000..0436c9027 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-flash_fp8_single_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for deepseek-v4-flash_fp8 (single, TP=4, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=4,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-pro_fp8_distributed_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-pro_fp8_distributed_config.json new file mode 100644 index 000000000..5fe7e7097 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-pro_fp8_distributed_config.json @@ -0,0 +1,120 @@ +{ + "_comment": "vllm distributed workload: deepseek-v4-pro_fp8 on MI300X, TP=8, PP=2 across 2 nodes. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_deepseek-v4-pro_fp8_distributed_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_deepseek-v4-pro_fp8_distributed_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true, + "kv-cache-dtype": "fp8" + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "VLLM_ROCM_USE_AITER": "1", + "GPU_ARCHS": "gfx942" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90", + "master_addr": "", + "master_port": "29501" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_deepseek-v4-pro_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_deepseek-v4-pro_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_deepseek-v4-pro_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_deepseek-v4-pro_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-pro_fp8_distributed_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-pro_fp8_distributed_threshold.json new file mode 100644 index 000000000..dd655acef --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-pro_fp8_distributed_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for deepseek-v4-pro_fp8 (distributed, TP=8, PP=2, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-pro_fp8_single_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-pro_fp8_single_config.json new file mode 100644 index 000000000..7b5beb6bd --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-pro_fp8_single_config.json @@ -0,0 +1,116 @@ +{ + "_comment": "vllm single workload: deepseek-v4-pro_fp8 on MI300X, TP=8 single node. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_deepseek-v4-pro_fp8_single_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_deepseek-v4-pro_fp8_single_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true, + "kv-cache-dtype": "fp8" + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "VLLM_ROCM_USE_AITER": "1", + "GPU_ARCHS": "gfx942" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_deepseek-v4-pro_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_deepseek-v4-pro_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_deepseek-v4-pro_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_deepseek-v4-pro_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-pro_fp8_single_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-pro_fp8_single_threshold.json new file mode 100644 index 000000000..c6e8b7771 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_deepseek-v4-pro_fp8_single_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for deepseek-v4-pro_fp8 (single, TP=8, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-51_fp8_distributed_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-51_fp8_distributed_config.json new file mode 100644 index 000000000..53a13b48f --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-51_fp8_distributed_config.json @@ -0,0 +1,119 @@ +{ + "_comment": "vllm distributed workload: glm-51_fp8 on MI300X, TP=8, PP=2 across 2 nodes. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_glm-51_fp8_distributed_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_glm-51_fp8_distributed_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "VLLM_ROCM_USE_AITER": "1", + "GPU_ARCHS": "gfx942" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90", + "master_addr": "", + "master_port": "29501" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_glm-51_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_glm-51_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_glm-51_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_glm-51_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-51_fp8_distributed_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-51_fp8_distributed_threshold.json new file mode 100644 index 000000000..7a4290e5b --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-51_fp8_distributed_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for glm-51_fp8 (distributed, TP=8, PP=2, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-51_fp8_single_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-51_fp8_single_config.json new file mode 100644 index 000000000..eeb0cc355 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-51_fp8_single_config.json @@ -0,0 +1,115 @@ +{ + "_comment": "vllm single workload: glm-51_fp8 on MI300X, TP=8 single node. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_glm-51_fp8_single_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_glm-51_fp8_single_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "VLLM_ROCM_USE_AITER": "1", + "GPU_ARCHS": "gfx942" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_glm-51_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_glm-51_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_glm-51_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_glm-51_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-51_fp8_single_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-51_fp8_single_threshold.json new file mode 100644 index 000000000..ebbf797f2 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-51_fp8_single_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for glm-51_fp8 (single, TP=8, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-52_fp8_distributed_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-52_fp8_distributed_config.json new file mode 100644 index 000000000..61da04fd3 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-52_fp8_distributed_config.json @@ -0,0 +1,119 @@ +{ + "_comment": "vllm distributed workload: glm-52_fp8 on MI300X, TP=8, PP=2 across 2 nodes. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_glm-52_fp8_distributed_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_glm-52_fp8_distributed_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "VLLM_ROCM_USE_AITER": "1", + "GPU_ARCHS": "gfx942" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90", + "master_addr": "", + "master_port": "29501" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_glm-52_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_glm-52_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_glm-52_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_glm-52_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-52_fp8_distributed_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-52_fp8_distributed_threshold.json new file mode 100644 index 000000000..4362ce182 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-52_fp8_distributed_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for glm-52_fp8 (distributed, TP=8, PP=2, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-52_fp8_single_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-52_fp8_single_config.json new file mode 100644 index 000000000..3bed1cd5d --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-52_fp8_single_config.json @@ -0,0 +1,115 @@ +{ + "_comment": "vllm single workload: glm-52_fp8 on MI300X, TP=8 single node. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_glm-52_fp8_single_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_glm-52_fp8_single_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "VLLM_ROCM_USE_AITER": "1", + "GPU_ARCHS": "gfx942" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_glm-52_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_glm-52_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_glm-52_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_glm-52_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-52_fp8_single_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-52_fp8_single_threshold.json new file mode 100644 index 000000000..32dc489ff --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_glm-52_fp8_single_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for glm-52_fp8 (single, TP=8, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_gpt-oss-20b_fp8_distributed_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_gpt-oss-20b_fp8_distributed_config.json new file mode 100644 index 000000000..1da0e7aef --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_gpt-oss-20b_fp8_distributed_config.json @@ -0,0 +1,115 @@ +{ + "_comment": "vllm distributed workload: gpt-oss-20b_fp8 on MI300X, TP=4, PP=2 across 2 nodes. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_gpt-oss-20b_fp8_distributed_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_gpt-oss-20b_fp8_distributed_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": {}, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "4", + "pipeline_parallel_size": "2", + "nnodes": "2", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90", + "master_addr": "", + "master_port": "29501" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_gpt-oss-20b_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_gpt-oss-20b_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_gpt-oss-20b_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_gpt-oss-20b_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_gpt-oss-20b_fp8_distributed_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_gpt-oss-20b_fp8_distributed_threshold.json new file mode 100644 index 000000000..009f3328f --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_gpt-oss-20b_fp8_distributed_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for gpt-oss-20b_fp8 (distributed, TP=4, PP=2, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=4,PP=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_gpt-oss-20b_fp8_single_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_gpt-oss-20b_fp8_single_config.json new file mode 100644 index 000000000..96549e36a --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_gpt-oss-20b_fp8_single_config.json @@ -0,0 +1,111 @@ +{ + "_comment": "vllm single workload: gpt-oss-20b_fp8 on MI300X, TP=4 single node. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_gpt-oss-20b_fp8_single_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_gpt-oss-20b_fp8_single_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": {}, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "4", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_gpt-oss-20b_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_gpt-oss-20b_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_gpt-oss-20b_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_gpt-oss-20b_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_gpt-oss-20b_fp8_single_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_gpt-oss-20b_fp8_single_threshold.json new file mode 100644 index 000000000..96a90f579 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_gpt-oss-20b_fp8_single_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for gpt-oss-20b_fp8 (single, TP=4, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=4,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k25_w4a8_distributed_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k25_w4a8_distributed_config.json new file mode 100644 index 000000000..34e51084a --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k25_w4a8_distributed_config.json @@ -0,0 +1,117 @@ +{ + "_comment": "vllm distributed workload: kimi-k25_w4a8 on MI300X, TP=4, PP=2 across 2 nodes. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_kimi-k25_w4a8_distributed_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_kimi-k25_w4a8_distributed_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "4", + "pipeline_parallel_size": "2", + "nnodes": "2", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90", + "master_addr": "", + "master_port": "29501" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_kimi-k25_w4a8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_kimi-k25_w4a8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_kimi-k25_w4a8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_kimi-k25_w4a8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k25_w4a8_distributed_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k25_w4a8_distributed_threshold.json new file mode 100644 index 000000000..3157a1250 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k25_w4a8_distributed_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for kimi-k25_w4a8 (distributed, TP=4, PP=2, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=4,PP=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k25_w4a8_single_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k25_w4a8_single_config.json new file mode 100644 index 000000000..bac58f38c --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k25_w4a8_single_config.json @@ -0,0 +1,113 @@ +{ + "_comment": "vllm single workload: kimi-k25_w4a8 on MI300X, TP=4 single node. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_kimi-k25_w4a8_single_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_kimi-k25_w4a8_single_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "4", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_kimi-k25_w4a8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_kimi-k25_w4a8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_kimi-k25_w4a8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_kimi-k25_w4a8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k25_w4a8_single_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k25_w4a8_single_threshold.json new file mode 100644 index 000000000..f30e6605c --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k25_w4a8_single_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for kimi-k25_w4a8 (single, TP=4, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=4,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k26_mxfp4_distributed_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k26_mxfp4_distributed_config.json new file mode 100644 index 000000000..c629c71a4 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k26_mxfp4_distributed_config.json @@ -0,0 +1,117 @@ +{ + "_comment": "vllm distributed workload: kimi-k26_mxfp4 on MI300X, TP=4, PP=2 across 2 nodes. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_kimi-k26_mxfp4_distributed_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_kimi-k26_mxfp4_distributed_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "4", + "pipeline_parallel_size": "2", + "nnodes": "2", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90", + "master_addr": "", + "master_port": "29501" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_kimi-k26_mxfp4_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_kimi-k26_mxfp4_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_kimi-k26_mxfp4_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_kimi-k26_mxfp4_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k26_mxfp4_distributed_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k26_mxfp4_distributed_threshold.json new file mode 100644 index 000000000..489bdad0d --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k26_mxfp4_distributed_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for kimi-k26_mxfp4 (distributed, TP=4, PP=2, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=4,PP=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k26_mxfp4_single_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k26_mxfp4_single_config.json new file mode 100644 index 000000000..6c3a7a599 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k26_mxfp4_single_config.json @@ -0,0 +1,113 @@ +{ + "_comment": "vllm single workload: kimi-k26_mxfp4 on MI300X, TP=4 single node. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_kimi-k26_mxfp4_single_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_kimi-k26_mxfp4_single_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "4", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_kimi-k26_mxfp4_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_kimi-k26_mxfp4_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_kimi-k26_mxfp4_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_kimi-k26_mxfp4_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k26_mxfp4_single_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k26_mxfp4_single_threshold.json new file mode 100644 index 000000000..3dc9cc7cb --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k26_mxfp4_single_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for kimi-k26_mxfp4 (single, TP=4, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=4,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k27-code_mxfp4_distributed_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k27-code_mxfp4_distributed_config.json new file mode 100644 index 000000000..126c0f2b3 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k27-code_mxfp4_distributed_config.json @@ -0,0 +1,117 @@ +{ + "_comment": "vllm distributed workload: kimi-k27-code_mxfp4 on MI300X, TP=8, PP=2 across 2 nodes. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_kimi-k27-code_mxfp4_distributed_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_kimi-k27-code_mxfp4_distributed_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90", + "master_addr": "", + "master_port": "29501" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_kimi-k27-code_mxfp4_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_kimi-k27-code_mxfp4_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_kimi-k27-code_mxfp4_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_kimi-k27-code_mxfp4_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k27-code_mxfp4_distributed_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k27-code_mxfp4_distributed_threshold.json new file mode 100644 index 000000000..c78187ebe --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k27-code_mxfp4_distributed_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for kimi-k27-code_mxfp4 (distributed, TP=8, PP=2, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k27-code_mxfp4_single_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k27-code_mxfp4_single_config.json new file mode 100644 index 000000000..3dba2524c --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k27-code_mxfp4_single_config.json @@ -0,0 +1,113 @@ +{ + "_comment": "vllm single workload: kimi-k27-code_mxfp4 on MI300X, TP=8 single node. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_kimi-k27-code_mxfp4_single_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_kimi-k27-code_mxfp4_single_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_kimi-k27-code_mxfp4_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_kimi-k27-code_mxfp4_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_kimi-k27-code_mxfp4_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_kimi-k27-code_mxfp4_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k27-code_mxfp4_single_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k27-code_mxfp4_single_threshold.json new file mode 100644 index 000000000..de5b9f607 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_kimi-k27-code_mxfp4_single_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for kimi-k27-code_mxfp4 (single, TP=8, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_llama33-70b_fp8_distributed_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_llama33-70b_fp8_distributed_config.json new file mode 100644 index 000000000..c1f4b23ba --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_llama33-70b_fp8_distributed_config.json @@ -0,0 +1,117 @@ +{ + "_comment": "vllm distributed workload: llama33-70b_fp8 on MI300X, TP=8, PP=2 across 2 nodes. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_llama33-70b_fp8_distributed_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_llama33-70b_fp8_distributed_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "kv-cache-dtype": "fp8" + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90", + "master_addr": "", + "master_port": "29501" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_llama33-70b_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_llama33-70b_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_llama33-70b_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_llama33-70b_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_llama33-70b_fp8_distributed_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_llama33-70b_fp8_distributed_threshold.json new file mode 100644 index 000000000..0d54845f7 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_llama33-70b_fp8_distributed_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for llama33-70b_fp8 (distributed, TP=8, PP=2, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_llama33-70b_fp8_single_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_llama33-70b_fp8_single_config.json new file mode 100644 index 000000000..17873a8ee --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_llama33-70b_fp8_single_config.json @@ -0,0 +1,113 @@ +{ + "_comment": "vllm single workload: llama33-70b_fp8 on MI300X, TP=8 single node. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_llama33-70b_fp8_single_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_llama33-70b_fp8_single_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "kv-cache-dtype": "fp8" + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_llama33-70b_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_llama33-70b_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_llama33-70b_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_llama33-70b_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_llama33-70b_fp8_single_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_llama33-70b_fp8_single_threshold.json new file mode 100644 index 000000000..ca4f45cb3 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_llama33-70b_fp8_single_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for llama33-70b_fp8 (single, TP=8, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mimo-v25-pro_fp8_distributed_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mimo-v25-pro_fp8_distributed_config.json new file mode 100644 index 000000000..c8db53267 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mimo-v25-pro_fp8_distributed_config.json @@ -0,0 +1,117 @@ +{ + "_comment": "vllm distributed workload: mimo-v25-pro_fp8 on MI300X, TP=8, PP=2 across 2 nodes. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_mimo-v25-pro_fp8_distributed_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_mimo-v25-pro_fp8_distributed_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90", + "master_addr": "", + "master_port": "29501" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_mimo-v25-pro_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_mimo-v25-pro_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_mimo-v25-pro_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_mimo-v25-pro_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mimo-v25-pro_fp8_distributed_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mimo-v25-pro_fp8_distributed_threshold.json new file mode 100644 index 000000000..9062ba299 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mimo-v25-pro_fp8_distributed_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for mimo-v25-pro_fp8 (distributed, TP=8, PP=2, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mimo-v25-pro_fp8_single_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mimo-v25-pro_fp8_single_config.json new file mode 100644 index 000000000..9cc6ec4fa --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mimo-v25-pro_fp8_single_config.json @@ -0,0 +1,113 @@ +{ + "_comment": "vllm single workload: mimo-v25-pro_fp8 on MI300X, TP=8 single node. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_mimo-v25-pro_fp8_single_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_mimo-v25-pro_fp8_single_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_mimo-v25-pro_fp8_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_mimo-v25-pro_fp8_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_mimo-v25-pro_fp8_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_mimo-v25-pro_fp8_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mimo-v25-pro_fp8_single_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mimo-v25-pro_fp8_single_threshold.json new file mode 100644 index 000000000..1916624b2 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mimo-v25-pro_fp8_single_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for mimo-v25-pro_fp8 (single, TP=8, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_minimax-m3_bf16_distributed_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_minimax-m3_bf16_distributed_config.json new file mode 100644 index 000000000..a70f0fcd5 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_minimax-m3_bf16_distributed_config.json @@ -0,0 +1,117 @@ +{ + "_comment": "vllm distributed workload: minimax-m3_bf16 on MI300X, TP=8, PP=2 across 2 nodes. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_minimax-m3_bf16_distributed_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_minimax-m3_bf16_distributed_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90", + "master_addr": "", + "master_port": "29501" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_minimax-m3_bf16_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_minimax-m3_bf16_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_minimax-m3_bf16_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_minimax-m3_bf16_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_minimax-m3_bf16_distributed_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_minimax-m3_bf16_distributed_threshold.json new file mode 100644 index 000000000..1b72fcf3f --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_minimax-m3_bf16_distributed_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for minimax-m3_bf16 (distributed, TP=8, PP=2, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_minimax-m3_bf16_single_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_minimax-m3_bf16_single_config.json new file mode 100644 index 000000000..5a8146086 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_minimax-m3_bf16_single_config.json @@ -0,0 +1,113 @@ +{ + "_comment": "vllm single workload: minimax-m3_bf16 on MI300X, TP=8 single node. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_minimax-m3_bf16_single_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_minimax-m3_bf16_single_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_minimax-m3_bf16_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_minimax-m3_bf16_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_minimax-m3_bf16_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_minimax-m3_bf16_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_minimax-m3_bf16_single_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_minimax-m3_bf16_single_threshold.json new file mode 100644 index 000000000..a0c0a3a40 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_minimax-m3_bf16_single_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for minimax-m3_bf16 (single, TP=8, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mistral-large-3_bf16_distributed_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mistral-large-3_bf16_distributed_config.json new file mode 100644 index 000000000..19bb3d2b5 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mistral-large-3_bf16_distributed_config.json @@ -0,0 +1,119 @@ +{ + "_comment": "vllm distributed workload: mistral-large-3_bf16 on MI300X, TP=8, PP=2 across 2 nodes. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_mistral-large-3_bf16_distributed_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_mistral-large-3_bf16_distributed_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "tokenizer-mode": "mistral", + "config-format": "mistral", + "load-format": "mistral" + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "tokenizer_mode": "mistral", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90", + "master_addr": "", + "master_port": "29501" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_mistral-large-3_bf16_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_mistral-large-3_bf16_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_mistral-large-3_bf16_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_mistral-large-3_bf16_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mistral-large-3_bf16_distributed_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mistral-large-3_bf16_distributed_threshold.json new file mode 100644 index 000000000..7d8698a63 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mistral-large-3_bf16_distributed_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for mistral-large-3_bf16 (distributed, TP=8, PP=2, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mistral-large-3_bf16_single_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mistral-large-3_bf16_single_config.json new file mode 100644 index 000000000..841f280c2 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mistral-large-3_bf16_single_config.json @@ -0,0 +1,115 @@ +{ + "_comment": "vllm single workload: mistral-large-3_bf16 on MI300X, TP=8 single node. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_mistral-large-3_bf16_single_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_mistral-large-3_bf16_single_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "tokenizer-mode": "mistral", + "config-format": "mistral", + "load-format": "mistral" + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "mistral", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_mistral-large-3_bf16_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_mistral-large-3_bf16_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_mistral-large-3_bf16_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_mistral-large-3_bf16_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mistral-large-3_bf16_single_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mistral-large-3_bf16_single_threshold.json new file mode 100644 index 000000000..74d249e26 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_mistral-large-3_bf16_single_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for mistral-large-3_bf16 (single, TP=8, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_qwen35-397b-a17b_bf16_distributed_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_qwen35-397b-a17b_bf16_distributed_config.json new file mode 100644 index 000000000..79c5c49ec --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_qwen35-397b-a17b_bf16_distributed_config.json @@ -0,0 +1,117 @@ +{ + "_comment": "vllm distributed workload: qwen35-397b-a17b_bf16 on MI300X, TP=8, PP=2 across 2 nodes. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_qwen35-397b-a17b_bf16_distributed_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_qwen35-397b-a17b_bf16_distributed_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + }, + "ib_hca_devices": "auto", + "ib_netdev": "" + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90", + "master_addr": "", + "master_port": "29501" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_qwen35-397b-a17b_bf16_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_qwen35-397b-a17b_bf16_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_qwen35-397b-a17b_bf16_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_qwen35-397b-a17b_bf16_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_qwen35-397b-a17b_bf16_distributed_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_qwen35-397b-a17b_bf16_distributed_threshold.json new file mode 100644 index 000000000..bbe34df8d --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_qwen35-397b-a17b_bf16_distributed_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for qwen35-397b-a17b_bf16 (distributed, TP=8, PP=2, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_qwen35-397b-a17b_bf16_single_config.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_qwen35-397b-a17b_bf16_single_config.json new file mode 100644 index 000000000..6d1d8f190 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_qwen35-397b-a17b_bf16_single_config.json @@ -0,0 +1,113 @@ +{ + "_comment": "vllm single workload: qwen35-397b-a17b_bf16 on MI300X, TP=8 single node. Sweep carries conc16 1k1k/1k8k/8k1k; only 1k1k is selected in sweep.runs. num_prompts is 320, not the 3200 schema default, to keep this a characterization pass rather than a full-length benchmark -- raise it to 3200 for comparable published numbers. Set container.image, model.id and the models mount before running.", + "schema_version": 1, + "framework": "vllm", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "mi300x_vllm_qwen35-397b-a17b_bf16_single_threshold.json", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.hf_token" + }, + "model": { + "id": "", + "remote": 0 + }, + "container": { + "lifetime": "per_run", + "name": "vllm_qwen35-397b-a17b_bf16_single_mi300x", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + ":/models:ro" + ], + "devices": [ + "/dev/dri", + "/dev/kfd" + ] + } + } + }, + "roles": { + "server": { + "serve_args": { + "trust-remote-code": true + }, + "env": { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1" + } + } + }, + "params": { + "backend": "vllm", + "base_url": "http://0.0.0.0", + "port_no": "8888", + "dataset_name": "random", + "burstiness": "1.0", + "seed": "0", + "request_rate": "inf", + "random_range_ratio": "0.0", + "random_prefix_len": "0", + "tensor_parallelism": "8", + "pipeline_parallel_size": "1", + "nnodes": "1", + "tokenizer_mode": "auto", + "percentile_metrics": "ttft,tpot,itl,e2el", + "metric_percentiles": "50,90,95,99", + "num_prompts": "320", + "client_poll_count": "90" + }, + "sweep": { + "sequence_combinations": [ + { + "name": "w_qwen35-397b-a17b_bf16_1k1k", + "isl": "1024", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_qwen35-397b-a17b_bf16_1k8k", + "isl": "1024", + "osl": "8192", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + }, + { + "name": "w_qwen35-397b-a17b_bf16_8k1k", + "isl": "8192", + "osl": "1024", + "goodput_slo": { + "ttft_ms": 1000000000.0, + "tpot_ms": 1000000000.0, + "e2el_ms": 1000000000.0 + } + } + ], + "runs": [ + { + "combo": "w_qwen35-397b-a17b_bf16_1k1k", + "concurrency": 16 + } + ] + }, + "accuracy": { + "tasks": [] + } +} diff --git a/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_qwen35-397b-a17b_bf16_single_threshold.json b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_qwen35-397b-a17b_bf16_single_threshold.json new file mode 100644 index 000000000..6e0c50386 --- /dev/null +++ b/cvs/input/config_file/inference/vllm_mi300x_workloads/mi300x_vllm_qwen35-397b-a17b_bf16_single_threshold.json @@ -0,0 +1,135 @@ +{ + "_comment": "Uncalibrated placeholders for qwen35-397b-a17b_bf16 (single, TP=8, CONC=16). Every value is 0 and means 'not yet measured' -- covers client.*, gpu.* and prom.*. enforce_thresholds is false in the config, so these record without gating. Replace with measured values from a calibration run before enabling enforcement.", + "_comment_accuracy": "The accuracy block is keyed by accuracy task id, then by lm-eval metric key, so it cannot be pre-enumerated the way client/gpu/prom can -- its contents follow whichever tasks config.json selects in accuracy.tasks, which is empty. Shape to fill in alongside a task: \"accuracy\": {\"\": {\".\": {\"kind\": \"min\", \"value\": 0}}}, where is the results.json key with commas replaced by __ (e.g. gsm8k.exact_match__strict-match). Exempt from the sweep-cell coverage check via NON_SWEEP_THRESHOLD_KEYS.", + "ISL=1024,OSL=1024,TP=8,CONC=16": { + "client.total_token_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.output_throughput": { + "kind": "min_tok_s", + "value": 0 + }, + "client.mean_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_ttft_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_tpot_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_itl_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.mean_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.median_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p90_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p95_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.p99_e2el_ms": { + "kind": "max_ms", + "value": 0 + }, + "client.success_rate": { + "kind": "min", + "value": 0 + }, + "client.failed": { + "kind": "max", + "value": 0 + }, + "gpu.peak_gpu_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_memory_mb": { + "kind": "max", + "value": 0 + }, + "gpu.model_load_s": { + "kind": "max", + "value": 0 + }, + "gpu.gpu_bandwidth_util_pct": { + "kind": "min", + "value": 0 + }, + "gpu.gpu_compute_util_pct": { + "kind": "min", + "value": 0 + }, + "prom.queue_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.queue_time_p95_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p50_ms": { + "kind": "max_ms", + "value": 0 + }, + "prom.prefill_time_p95_ms": { + "kind": "max_ms", + "value": 0 + } + }, + "accuracy": {} +} diff --git a/cvs/input/config_file/preflight/README_preflight_config.md b/cvs/input/config_file/preflight/README_preflight_config.md index f6e2ab29d..d4575b894 100644 --- a/cvs/input/config_file/preflight/README_preflight_config.md +++ b/cvs/input/config_file/preflight/README_preflight_config.md @@ -8,10 +8,11 @@ The preflight checks system validates essential cluster health before running pe 1. **Node Health** - Checks GPU visibility, AMDGPU/KFD, kernel health, and ROCm consistency 2. **MI4XX Scale-up Fabric Admission** - Optionally validates AIFM/AFM/vPOD membership, station masks, and IFoE port state -3. **IFoE L2 Connectivity** - Optionally runs strict `afmctl test ping` coverage before TransferBench and RDMA +3. **IFoE L2 Connectivity (AIMVT-180; opt-in)** - Optionally runs strict `afmctl test ping` coverage before TransferBench and RDMA 4. **TransferBench** - Optionally validates the IFoE data path per node or with a multi-rank cluster run -5. **GID and Interface Consistency** - Ensures configured RDMA interfaces and GID entries are present and consistent -6. **RDMA Connectivity** - Tests node-to-node RDMA communication using `ibv_rc_pingpong` +5. **Primus Node Smoke (opt-in)** - Per-node host / GPU / RDMA roll-call via `primus-cli direct -- node_smoke` +6. **GID and Interface Consistency** - Ensures configured RDMA interfaces and GID entries are present and consistent +7. **RDMA Connectivity** - Tests node-to-node RDMA communication using `ibv_rc_pingpong` ## Configuration File Structure @@ -51,6 +52,13 @@ The preflight configuration file follows this structure: } } }, + "node_smoke": { + "connectivity_mode": "skip", + "auto_setup": true, + "primus_dir": "/home/{user-id}/INSTALL/Primus", + "venv_activate": "/home/{user-id}/envs/preflight/.venv/bin/activate", + "gpus_per_node": 8 + }, "reporting": { "generate_html_report": true, "artifacts_root_dir": "/tmp/{user-id}/preflight", @@ -77,6 +85,7 @@ preflight/ │ └── ifoe/ # MI4XX scale-up fabric checks │ ├── l2ping/ # Strict IFoE L2 connectivity gate │ └── transferbench/ # IFoE data-path validation +├── node_smoke/ # Primus node_smoke per-node health screening (opt-in) ├── reporting/ # Output and report generation └── debug/ # Debug and troubleshooting options ``` @@ -253,6 +262,71 @@ port and validates per-port and aggregate summary accounting. - **`warmup_iterations`** (default: `0`) - Warmup iterations performed before validation +#### Node Smoke Settings (`node_smoke`) — opt-in (Primus Tier 1) + +Runs Primus `node_smoke` on each reachable node via `primus-cli direct --single -- node_smoke` +over parallel SSH (no Slurm required). Reference: Primus `docs/node-smoke-test-instruction.md` +on branch `dev/preflight-direct-test`. + +- **`connectivity_mode`** (default: `"skip"`) + - `"run"` — execute node_smoke on every reachable node + - `"skip"` — preflight records a SKIPPED result and does not invoke Primus +- **`auto_setup`** (default: `true`) + - Clone/update Primus and create the venv with minimal deps (ROCm PyTorch) before node_smoke +- **`setup_timeout`** (default: `600`) + - SSH timeout (seconds) for the per-node Primus auto_setup step +- **`force_reclone`** (default: `false`) + - Remove `primus_dir` and clone fresh on every run (destructive) +- **`shared_install`** (default: `true`) + - Leader node clones/installs on shared NFS home; other nodes wait (recommended for shared home) +- **`pip_install_mode`** (default: `"minimal"`) + - `"minimal"` — ROCm PyTorch only; `"requirements"` — `pip install -r requirements.txt`; `"skip"` — venv only +- **`torch_pip_index_url`** (default: `"https://download.pytorch.org/whl/rocm6.2"`) + - PyTorch wheel index for minimal install; match your ROCm version +- **`primus_git_url`** (default: `"https://github.com/AMD-AIG-AIMA/Primus.git"`) +- **`primus_git_branch`** (default: `"dev/preflight-direct-test"`) +- **`primus_git_recurse_submodules`** (default: `false`) +- **`primus_dir`** (default: `"/home/{user-id}/INSTALL/Primus"`) + - Required when `connectivity_mode` is `"run"`; `{user-id}` is resolved at runtime +- **`venv_activate`** (default: `"/home/{user-id}/envs/preflight/.venv/bin/activate"`) + - Required when `connectivity_mode` is `"run"` +- **`gpus_per_node`** (default: `8`) +- **`master_port`** (default: `1234`) +- **`dump_path`** (default: `""`) + - Per-node smoke JSON output; empty uses `/node_smoke` +- **`expected_rdma_nics`** (default: `null`) + - Defaults to `len(node_check.rdma_interfaces)` when null +- **`ulimit_l_min_gb`** (default: `32`) — FAIL below this memlock limit; `0` disables +- **`shm_min_gb`** (default: `8`) — FAIL below this `/dev/shm` size; `0` disables +- **`skip_dmesg`** (default: `false`) +- **`allow_foreign_procs`** (default: `false`) +- **`allowed_procs`** (default: `"gpuagent,rocm-smi-daemon,amd-smi,dcgm-exporter"`) +- **`require_tools`** (default: `""`) — empty = warn only +- **`nccl_socket_ifname`** / **`gloo_socket_ifname`** (default: `""`) +- **`nccl_ib_hca`** (default: `""`) — defaults to comma-joined `node_check.rdma_interfaces` +- **`nccl_ib_gid_index`** (default: `null`) — defaults to `node_check.gid_index` +- **`ssh_timeout`** (default: `300`) +- **`extra_args`** (default: `[]`) — additional flags forwarded to primus-cli + +#### Tier 2 perf sanity (`node_smoke.tier2_perf`) — optional + +When `tier2_perf` is `true`, preflight forwards `--tier2-perf` to Primus `node_smoke`, enabling all three Tier 2 checks on each node (same as `launch_nodesmoke_ssh.sh -- --tier2-perf`): + +1. **Large GEMM TFLOPS floor** — 8192³ bf16 `torch.matmul`; FAIL below `gemm_tflops_min` (default 600) +2. **HBM D2D bandwidth** — 512 MB device-to-device copy; FAIL below `hbm_gbs_min` (default 2000 GB/s) +3. **Local multi-GPU RCCL all-reduce** — node-local only; FAIL below `rccl_gbs_min` (default 100 GB/s) + +Set `NCCL_IB_HCA`, `NCCL_SOCKET_IFNAME`, and `NCCL_IB_GID_INDEX` (via `node_smoke` config or cluster `env_vars`) before enabling Tier 2 — RCCL init enumerates every transport even though the all-reduce is local-only. + +- **`tier2_perf`** (default: `false`) — master switch; maps to `--tier2-perf` +- **`gemm_tflops_min`** (default: `600`) — `--gemm-tflops-min` +- **`hbm_gbs_min`** (default: `2000`) — `--hbm-gbs-min` +- **`rccl_gbs_min`** (default: `100`) — `--rccl-gbs-min` +- **`rccl_size_mb`** (default: `64`) — `--rccl-size-mb` +- **`rccl_timeout_sec`** (default: `120`) — `--rccl-timeout-sec` + +Tier 2 runs need a longer SSH budget; when `tier2_perf` is enabled the effective timeout is at least 600 seconds even if `ssh_timeout` is lower. + ### Reporting Settings (`reporting`) - **`generate_html_report`** (default: `true`) @@ -335,6 +409,57 @@ port and validates per-port and aggregate summary accounting. } ``` +### Enable Primus Node Smoke with Tier 2 perf + +```json +{ + "preflight": { + "node_check": { + "gid_index": "3", + "expected_rocm_version": "6.4.2", + "rdma_interfaces": ["rdma0", "rdma1", "rdma2", "rdma3", "rdma4", "rdma5", "rdma6", "rdma7"] + }, + "node_smoke": { + "connectivity_mode": "run", + "auto_setup": true, + "shared_install": true, + "primus_dir": "/home/{user-id}/INSTALL/Primus", + "venv_activate": "/home/{user-id}/envs/preflight/.venv/bin/activate", + "gpus_per_node": 8, + "tier2_perf": true, + "gemm_tflops_min": 700, + "hbm_gbs_min": 4500, + "rccl_gbs_min": 180, + "nccl_ib_hca": "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7", + "nccl_ib_gid_index": 3, + "ssh_timeout": 600 + } + } +} +``` + +### Enable Primus Node Smoke + +```json +{ + "preflight": { + "node_check": { + "gid_index": "3", + "expected_rocm_version": "6.4.2", + "rdma_interfaces": ["rdma0", "rdma1", "rdma2", "rdma3", "rdma4", "rdma5", "rdma6", "rdma7"] + }, + "node_smoke": { + "connectivity_mode": "run", + "auto_setup": true, + "shared_install": true, + "primus_dir": "/home/{user-id}/INSTALL/Primus", + "venv_activate": "/home/{user-id}/envs/preflight/.venv/bin/activate", + "gpus_per_node": 8 + } + } +} +``` + ### Advanced Configuration with Debug and Tuning ```json @@ -377,6 +502,11 @@ port and validates per-port and aggregate summary accounting. # Basic usage with default config cvs run preflight_checks --cluster_file cluster.json --config_file preflight_config.json +# Run only the node_smoke check +cvs run preflight_checks test_node_smoke \ + --cluster_file cluster.json \ + --config_file preflight_config.json + # With custom HTML output cvs run preflight_checks \ --cluster_file cluster.json \ @@ -426,6 +556,13 @@ cvs run preflight_checks \ - Confirm all admitted nodes resolve to one consistent vPOD - Reduce to `scope: "node"` to isolate a failing host before retrying cluster scope +8. **Node Smoke Failures** + - Set `node_smoke.connectivity_mode` to `"run"` (default is `"skip"`) + - Verify `primus_dir` and `venv_activate`, or enable `auto_setup: true` + - On shared NFS home, use `shared_install: true` to avoid parallel clone races + - Match `torch_pip_index_url` to your ROCm version + - Review per-node fail reasons in the preflight HTML report + ### Performance Considerations **RDMA Connectivity Testing Times:** @@ -433,6 +570,10 @@ cvs run preflight_checks \ - **Full mesh mode**: ~5-10 minutes for 8 nodes - **Skip mode**: fastest path when validating only node-local checks +**Node Smoke Testing Times:** +- **First run with auto_setup**: several minutes per node (clone + ROCm PyTorch install) +- **Subsequent runs**: ~30–60 seconds per node + **Parallel Processing Impact:** - **Small nodes_per_full_mesh_group (16-32)**: More rounds, less resource usage per node, better for resource-constrained environments - **Large nodes_per_full_mesh_group (128+)**: Fewer rounds, more resource usage per node, faster overall completion diff --git a/cvs/input/config_file/preflight/preflight_config.json b/cvs/input/config_file/preflight/preflight_config.json index f5cdd3d04..4e2cfc91c 100644 --- a/cvs/input/config_file/preflight/preflight_config.json +++ b/cvs/input/config_file/preflight/preflight_config.json @@ -113,6 +113,118 @@ } }, + "node_smoke": { + "_comment": "Primus node_smoke checks via primus-cli direct (opt-in; default skip). See Primus docs/node-smoke-test-instruction.md", + + "_setup_comment": "Primus clone/venv setup runs automatically when auto_setup is true (default). Manual equivalent:", + "_setup_step_1": "git clone --recurse-submodules https://github.com/AMD-AIG-AIMA/Primus.git /home/{user-id}/INSTALL/Primus", + "_setup_step_2": "cd /home/{user-id}/INSTALL/Primus && git checkout dev/preflight-direct-test", + "_setup_step_3": "python3 -m venv /home/{user-id}/envs/preflight/.venv && pip install torch --index-url https://download.pytorch.org/whl/rocm6.2", + "_setup_note": "Paths use {user-id} resolved at runtime. Set auto_setup to false to skip automatic install.", + + "auto_setup": true, + "_comment_auto_setup": "When true, clone/update Primus and create venv with minimal deps (torch) on each node before node_smoke.", + + "setup_timeout": 600, + "_comment_setup_timeout": "SSH timeout (seconds) for the per-node Primus auto_setup step (clone + pip install).", + + "force_reclone": false, + "_comment_force_reclone": "When true, rm -rf primus_dir and clone fresh on every run (destructive). With shared_install (default), only the leader node reclones.", + + "shared_install": true, + "_comment_shared_install": "When true (default), only the first node clones/updates Primus and installs the venv on shared NFS home; other nodes wait. Set false only if primus_dir and venv_activate are local per node.", + + "pip_install_mode": "minimal", + "_comment_pip_install_mode": "Venv deps after clone: 'minimal' (torch only), 'requirements' (pip install -r requirements.txt), or 'skip' (venv only).", + + "torch_pip_index_url": "https://download.pytorch.org/whl/rocm6.2", + "_comment_torch_pip_index_url": "PyTorch wheel index for minimal install. Match your ROCm version (e.g. rocm6.2, rocm7.1).", + + "primus_git_url": "https://github.com/AMD-AIG-AIMA/Primus.git", + "_comment_primus_git_url": "Primus repository URL for one-time clone.", + + "primus_git_branch": "dev/preflight-direct-test", + "_comment_primus_git_branch": "Git branch to checkout after clone. node_smoke and primus-cli direct preflight live on this branch.", + + "primus_git_recurse_submodules": false, + "_comment_primus_git_recurse_submodules": "Clone submodules (Megatron, etc.). false is recommended for node_smoke — submodules are not required and slow setup.", + + "primus_dir": "/home/{user-id}/INSTALL/Primus", + "_comment_primus_dir": "Path where Primus is cloned on each cluster node. Must match the clone target in setup step 1. Required when connectivity_mode is 'run'.", + + "venv_activate": "/home/{user-id}/envs/preflight/.venv/bin/activate", + "_comment_venv_activate": "Path to the Python virtualenv activate script used by primus-cli direct. Required when connectivity_mode is 'run'.", + + "connectivity_mode": "skip", + "_comment_connectivity_mode": "Options: 'run' (host/GPU/RDMA roll-call via node_smoke) or 'skip' (default).", + + "gpus_per_node": 8, + "_comment_gpus_per_node": "Expected GPU count per node (exported as GPUS_PER_NODE and passed to node_smoke --expected-gpus).", + + "master_port": 1234, + "_comment_master_port": "MASTER_PORT for the distributed env primus-cli sets up across SSH-launched ranks.", + + "dump_path": "", + "_comment_dump_path": "Directory for per-node smoke/*.json output. Leave empty to use /node_smoke.", + + "expected_rdma_nics": null, + "_comment_expected_rdma_nics": "Hard-fail when training RDMA NIC count differs. null defaults to len(node_check.rdma_interfaces). Example: 8.", + + "ulimit_l_min_gb": 32, + "_comment_ulimit_l_min_gb": "FAIL when RLIMIT_MEMLOCK is below this many GiB. 0 disables.", + + "shm_min_gb": 8, + "_comment_shm_min_gb": "FAIL when /dev/shm is below this many GiB. 0 disables.", + + "skip_dmesg": false, + "_comment_skip_dmesg": "Skip the dmesg recent-error scan (use inside unprivileged containers).", + + "allow_foreign_procs": false, + "_comment_allow_foreign_procs": "Do not FAIL on foreign GPU processes. Recommended inside containers where proc names resolve to N/A.", + + "allowed_procs": "gpuagent,rocm-smi-daemon,amd-smi,dcgm-exporter", + "_comment_allowed_procs": "Comma-separated process names allowed to hold GPUs without failing the node.", + + "require_tools": "", + "_comment_require_tools": "Comma-separated tools that must exist in PATH for PASS (amd-smi, rocm-smi, lsof). Empty = warn only.", + + "nccl_socket_ifname": "", + "_comment_nccl_socket_ifname": "Optional NCCL_SOCKET_IFNAME / GLOO_SOCKET_IFNAME override for node_smoke.", + + "gloo_socket_ifname": "", + "_comment_gloo_socket_ifname": "Optional GLOO_SOCKET_IFNAME override (defaults to nccl_socket_ifname when empty).", + + "nccl_ib_hca": "", + "_comment_nccl_ib_hca": "Optional NCCL_IB_HCA override. Defaults to comma-joined node_check.rdma_interfaces.", + + "nccl_ib_gid_index": null, + "_comment_nccl_ib_gid_index": "Optional NCCL_IB_GID_INDEX override. Defaults to node_check.gid_index.", + + "ssh_timeout": 300, + "_comment_ssh_timeout": "SSH timeout in seconds for each node's node_smoke invocation (~30s Tier 1; use 600+ with tier2_perf).", + + "tier2_perf": false, + "_comment_tier2_perf": "Enable Tier 2 perf sanity (--tier2-perf): 8192³ GEMM TFLOPS, HBM D2D copy bandwidth, local multi-GPU RCCL all-reduce. Requires NCCL_IB_HCA / NCCL_SOCKET_IFNAME (see launch_nodesmoke_ssh.sh).", + + "gemm_tflops_min": 600, + "_comment_gemm_tflops_min": "Tier 2 FAIL below this large GEMM TFLOPS (--gemm-tflops-min). MI300X healthy nodes typically exceed 600.", + + "hbm_gbs_min": 2000, + "_comment_hbm_gbs_min": "Tier 2 FAIL below this HBM device-to-device bandwidth in GB/s (--hbm-gbs-min). MI300X healthy ≈ 4500–5000.", + + "rccl_gbs_min": 100, + "_comment_rccl_gbs_min": "Tier 2 FAIL below this local multi-GPU RCCL all-reduce bandwidth in GB/s (--rccl-gbs-min).", + + "rccl_size_mb": 64, + "_comment_rccl_size_mb": "Tier 2 local RCCL all-reduce tensor size in MB (--rccl-size-mb).", + + "rccl_timeout_sec": 120, + "_comment_rccl_timeout_sec": "Tier 2 local RCCL all-reduce hard timeout in seconds (--rccl-timeout-sec).", + + "extra_args": [], + "_comment_extra_args": "Additional node_smoke CLI flags forwarded to primus-cli. Example: [\"--no-clean-dump-path\"]." + }, + "reporting": { "_comment": "Post-test reporting and output", diff --git a/cvs/input/config_file/training/jaxmaxtext/README.md b/cvs/input/config_file/training/jaxmaxtext/README.md new file mode 100644 index 000000000..2057fe990 --- /dev/null +++ b/cvs/input/config_file/training/jaxmaxtext/README.md @@ -0,0 +1,170 @@ +# JAX MaxText Training - Config and Threshold Files + +This folder holds the input files for the `jaxmaxtext_single` / +`jaxmaxtext_distributed` suites (see +`cvs/tests/training/jaxmaxtext/README.md` for how to run them). Each **config** +file has a sibling **threshold** file (referenced by its `threshold_json` +field). One config = one GPU arch + mode (single or distributed). + +## File inventory + +| Config | Threshold | Arch / mode | +|---|---|---| +| `mi300x_jaxmaxtext_llama-3.3-70b_single.json` | `mi300x_jaxmaxtext_llama-3.3-70b_single_threshold.json` | MI300X, single-node | +| `mi300x_jaxmaxtext_llama-3.3-70b_distributed.json` | `mi300x_jaxmaxtext_llama-3.3-70b_distributed_threshold.json` | MI300X, distributed | +| `mi325x_jaxmaxtext_llama-3.3-70b_distributed.json` | `mi325x_jaxmaxtext_llama-3.3-70b_distributed_threshold.json` | MI325X, distributed | + +Add analogous config + threshold pairs for other archs (e.g. MI355X) as needed. + +Keys prefixed with `_` (e.g. `_error_patterns_comment`) are inline comments and +are ignored by the loader. + +## What you MUST change for your cluster / setup + +Start from the config closest to your target arch/mode and edit these: + +| Where | Variable | Change to | +|---|---|---| +| `container.image` | container image | Your MaxText/JAX ROCm image tag present on the nodes | +| `container.name` | container name | Any unique name (optional) | +| `paths.shared_fs` | base path (`/home/{user-id}`) | A path reachable from all nodes; `{user-id}` resolves to the cluster/OS user | +| `paths.hf_token_file` | HF token path | Location of your Hugging Face token file on the nodes | +| `training.tokenizer.hf_model_id` | tokenizer repo | The HF tokenizer to download (matches the model) | +| `training.tokenizer.tokenizer_path` | in-container tokenizer dir | Where the tokenizer is written (usually under `{paths.models_dir}`) | +| `training.gpus_per_node` | GPUs per node | Your node's GPU count (default 8); feeds total-throughput/scaling metrics - do not assume a fixed topology | +| `training.nic_type` | NIC type | `thor2` (Broadcom) etc. for distributed; `none` for single-node | +| `training.rdma_lib.*` | RDMA lib paths | Host/container paths for the NIC's `libibverbs` provider (distributed) | +| `training.nccl.ib_hca` / `ib_hca_list` | RDMA HCA devices | Your nodes' RDMA device names (e.g. `rdma0..rdma7`) | +| `training.nccl.socket_ifname` / `gloo_socket_ifname` | control NIC | Your management interface name (e.g. `eno0`) | +| `training.jax_distributed.coordinator_ip` | JAX coordinator | Keep `auto` (uses the first node in the cluster `node_dict`), or set a specific IP | +| `training.sweeps[].maxtext_overrides.quantization` | FP8 recipe | `nanoo_fp8` on MI300X/MI325X (CDNA3); `fp8` on MI355X/MI350X (CDNA4) | +| `training.scaling_baseline.tokens_per_sec_total` | 1-node baseline | Your measured single-node total tok/s (0.0 disables scaling efficiency) | +| `.json` gated values | thresholds | Calibrated PASS/FAIL bounds for your hardware | +| cluster file `node_dict` | node IPs | Your node IPs (first entry is the coordinator when `coordinator_ip: auto`) | + +Also set `training.enabled_sweep_list` to the sweep(s) you want to run (each is a +full training run), and `enforce_thresholds` to `true` for real PASS/FAIL or +`false` for record-only. + +## Placeholder substitution + +Configs use placeholders resolved at load time: + +- `{user-id}` - the cluster username (or the local OS user as fallback). +- `{shared_fs}` - self-reference within the `paths` block. +- `{paths.models_dir}` (and other `{paths.*}`) - cross-referenced anywhere. + +`threshold_json` is a literal filename resolved next to the config; no +placeholder substitution is applied to it. + +## Config structure + +Top-level (framework-agnostic) fields: + +| Field | Meaning | +|---|---| +| `schema_version` | Always `1` | +| `framework` | `jaxmaxtext` | +| `gpu_arch` | `mi300x` / `mi325x` / `mi355x` (labels the run) | +| `enforce_thresholds` | `true` = metrics gate PASS/FAIL; `false` = record-only | +| `threshold_json` | Sibling threshold filename | +| `paths` | `shared_fs`, `models_dir`, `log_dir`, `hf_token_file` | +| `model` | `id`, `remote` (0 = already cached), `precision` (label) | +| `container` | `lifetime`, `name`, `image`, `runtime` (docker `args`: network/ipc/privileged/shm-size/ulimit/volumes) | + +### `training` block + +| Field | Meaning | +|---|---| +| `distributed` | `true` for multi-node (adds the RDMA setup stage), `false` for single-node | +| `gpus_per_node` | GPUs per node (default 8); `num_gpus = num_nodes x gpus_per_node` feeds `tokens_per_sec_total` and scaling efficiency | +| `verify_dmesg` | Scan host `dmesg` on all nodes for GPU/HW/kernel faults over the training window (default `true`); set `false` on clusters without passwordless `sudo` for `dmesg` | +| `steps` | Training steps; also drives completion detection and poll budget | +| `enable_checkpointing` | Whether MaxText writes checkpoints | +| `train_script_paths` | Candidate in-container paths to the MaxText train entrypoint; the job picks the first one that exists in the running container. List them newest-first (e.g. v26.4+ path before the v26.3 path) so a version bump only needs a new entry, not an edit. `train_script` (single path) is still accepted as a deprecated fallback. | +| `maxtext_config` | MaxText YAML params written verbatim (see below) | +| `tokenizer` | `hf_model_id` (download source), `tokenizer_path` (in-container dir) | +| `nic_type` | NIC family; `thor2` triggers the RDMA-lib copy, `none` skips it | +| `rdma_lib` | Host/container paths for the NIC's libibverbs provider (distributed) | +| `env_vars` | Environment exported before training (NCCL/NVTE/HIP/XLA client) | +| `xla_flags` | `XLA_FLAGS` passed to the run | +| `nccl` | RDMA HCA list + control interface names for distributed comms | +| `jax_distributed` | `coordinator_ip` (`auto` = first node), `coordinator_port`, init/heartbeat timeouts | +| `scaling_baseline` | 1-node `tokens_per_sec_total` + `num_nodes` for scaling-efficiency % | +| `convergence` | `target_metric` (`auto`/`train_loss`/`eval_loss`) + `target_value` for time-to-target | +| `loss_curve` | `sample_every`, `milestone_steps`, `max_slope`, `enforce` for the loss-curve check | +| `error_patterns` | `{name: regex}` scanned in the training log (see below) | +| `sweeps` | List of `{name, maxtext_overrides}`; `name` is the threshold cell key | +| `enabled_sweep_list` | Subset of sweep names to actually run | + +### `maxtext_config` (selected keys) + +Written straight into the MaxText YAML, so any valid MaxText param can be set +here. Common ones: `base_config`, `hardware`, `attention`, `dtype`, +`weight_dtype`, `quantization`, `dataset_type`, `per_device_batch_size`, +`max_target_length`, the `ici_*` / `dcn_*` parallelism dims, `remat_policy`, +`scan_layers`, and `eval_interval` / `eval_steps` (set `eval_interval > 0` with a +validation dataset to produce `eval_loss`). Note `steps`, `enable_checkpointing`, +`run_name`, `base_output_directory`, and `tokenizer_path` are injected by the +driver and should not be set here. + +### Sweeps and FP8 + +Each sweep is a full training run; `maxtext_overrides` merges onto +`maxtext_config` for that run. The `name` encodes the cell as +`NNODES=..,STEPS=..,PRECISION=..,BATCH=..,GBS=..,SEQLEN=..` and must match the +key used in the threshold file. `NNODES` (cluster), `STEPS` (`training.steps`), +and `GBS` (derived = `per_device_batch_size x total GPUs`) are labels only - set +the real knobs (`per_device_batch_size`, `max_target_length`, precision) in +`maxtext_overrides`. + +FP8 quantization value by arch: + +| Arch | `quantization` for FP8 | +|---|---| +| MI300X, MI325X (CDNA3) | `nanoo_fp8` | +| MI355X, MI350X (CDNA4) | `fp8` | + +BF16 sweeps use `quantization: ""` and keep `dtype`/`weight_dtype: bfloat16`. + +### `error_patterns` + +`{name: regex}` scanned in each node's `training.log` during polling; a match +fails that sweep's `test_training_run` with the matched name. Add/remove entries +as you find new signatures. Remove the whole block to fall back to the driver's +built-in defaults. Escape backslashes per JSON (e.g. two backslashes for `\d`). + +## Threshold files + +A threshold file maps each **sweep name** (cell key) to `{metric: spec}`. A +metric is gated only when `enforce_thresholds: true` and it has a numeric spec; +otherwise it is recorded. Metrics with no value this run report `N/A`. + +Threshold kinds: + +| kind | Passes when | Notes | +|---|---|---| +| `min` | `actual >= value` | lower bound | +| `max` | `actual <= value` | upper bound | +| `max_ms` | `actual <= value` | upper bound, `ms` in the message | +| `min_tok_s` | `actual >= value` | lower bound, `tok/s` in the message | +| `within` | `value +/- tolerance_pct%` | needs `tolerance_pct` | +| `min_ratio` | `actual / actuals[reference] >= value` | needs `reference` | +| `info` | always | record-only; retains a default `value` placeholder to calibrate later | + +Example cell: + +```json +"NNODES=2,STEPS=30,PRECISION=FP8,BATCH=3,GBS=48,SEQLEN=8192": { + "training.tflops_per_sec_per_gpu": { "kind": "min", "value": 350.0 }, + "training.tokens_per_sec_per_gpu": { "kind": "min", "value": 700.0 }, + "training.final_loss": { "kind": "max", "value": 15.0 }, + "training.loss_decreased": { "kind": "min", "value": 1 }, + "training.step_time_p95_ms": { "kind": "info", "value": 3600000.0 } +} +``` + +To start gating a metric currently marked `info`: replace `"kind": "info"` with +`min`/`max`/etc. and set a calibrated `value`. The threshold cell key must match +the sweep's `name` exactly (including `NNODES`), or the metric falls back to +`RECORD`. diff --git a/cvs/input/config_file/training/jaxmaxtext/mi300x_jaxmaxtext_llama-3.3-70b_distributed.json b/cvs/input/config_file/training/jaxmaxtext/mi300x_jaxmaxtext_llama-3.3-70b_distributed.json new file mode 100644 index 000000000..89e0e2ab6 --- /dev/null +++ b/cvs/input/config_file/training/jaxmaxtext/mi300x_jaxmaxtext_llama-3.3-70b_distributed.json @@ -0,0 +1,230 @@ +{ + "schema_version": 1, + "framework": "jaxmaxtext", + "gpu_arch": "mi300x", + "enforce_thresholds": true, + "threshold_json": "mi300x_jaxmaxtext_llama-3.3-70b_distributed_threshold.json", + + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "{shared_fs}/cache/maxtext", + "log_dir": "{shared_fs}/LOGS/jaxmaxtext", + "hf_token_file": "{shared_fs}/.hf_token" + }, + + "model": { + "id": "llama3.3-70b", + "remote": 0, + "precision": "bfloat16" + }, + + "__image": "rocm/jax-training:maxtext-v26.4", + "container": { + "lifetime": "per_run", + "name": "rocm-jaxmaxtext-llama3.3-70b", + "image": "rocm/jax-training:maxtext-v26.4", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm-size": "256G", + "ulimit": ["nofile=65535:65535"], + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/dev/infiniband:/dev/infiniband", + "/usr/local/lib/libbnxt_re-rdmav34.so:/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so.host", + "/lib/libibverbs.d:/lib/libibverbs.d", + "/tmp/{user-id}/jax/TRAINING_LOGS:/workspace/maxtext/output" + ] + } + } + }, + + "__maxtext_version<=26.3__train_script": "/workspace/maxtext/src/MaxText/train.py", + "__maxtext_version>=26.4__train_script": "/workspace/maxtext/src/maxtext/trainers/pre_train/train.py", + "training": { + "distributed": true, + "gpus_per_node": 8, + "steps": 10, + "enable_checkpointing": false, + "train_script_paths": [ + "/workspace/maxtext/src/maxtext/trainers/pre_train/train.py", + "/workspace/maxtext/src/MaxText/train.py" + ], + + "maxtext_config": { + "base_config": "base.yml", + "hardware": "gpu", + "attention": "cudnn_flash_te", + "dtype": "bfloat16", + "dataset_type": "synthetic", + "remat_policy": "full", + "use_iota_embed": true, + "scan_layers": true, + "per_device_batch_size": 2, + "max_target_length": 8192, + "async_checkpointing": false, + "quantization": "", + "weight_dtype": "bfloat16", + "shardy": false, + "logits_dot_in_fp32": false, + "megablox": false, + "packing": true, + "enable_goodput_recording": false, + "monitor_goodput": false, + "optimizer_memory_host_offload": false, + "param_scan_axis": 1, + "ici_fsdp_parallelism": 8, + "ici_data_parallelism": 1, + "ici_sequence_parallelism": 1, + "ici_tensor_parallelism": 1, + "ici_pipeline_parallelism": 1, + "dcn_data_parallelism": -1, + "dcn_fsdp_parallelism": 1, + "dcn_pipeline_parallelism": 1, + "dcn_tensor_parallelism": 1, + "dcn_sequence_parallelism": 1, + "max_segments_per_seq": 32, + "skip_first_n_steps_for_profiler": 3, + "eval_interval": -1, + "eval_steps": -1 + }, + + "tokenizer": { + "hf_model_id": "NousResearch/Meta-Llama-3-70B", + "tokenizer_path": "{paths.models_dir}/Meta-Llama-70-B" + }, + + "nic_type": "thor2", + + "rdma_lib": { + "host_source_file": "/usr/local/lib/libbnxt_re-rdmav34.so", + "container_mount_file": "/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so.host", + "container_dest_file": "/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so" + }, + + "_NCCL_IB_DISABLE_comment": "IB/RoCE over Broadcom bnxt_re segfaults in RCCL QP setup on these nodes (crashes on the first inter-node channel, independent of GDR / matched rdma-core / channel-count). TCP works. Remove this once the image ships an RCCL build validated against the bnxt_re RoCE stack.", + "env_vars": { + "NNODES": "2", + "GPU_MAX_HW_QUEUES": "2", + "HSA_FORCE_FINE_GRAIN_PCIE": "1", + "HIP_FORCE_DEV_KERNARG": "1", + "XLA_PYTHON_CLIENT_MEM_FRACTION": "0.93", + "NCCL_DEBUG": "ERROR", + "NCCL_IB_DISABLE": "1", + "NCCL_PROTO": "Simple", + "NCCL_IB_TC": "41", + "NCCL_IB_SL": "0", + "NCCL_IB_GID_INDEX": "3", + "NCCL_CHECKS_DISABLE": "1", + "NCCL_CROSS_NIC": "0", + "NVTE_ALLOW_NONDETERMINISTIC_ALGO": "1", + "NVTE_USE_HIPBLASLT": "1", + "NVTE_FUSED_ATTN": "1", + "NVTE_CK_USES_BWD_V3": "1", + "NVTE_CK_USES_FWD_V3": "1", + "NVTE_CK_IS_V3_ATOMIC_FP32": "0", + "NVTE_CK_HOW_V3_BF16_CVT": "2", + "NVTE_FUSED_ATTN_CK": "1", + "NVTE_FUSED_ATTN_AOTRITON": "0" + }, + + "xla_flags": { + "xla_gpu_enable_latency_hiding_scheduler": "True", + "xla_gpu_enable_triton_gemm": "False", + "xla_gpu_memory_limit_slop_factor": "95", + "xla_gpu_enable_command_buffer": "''", + "xla_gpu_enable_cublaslt": "True", + "xla_gpu_autotune_level": "0", + "xla_gpu_enable_reduce_scatter_combine_by_dim": "false", + "xla_gpu_reduce_scatter_combine_threshold_bytes": "8589934592", + "xla_gpu_all_reduce_combine_threshold_bytes": "8589934592", + "xla_gpu_all_gather_combine_threshold_bytes": "8589934592", + "xla_gpu_enable_all_gather_combine_by_dim": "FALSE" + }, + + "_nccl_comment": "Cluster-specific RDMA/NIC devices. Replace every '' with your node's values (see the sibling _example_* entries); distributed runs hard-exit at config load until you do. Discover them with `ibv_devices` (HCAs) and `ip -br link` (host interface).", + "nccl": { + "_example_ib_hca_list": "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7", + "ib_hca_list": "", + "_example_ib_hca": "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7", + "ib_hca": "", + "_example_socket_ifname": "eno0", + "socket_ifname": "", + "_example_gloo_socket_ifname": "eno0", + "gloo_socket_ifname": "", + "ib_tc": "41", + "ib_sl": "0", + "ib_gid_index": "3" + }, + + "jax_distributed": { + "coordinator_ip": "auto", + "coordinator_port": "12346", + "initialization_timeout_seconds": "1800", + "heartbeat_timeout_seconds": "900" + }, + + "_scaling_baseline_comment": "1-node total tokens/sec baseline for scaling-efficiency %. Sourced from a prior single-node run (num_slices=1, 8 GPUs): 49569.59 tok/s/GPU * 8. Re-measure when the model/precision/seqlen changes.", + "scaling_baseline": { + "tokens_per_sec_total": 396556.72, + "num_nodes": 1 + }, + + "_convergence_comment": "to enable validation loss set eval_interval > 0 (and eval_steps) in maxtext_config; this needs a validation dataset (eval_split/eval_dataset_name). target_value <= 0 disables convergence (record-only). target_metric 'auto' uses eval_loss when eval runs, else training loss.", + "convergence": { + "target_metric": "auto", + "target_value": 0.0 + }, + + "_loss_curve_comment": "Row 32: sample training loss every `sample_every` steps (plus milestone_steps) and pass when the least-squares slope < max_slope. enforce=false makes it record-only.", + "loss_curve": { + "sample_every": 10, + "milestone_steps": [100, 500, 1000, 5000], + "max_slope": 0.0, + "enforce": true + }, + + "_error_patterns_comment": "Regexes (name -> pattern) scanned in each node's training.log during polling; a match fails that sweep's training_run with the matched name. Add/remove entries as you find new error signatures. Remove this block to use the built-in defaults. Escape backslashes per JSON (e.g. two backslashes for a regex \\d).", + "error_patterns": { + "NCCL ERROR": "NCCL ERROR|NCCL timeout|local work queue catastrophic error", + "GPU HW ERROR": "HW Exception by GPU|GPU Hang|Uncorrectable error|GPU Reset", + "AssertionError": "AssertionError|ValueError:|JaxStackTrace|During handling of the above exception|triggered the following exception", + "rocm Err": "FAILED_PRECONDITION: No visible GPU devices|failed call to hipInit: HIP_ERROR_NoDevice|librocm reported version is: NOT_FOUND", + "python err": "ModuleNotFoundError: No module named|Fatal Python error:", + "tensorflow": "tensorflow.CoordinationServiceError|tensorflow.BarrierError|CoordinationServiceError", + "resource": "RESOURCE_EXHAUSTED: Out of memory|failed: RESOURCE_EXHAUSTED", + "segfault": "Segmentation fault|SIGSEGV|core dumped|std::bad_alloc" + }, + + "_sweeps_comment": "Each sweep = one full training run; `name` is the threshold cell key. Add more sweeps (e.g. FP8) and list them in enabled_sweep_list to run them.", + "sweeps": [ + { + "name": "NNODES=2,STEPS=10,PRECISION=BF16,BATCH=2,GBS=32,SEQLEN=8192", + "maxtext_overrides": { + "per_device_batch_size": 2, + "max_target_length": 8192, + "dtype": "bfloat16", + "weight_dtype": "bfloat16", + "quantization": "" + } + }, + { + "name": "NNODES=2,STEPS=10,PRECISION=FP8,BATCH=2,GBS=32,SEQLEN=8192", + "maxtext_overrides": { + "per_device_batch_size": 2, + "max_target_length": 8192, + "dtype": "bfloat16", + "weight_dtype": "bfloat16", + "quantization": "nanoo_fp8" + } + } + ], + "enabled_sweep_list": [ + "NNODES=2,STEPS=10,PRECISION=BF16,BATCH=2,GBS=32,SEQLEN=8192", + "NNODES=2,STEPS=10,PRECISION=FP8,BATCH=2,GBS=32,SEQLEN=8192" + ] + } +} diff --git a/cvs/input/config_file/training/jaxmaxtext/mi300x_jaxmaxtext_llama-3.3-70b_distributed_threshold.json b/cvs/input/config_file/training/jaxmaxtext/mi300x_jaxmaxtext_llama-3.3-70b_distributed_threshold.json new file mode 100644 index 000000000..83074e0ad --- /dev/null +++ b/cvs/input/config_file/training/jaxmaxtext/mi300x_jaxmaxtext_llama-3.3-70b_distributed_threshold.json @@ -0,0 +1,35 @@ +{ + "_comment": "Thresholds for Llama-3.3-70B on MI300X distributed, keyed by sweep name. Gated metrics (min/max) drive PASS/FAIL; kind='info' metrics always PASS (record-only) but retain a default `value` as a placeholder to calibrate later. Requires enforce_thresholds=true.", + + "NNODES=2,STEPS=10,PRECISION=BF16,BATCH=2,GBS=32,SEQLEN=8192": { + "training.tflops_per_sec_per_gpu": { "kind": "min", "value": 260.0 }, + "training.tokens_per_sec_per_gpu": { "kind": "min", "value": 962.0 }, + "training.tokens_per_sec_total": { "kind": "info", "value": 0 }, + "training.scaling_efficiency_pct": { "kind": "info", "value": 80.0 }, + "training.step_time_seconds": { "kind": "info", "value": 3600.0 }, + "training.step_time_mean_ms": { "kind": "info", "value": 3600000.0 }, + "training.step_time_p50_ms": { "kind": "info", "value": 3600000.0 }, + "training.step_time_p95_ms": { "kind": "info", "value": 3600000.0 }, + "training.final_loss": { "kind": "max", "value": 15.0 }, + "training.loss_decreased": { "kind": "min", "value": 1 }, + "training.eval_loss": { "kind": "info", "value": 100.0 }, + "training.steps_to_target": { "kind": "info", "value": 1000000 }, + "training.time_to_target_seconds": { "kind": "info", "value": 1000000.0 } + }, + + "NNODES=2,STEPS=10,PRECISION=FP8,BATCH=2,GBS=32,SEQLEN=8192": { + "training.tflops_per_sec_per_gpu": { "kind": "min", "value": 300.0 }, + "training.tokens_per_sec_per_gpu": { "kind": "min", "value": 1472.0 }, + "training.tokens_per_sec_total": { "kind": "info", "value": 0 }, + "training.scaling_efficiency_pct": { "kind": "info", "value": 80.0 }, + "training.step_time_seconds": { "kind": "info", "value": 3600.0 }, + "training.step_time_mean_ms": { "kind": "info", "value": 3600000.0 }, + "training.step_time_p50_ms": { "kind": "info", "value": 3600000.0 }, + "training.step_time_p95_ms": { "kind": "info", "value": 3600000.0 }, + "training.final_loss": { "kind": "max", "value": 15.0 }, + "training.loss_decreased": { "kind": "min", "value": 1 }, + "training.eval_loss": { "kind": "info", "value": 100.0 }, + "training.steps_to_target": { "kind": "info", "value": 1000000 }, + "training.time_to_target_seconds": { "kind": "info", "value": 1000000.0 } + } +} diff --git a/cvs/input/config_file/training/jaxmaxtext/mi300x_jaxmaxtext_llama-3.3-70b_single.json b/cvs/input/config_file/training/jaxmaxtext/mi300x_jaxmaxtext_llama-3.3-70b_single.json new file mode 100644 index 000000000..a9e9dce45 --- /dev/null +++ b/cvs/input/config_file/training/jaxmaxtext/mi300x_jaxmaxtext_llama-3.3-70b_single.json @@ -0,0 +1,177 @@ +{ + "schema_version": 1, + "framework": "jaxmaxtext", + "gpu_arch": "mi300x", + "enforce_thresholds": true, + "threshold_json": "mi300x_jaxmaxtext_llama-3.3-70b_single_threshold.json", + + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "{shared_fs}/cache/maxtext", + "log_dir": "{shared_fs}/LOGS/jaxmaxtext", + "hf_token_file": "{shared_fs}/.hf_token" + }, + + "model": { + "id": "llama3.3-70b", + "remote": 0, + "precision": "bfloat16" + }, + + "container": { + "lifetime": "per_run", + "name": "rocm-jaxmaxtext-llama3.3-70b-single", + "image": "rocm/jax-training:maxtext-v26.4", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm-size": "256G", + "ulimit": ["nofile=65535:65535"], + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/tmp/{user-id}/jax/TRAINING_LOGS:/workspace/maxtext/output" + ] + } + } + }, + + "__maxtext_version<=26.3__train_script": "/workspace/maxtext/src/MaxText/train.py", + "__maxtext_version>=26.4__train_script": "/workspace/maxtext/src/maxtext/trainers/pre_train/train.py", + "training": { + "distributed": false, + "gpus_per_node": 8, + "steps": 30, + "enable_checkpointing": false, + "train_script_paths": [ + "/workspace/maxtext/src/maxtext/trainers/pre_train/train.py", + "/workspace/maxtext/src/MaxText/train.py" + ], + + "maxtext_config": { + "base_config": "base.yml", + "hardware": "gpu", + "attention": "cudnn_flash_te", + "dtype": "bfloat16", + "dataset_type": "synthetic", + "remat_policy": "full", + "use_iota_embed": true, + "scan_layers": true, + "per_device_batch_size": 2, + "max_target_length": 8192, + "async_checkpointing": false, + "quantization": "", + "weight_dtype": "bfloat16", + "shardy": false, + "logits_dot_in_fp32": false, + "megablox": false, + "packing": true, + "enable_goodput_recording": false, + "monitor_goodput": false, + "optimizer_memory_host_offload": false, + "param_scan_axis": 1, + "ici_fsdp_parallelism": 8, + "ici_data_parallelism": 1, + "ici_sequence_parallelism": 1, + "ici_tensor_parallelism": 1, + "ici_pipeline_parallelism": 1, + "max_segments_per_seq": 32, + "skip_first_n_steps_for_profiler": 3, + "eval_interval": -1, + "eval_steps": -1 + }, + + "tokenizer": { + "hf_model_id": "NousResearch/Meta-Llama-3-70B", + "tokenizer_path": "{paths.models_dir}/Meta-Llama-70-B" + }, + + "nic_type": "none", + + "_NCCL_IB_DISABLE_comment": "Single-node run uses intra-node RCCL (no inter-node IB), so this is a no-op here; kept for parity with the distributed configs where IB/RoCE over Broadcom bnxt_re segfaults and TCP is required.", + "env_vars": { + "NNODES": "1", + "NODE_RANK": "0", + "GPU_MAX_HW_QUEUES": "2", + "HSA_FORCE_FINE_GRAIN_PCIE": "1", + "HIP_FORCE_DEV_KERNARG": "1", + "XLA_PYTHON_CLIENT_MEM_FRACTION": "0.93", + "NCCL_DEBUG": "ERROR", + "NCCL_IB_DISABLE": "1", + "NVTE_ALLOW_NONDETERMINISTIC_ALGO": "1", + "NVTE_USE_HIPBLASLT": "1", + "NVTE_FUSED_ATTN": "1", + "NVTE_CK_USES_BWD_V3": "1", + "NVTE_CK_USES_FWD_V3": "1", + "NVTE_CK_IS_V3_ATOMIC_FP32": "0", + "NVTE_CK_HOW_V3_BF16_CVT": "2", + "NVTE_FUSED_ATTN_CK": "1", + "NVTE_FUSED_ATTN_AOTRITON": "0" + }, + + "xla_flags": { + "xla_gpu_enable_latency_hiding_scheduler": "True", + "xla_gpu_enable_triton_gemm": "False", + "xla_gpu_memory_limit_slop_factor": "95", + "xla_gpu_enable_command_buffer": "", + "xla_gpu_enable_cublaslt": "True", + "xla_gpu_autotune_level": "0" + }, + + "_convergence_comment": "to enable validation loss set eval_interval > 0 (and eval_steps) in maxtext_config; this needs a validation dataset (eval_split/eval_dataset_name). target_value <= 0 disables convergence (record-only). target_metric 'auto' uses eval_loss when eval runs, else training loss.", + "convergence": { + "target_metric": "auto", + "target_value": 0.0 + }, + + "_loss_curve_comment": "sample training loss every `sample_every` steps (plus milestone_steps) and pass when the least-squares slope < max_slope. enforce=false makes it record-only.", + "loss_curve": { + "sample_every": 10, + "milestone_steps": [100, 500, 1000, 5000], + "max_slope": 0.0, + "enforce": true + }, + + "_error_patterns_comment": "Regexes (name -> pattern) scanned in each node's training.log during polling; a match fails that sweep's training_run with the matched name. Add/remove entries as you find new error signatures. Remove this block to use the built-in defaults. Escape backslashes per JSON (e.g. two backslashes for a regex \\d).", + "error_patterns": { + "NCCL ERROR": "NCCL ERROR|NCCL timeout|local work queue catastrophic error", + "GPU HW ERROR": "HW Exception by GPU|GPU Hang|Uncorrectable error|GPU Reset", + "AssertionError": "AssertionError|ValueError:|JaxStackTrace|During handling of the above exception|triggered the following exception", + "rocm Err": "FAILED_PRECONDITION: No visible GPU devices|failed call to hipInit: HIP_ERROR_NoDevice|librocm reported version is: NOT_FOUND", + "python err": "ModuleNotFoundError: No module named|Fatal Python error:", + "tensorflow": "tensorflow.CoordinationServiceError|tensorflow.BarrierError|CoordinationServiceError", + "resource": "RESOURCE_EXHAUSTED: Out of memory|failed: RESOURCE_EXHAUSTED", + "segfault": "Segmentation fault|SIGSEGV|core dumped|std::bad_alloc" + }, + + "_sweeps_comment": "Each sweep = one full training run; `name` is the threshold cell key. FP8 on MI300X (CDNA3) uses quantization=nanoo_fp8. STEPS/GBS/NNODES in the name are labels: steps comes from training.steps, GBS = per_device_batch_size * total GPUs, NNODES from the cluster. enabled_sweep_list selects which sweeps to run.", + "sweeps": [ + { + "name": "NNODES=1,STEPS=30,PRECISION=BF16,BATCH=5,GBS=40,SEQLEN=8192", + "maxtext_overrides": { + "per_device_batch_size": 5, + "max_target_length": 8192, + "dtype": "bfloat16", + "weight_dtype": "bfloat16", + "quantization": "" + } + }, + { + "name": "NNODES=1,STEPS=30,PRECISION=FP8,BATCH=5,GBS=40,SEQLEN=8192", + "maxtext_overrides": { + "per_device_batch_size": 5, + "max_target_length": 8192, + "dtype": "bfloat16", + "weight_dtype": "bfloat16", + "quantization": "nanoo_fp8" + } + } + ], + "enabled_sweep_list": [ + "NNODES=1,STEPS=30,PRECISION=BF16,BATCH=5,GBS=40,SEQLEN=8192", + "NNODES=1,STEPS=30,PRECISION=FP8,BATCH=5,GBS=40,SEQLEN=8192" + ] + } +} diff --git a/cvs/input/config_file/training/jaxmaxtext/mi300x_jaxmaxtext_llama-3.3-70b_single_threshold.json b/cvs/input/config_file/training/jaxmaxtext/mi300x_jaxmaxtext_llama-3.3-70b_single_threshold.json new file mode 100644 index 000000000..109c8bcb7 --- /dev/null +++ b/cvs/input/config_file/training/jaxmaxtext/mi300x_jaxmaxtext_llama-3.3-70b_single_threshold.json @@ -0,0 +1,35 @@ +{ + "_comment": "Thresholds for Llama-3.3-70B on MI300X single-node, keyed by sweep name. Gated metrics (min/max) drive PASS/FAIL; kind='info' metrics always PASS (record-only) but retain a default `value` as a placeholder to calibrate later. Requires enforce_thresholds=true.", + + "NNODES=1,STEPS=30,PRECISION=BF16,BATCH=5,GBS=40,SEQLEN=8192": { + "training.tflops_per_sec_per_gpu": { "kind": "min", "value": 260.0 }, + "training.tokens_per_sec_per_gpu": { "kind": "min", "value": 962.0 }, + "training.tokens_per_sec_total": { "kind": "info", "value": 0 }, + "training.scaling_efficiency_pct": { "kind": "info", "value": 80.0 }, + "training.step_time_seconds": { "kind": "info", "value": 3600.0 }, + "training.step_time_mean_ms": { "kind": "info", "value": 3600000.0 }, + "training.step_time_p50_ms": { "kind": "info", "value": 3600000.0 }, + "training.step_time_p95_ms": { "kind": "info", "value": 3600000.0 }, + "training.final_loss": { "kind": "max", "value": 15.0 }, + "training.loss_decreased": { "kind": "min", "value": 1 }, + "training.eval_loss": { "kind": "info", "value": 100.0 }, + "training.steps_to_target": { "kind": "info", "value": 1000000 }, + "training.time_to_target_seconds": { "kind": "info", "value": 1000000.0 } + }, + + "NNODES=1,STEPS=30,PRECISION=FP8,BATCH=5,GBS=40,SEQLEN=8192": { + "training.tflops_per_sec_per_gpu": { "kind": "min", "value": 300.0 }, + "training.tokens_per_sec_per_gpu": { "kind": "min", "value": 1472.0 }, + "training.tokens_per_sec_total": { "kind": "info", "value": 0 }, + "training.scaling_efficiency_pct": { "kind": "info", "value": 80.0 }, + "training.step_time_seconds": { "kind": "info", "value": 3600.0 }, + "training.step_time_mean_ms": { "kind": "info", "value": 3600000.0 }, + "training.step_time_p50_ms": { "kind": "info", "value": 3600000.0 }, + "training.step_time_p95_ms": { "kind": "info", "value": 3600000.0 }, + "training.final_loss": { "kind": "max", "value": 15.0 }, + "training.loss_decreased": { "kind": "min", "value": 1 }, + "training.eval_loss": { "kind": "info", "value": 100.0 }, + "training.steps_to_target": { "kind": "info", "value": 1000000 }, + "training.time_to_target_seconds": { "kind": "info", "value": 1000000.0 } + } +} diff --git a/cvs/input/config_file/training/jaxmaxtext/mi325x_jaxmaxtext_llama-3.3-70b_distributed.json b/cvs/input/config_file/training/jaxmaxtext/mi325x_jaxmaxtext_llama-3.3-70b_distributed.json new file mode 100644 index 000000000..3da76272e --- /dev/null +++ b/cvs/input/config_file/training/jaxmaxtext/mi325x_jaxmaxtext_llama-3.3-70b_distributed.json @@ -0,0 +1,229 @@ +{ + "schema_version": 1, + "framework": "jaxmaxtext", + "gpu_arch": "mi325x", + "enforce_thresholds": true, + "threshold_json": "mi325x_jaxmaxtext_llama-3.3-70b_distributed_threshold.json", + + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "{shared_fs}/cache/maxtext", + "log_dir": "{shared_fs}/LOGS/jaxmaxtext", + "hf_token_file": "{shared_fs}/.hf_token" + }, + + "model": { + "id": "llama3.3-70b", + "remote": 0, + "precision": "bfloat16" + }, + + "container": { + "lifetime": "per_run", + "name": "rocm-jaxmaxtext-llama3.3-70b", + "image": "rocm/jax-training:maxtext-v26.4", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm-size": "256G", + "ulimit": ["nofile=65535:65535"], + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/dev/infiniband:/dev/infiniband", + "/usr/local/lib/libbnxt_re-rdmav34.so:/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so.host", + "/lib/libibverbs.d:/lib/libibverbs.d", + "/tmp/{user-id}/jax/TRAINING_LOGS:/workspace/maxtext/output" + ] + } + } + }, + + "__maxtext_version<=26.3__train_script": "/workspace/maxtext/src/MaxText/train.py", + "__maxtext_version>=26.4__train_script": "/workspace/maxtext/src/maxtext/trainers/pre_train/train.py", + "training": { + "distributed": true, + "gpus_per_node": 8, + "steps": 30, + "enable_checkpointing": false, + "train_script_paths": [ + "/workspace/maxtext/src/maxtext/trainers/pre_train/train.py", + "/workspace/maxtext/src/MaxText/train.py" + ], + + "maxtext_config": { + "base_config": "base.yml", + "hardware": "gpu", + "attention": "cudnn_flash_te", + "dtype": "bfloat16", + "dataset_type": "synthetic", + "remat_policy": "full", + "use_iota_embed": true, + "scan_layers": true, + "per_device_batch_size": 3, + "max_target_length": 8192, + "async_checkpointing": false, + "quantization": "", + "weight_dtype": "bfloat16", + "shardy": false, + "logits_dot_in_fp32": false, + "megablox": false, + "packing": true, + "enable_goodput_recording": false, + "monitor_goodput": false, + "optimizer_memory_host_offload": false, + "param_scan_axis": 1, + "ici_fsdp_parallelism": 8, + "ici_data_parallelism": 1, + "ici_sequence_parallelism": 1, + "ici_tensor_parallelism": 1, + "ici_pipeline_parallelism": 1, + "dcn_data_parallelism": -1, + "dcn_fsdp_parallelism": 1, + "dcn_pipeline_parallelism": 1, + "dcn_tensor_parallelism": 1, + "dcn_sequence_parallelism": 1, + "max_segments_per_seq": 32, + "skip_first_n_steps_for_profiler": 3, + "eval_interval": -1, + "eval_steps": -1 + }, + + "tokenizer": { + "hf_model_id": "NousResearch/Meta-Llama-3-70B", + "tokenizer_path": "{paths.models_dir}/Meta-Llama-70-B" + }, + + "nic_type": "thor2", + + "rdma_lib": { + "host_source_file": "/usr/local/lib/libbnxt_re-rdmav34.so", + "container_mount_file": "/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so.host", + "container_dest_file": "/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so" + }, + + "_NCCL_IB_DISABLE_comment": "IB/RoCE over Broadcom bnxt_re segfaults in RCCL QP setup on these nodes (crashes on the first inter-node channel, independent of GDR / matched rdma-core / channel-count). TCP works. Remove this once the image ships an RCCL build validated against the bnxt_re RoCE stack.", + "env_vars": { + "NNODES": "2", + "GPU_MAX_HW_QUEUES": "2", + "HSA_FORCE_FINE_GRAIN_PCIE": "1", + "HIP_FORCE_DEV_KERNARG": "1", + "XLA_PYTHON_CLIENT_MEM_FRACTION": "0.93", + "NCCL_DEBUG": "ERROR", + "NCCL_IB_DISABLE": "1", + "NCCL_PROTO": "Simple", + "NCCL_IB_TC": "41", + "NCCL_IB_SL": "0", + "NCCL_IB_GID_INDEX": "3", + "NCCL_CHECKS_DISABLE": "1", + "NCCL_CROSS_NIC": "0", + "NVTE_ALLOW_NONDETERMINISTIC_ALGO": "1", + "NVTE_USE_HIPBLASLT": "1", + "NVTE_FUSED_ATTN": "1", + "NVTE_CK_USES_BWD_V3": "1", + "NVTE_CK_USES_FWD_V3": "1", + "NVTE_CK_IS_V3_ATOMIC_FP32": "0", + "NVTE_CK_HOW_V3_BF16_CVT": "2", + "NVTE_FUSED_ATTN_CK": "1", + "NVTE_FUSED_ATTN_AOTRITON": "0" + }, + + "xla_flags": { + "xla_gpu_enable_latency_hiding_scheduler": "True", + "xla_gpu_enable_triton_gemm": "False", + "xla_gpu_memory_limit_slop_factor": "95", + "xla_gpu_enable_command_buffer": "", + "xla_gpu_enable_cublaslt": "True", + "xla_gpu_autotune_level": "0", + "xla_gpu_enable_reduce_scatter_combine_by_dim": "false", + "xla_gpu_reduce_scatter_combine_threshold_bytes": "8589934592", + "xla_gpu_all_reduce_combine_threshold_bytes": "8589934592", + "xla_gpu_all_gather_combine_threshold_bytes": "8589934592", + "xla_gpu_enable_all_gather_combine_by_dim": "FALSE" + }, + + "_nccl_comment": "Cluster-specific RDMA/NIC devices. Replace every '' with your node's values (see the sibling _example_* entries); distributed runs hard-exit at config load until you do. Discover them with `ibv_devices` (HCAs) and `ip -br link` (host interface).", + "nccl": { + "_example_ib_hca_list": "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7", + "ib_hca_list": "", + "_example_ib_hca": "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7", + "ib_hca": "", + "_example_socket_ifname": "eno0", + "socket_ifname": "", + "_example_gloo_socket_ifname": "eno0", + "gloo_socket_ifname": "", + "ib_tc": "41", + "ib_sl": "0", + "ib_gid_index": "3" + }, + + "jax_distributed": { + "coordinator_ip": "auto", + "coordinator_port": "12346", + "initialization_timeout_seconds": "1800", + "heartbeat_timeout_seconds": "900" + }, + + "_scaling_baseline_comment": "1-node total tokens/sec baseline for scaling-efficiency %. Set to 0.0 = disabled (record-only). Populate from a prior MI325X single-node run (tok/s/GPU * 8) to enable the metric.", + "scaling_baseline": { + "tokens_per_sec_total": 394000.0, + "num_nodes": 1 + }, + + "_convergence_comment": "to enable validation loss set eval_interval > 0 (and eval_steps) in maxtext_config; this needs a validation dataset (eval_split/eval_dataset_name). target_value <= 0 disables convergence (record-only). target_metric 'auto' uses eval_loss when eval runs, else training loss.", + "convergence": { + "target_metric": "auto", + "target_value": 10.0 + }, + + "_loss_curve_comment": "Row 32: sample training loss every `sample_every` steps (plus milestone_steps) and pass when the least-squares slope < max_slope. enforce=false makes it record-only.", + "loss_curve": { + "sample_every": 10, + "milestone_steps": [100, 500, 1000, 5000], + "max_slope": 0.0, + "enforce": true + }, + + "_error_patterns_comment": "Regexes (name -> pattern) scanned in each node's training.log during polling; a match fails that sweep's training_run with the matched name. Add/remove entries as you find new error signatures. Remove this block to use the built-in defaults. Escape backslashes per JSON (e.g. two backslashes for a regex \\d).", + "error_patterns": { + "NCCL ERROR": "NCCL ERROR|NCCL timeout|local work queue catastrophic error", + "GPU HW ERROR": "HW Exception by GPU|GPU Hang|Uncorrectable error|GPU Reset", + "AssertionError": "AssertionError|ValueError:|JaxStackTrace|During handling of the above exception|triggered the following exception", + "rocm Err": "FAILED_PRECONDITION: No visible GPU devices|failed call to hipInit: HIP_ERROR_NoDevice|librocm reported version is: NOT_FOUND", + "python err": "ModuleNotFoundError: No module named|Fatal Python error:", + "tensorflow": "tensorflow.CoordinationServiceError|tensorflow.BarrierError|CoordinationServiceError", + "resource": "RESOURCE_EXHAUSTED: Out of memory|failed: RESOURCE_EXHAUSTED", + "segfault": "Segmentation fault|SIGSEGV|core dumped|std::bad_alloc" + }, + + "_sweeps_comment": "Each sweep = one full training run with per-run maxtext overrides; `name` is the threshold cell key. BF16 uses the base maxtext_config as-is. FP8 on MI300X/MI325X (CDNA3) must use quantization=nanoo_fp8 (the plain 'fp8' value is the NVIDIA path, also used on MI355X/MI350X). weight_dtype stays bfloat16 (master weights). enabled_sweep_list selects which sweeps to run.", + "sweeps": [ + { + "name": "NNODES=2,STEPS=30,PRECISION=BF16,BATCH=3,GBS=48,SEQLEN=8192", + "maxtext_overrides": { + "per_device_batch_size": 3, + "max_target_length": 8192, + "dtype": "bfloat16", + "weight_dtype": "bfloat16", + "quantization": "" + } + }, + { + "name": "NNODES=2,STEPS=30,PRECISION=FP8,BATCH=3,GBS=48,SEQLEN=8192", + "maxtext_overrides": { + "per_device_batch_size": 3, + "max_target_length": 8192, + "dtype": "bfloat16", + "weight_dtype": "bfloat16", + "quantization": "nanoo_fp8" + } + } + ], + "enabled_sweep_list": [ + "NNODES=2,STEPS=30,PRECISION=BF16,BATCH=3,GBS=48,SEQLEN=8192", + "NNODES=2,STEPS=30,PRECISION=FP8,BATCH=3,GBS=48,SEQLEN=8192" + ] + } +} diff --git a/cvs/input/config_file/training/jaxmaxtext/mi325x_jaxmaxtext_llama-3.3-70b_distributed_threshold.json b/cvs/input/config_file/training/jaxmaxtext/mi325x_jaxmaxtext_llama-3.3-70b_distributed_threshold.json new file mode 100644 index 000000000..445cb08b8 --- /dev/null +++ b/cvs/input/config_file/training/jaxmaxtext/mi325x_jaxmaxtext_llama-3.3-70b_distributed_threshold.json @@ -0,0 +1,35 @@ +{ + "_comment": "Thresholds for Llama-3.3-70B on MI325X distributed, keyed by sweep name. Gated metrics (min/max) drive PASS/FAIL; kind='info' metrics always PASS (record-only) but retain a default `value` as a placeholder to calibrate later (flip kind to min/max once reference values are known). Requires enforce_thresholds=true.", + + "NNODES=2,STEPS=30,PRECISION=BF16,BATCH=3,GBS=48,SEQLEN=8192": { + "training.tflops_per_sec_per_gpu": { "kind": "min", "value": 260.0 }, + "training.tokens_per_sec_per_gpu": { "kind": "min", "value": 1217.0 }, + "training.tokens_per_sec_total": { "kind": "info", "value": 0 }, + "training.scaling_efficiency_pct": { "kind": "info", "value": 80.0 }, + "training.step_time_seconds": { "kind": "info", "value": 3600.0 }, + "training.step_time_mean_ms": { "kind": "info", "value": 3600000.0 }, + "training.step_time_p50_ms": { "kind": "info", "value": 3600000.0 }, + "training.step_time_p95_ms": { "kind": "info", "value": 3600000.0 }, + "training.final_loss": { "kind": "max", "value": 15.0 }, + "training.loss_decreased": { "kind": "min", "value": 1 }, + "training.eval_loss": { "kind": "info", "value": 100.0 }, + "training.steps_to_target": { "kind": "info", "value": 1000000 }, + "training.time_to_target_seconds": { "kind": "info", "value": 1000000.0 } + }, + + "NNODES=2,STEPS=30,PRECISION=FP8,BATCH=3,GBS=48,SEQLEN=8192": { + "training.tflops_per_sec_per_gpu": { "kind": "min", "value": 300.0 }, + "training.tokens_per_sec_per_gpu": { "kind": "min", "value": 1836.0 }, + "training.tokens_per_sec_total": { "kind": "info", "value": 0 }, + "training.scaling_efficiency_pct": { "kind": "info", "value": 80.0 }, + "training.step_time_seconds": { "kind": "info", "value": 3600.0 }, + "training.step_time_mean_ms": { "kind": "info", "value": 3600000.0 }, + "training.step_time_p50_ms": { "kind": "info", "value": 3600000.0 }, + "training.step_time_p95_ms": { "kind": "info", "value": 3600000.0 }, + "training.final_loss": { "kind": "max", "value": 15.0 }, + "training.loss_decreased": { "kind": "min", "value": 1 }, + "training.eval_loss": { "kind": "info", "value": 100.0 }, + "training.steps_to_target": { "kind": "info", "value": 1000000 }, + "training.time_to_target_seconds": { "kind": "info", "value": 1000000.0 } + } +} diff --git a/cvs/input/config_file/training/megatron/README.md b/cvs/input/config_file/training/megatron/README.md new file mode 100644 index 000000000..cb025d150 --- /dev/null +++ b/cvs/input/config_file/training/megatron/README.md @@ -0,0 +1,185 @@ +# Megatron Training — Config and Threshold Files + +This folder holds the input files for the `megatron_single` / `megatron_distributed` suites (see `cvs/tests/training/megatron/README.md` for how to run them). Each config file has a sibling threshold file referenced by its `threshold_json` field. One config = one GPU arch + mode (single or distributed). + +Keys prefixed with `_` (e.g. `_scaling_baseline_comment`) are inline comments and are ignored by the loader. + +## File Inventory + +| Config | Threshold | Arch / mode | +|---|---|---| +| `mi325x_megatron_llama-3.1-8b_single.json` | `mi325x_megatron_llama-3.1-8b_single_threshold.json` | MI325X, single-node | +| `mi325x_megatron_llama-3.3-70b_single.json` | `mi325x_megatron_llama-3.3-70b_single_threshold.json` | MI325X, single-node | +| `mi325x_megatron_llama-3.3-70b_distributed.json` | `mi325x_megatron_llama-3.3-70b_distributed_threshold.json` | MI325X, distributed | +| `mi325x_megatron_deepseek-v2-lite_single.json` | `mi325x_megatron_deepseek-v2-lite_single_threshold.json` | MI325X, single-node | + +Add analogous config + threshold pairs for other archs (e.g. MI355X) as needed. + +## What You MUST Change for Your Cluster + +Start from the config closest to your target arch/mode and edit these: + +| Where | Field | Change to | +|---|---|---| +| `container.image` | Docker image | Your Megatron-LM ROCm image tag accessible on all nodes | +| `container.name` | Container name | Any unique name (optional) | +| `config.hf_token_file` | HF token path | Location of your Hugging Face token file on the nodes | +| `config.log_dir` / `scripts_dir` / `data_cache_dir` | Paths | Replace `{user-id}` with your actual username | +| `config.megatron_root` | Megatron path | In-container path to Megatron-LM (default `/workspace/Megatron-LM`) | +| `config.nnodes` | Node count | Number of nodes in your cluster (**distributed only**) | +| `config.master_address` | Head node IP | IP of the head node (**distributed only**) | +| `config.nic_type` | NIC family | `thor2` (Broadcom) or your NIC type (**distributed only**) | +| `config.nccl_ib_hca_list` / `nccl_ib_hca` | RDMA HCA devices | Your nodes' RDMA device names (e.g. `bnxt_re0,...,bnxt_re7`) (**distributed only**) | +| `config.nccl_socket_ifname` / `gloo_socket_ifname` | Control NIC | Your management interface name (e.g. `ensf1np1`) (**distributed only**) | +| `scaling_baseline.tokens_per_sec_total` | 1-node baseline | Your measured single-node total tok/s (`tok/s/GPU × 8`); `0.0` disables scaling efficiency (**distributed only**) | +| Threshold JSON gated values | Thresholds | Calibrated PASS/FAIL bounds for your hardware | +| Cluster file | Node IPs | Your node IPs (first entry is the coordinator node) | + +Also set `sweep.runs` to the combo(s) you want to run, and `enforce_thresholds` to `true` for real PASS/FAIL or `false` for record-only. + +## Placeholder Substitution + +Configs use `{user-id}` in path fields, resolved at load time to the cluster username (or local OS user as fallback). Unresolved `` placeholders cause a hard exit at startup. + +## Config Structure + +Top-level fields: + +| Field | Meaning | +|---|---| +| `schema_version` | Always `1` | +| `framework` | `megatron_single` (single-node) or `megatron_distributed` (multi-node) | +| `gpu_arch` | `MI325X` / `MI300X` etc. — labels the run, informational | +| `enforce_thresholds` | `true` = metrics gate PASS/FAIL; `false` = record-only | +| `threshold_json` | Sibling threshold filename; resolved next to the config | +| `scaling_baseline` | 1-node baseline for scaling efficiency % (distributed only) | +| `config` | Runtime, paths, NCCL, and NIC settings | +| `model_params` | Model architecture and default hyperparameters | +| `container` | Docker container settings | +| `sweep` | Training combinations and the ordered run list | + +### `config` block + +| Field | Default | Description | +|---|---|---| +| `hf_token_file` | `/home/{user-id}/.hf_token` | Hugging Face access token file path | +| `log_dir` | `/home/{user-id}/LOGS/megatron` | Training log output directory | +| `scripts_dir` | `/home/{user-id}/SCRIPTS/megatron` | Generated per-node wrapper scripts directory | +| `data_cache_dir` | `/home/{user-id}/cache` | Tokenizer and dataset cache directory | +| `rocm_dir` | `""` | ROCm path; empty string triggers auto-detection | +| `megatron_root` | `/workspace/Megatron-LM` | Megatron-LM path inside the container | +| `training_iterations` | `"10"` | Training iterations per combo | +| `nnodes` | `"1"` / `` | Node count; must match cluster file | +| `master_address` | `"127.0.0.1"` / `` | Head-node IP for distributed coordination | +| `nic_type` | `"thor2"` | NIC family; `thor2` triggers Broadcom RDMA-lib copy | +| `nccl_ib_hca_list` / `nccl_ib_hca` | `` | Comma-separated RDMA HCA list | +| `nccl_socket_ifname` / `gloo_socket_ifname` | `"ensf1np1"` | Control-plane interface name | +| `hca_id_pattern` | `"bnxt_\|rocep"` | `\|`-separated NIC prefixes for ibv_devinfo validation | +| `nccl_ib_gid_index` | `"3"` | GID index for RoCE (standard `"3"` for Broadcom) | +| `nccl_debug` | `"ERROR"` | NCCL log verbosity (`"ERROR"`, `"WARN"`, `"INFO"`, `"TRACE"`) | +| `verify_network_errors` | `"False"` | `"True"` to compare RDMA/ethtool error counters before and after training | + +### `model_params` block + +Defaults applied to every sweep combo; individual combos override them. + +| Field | Description | +|---|---| +| `model_name` | Friendly name used in log paths and labels | +| `tokenizer_model` | Hugging Face repo ID (e.g. `"meta-llama/Llama-3.1-8B"`) | +| `model_size` | Parameter count in billions (e.g. `"8"`, `"70"`) | +| `sequence_length` | Sequence length in tokens | +| `micro_batch_size` | Default micro-batch size (overridden by sweep) | +| `global_batch_size` | Default global batch size (overridden by sweep) | +| `tensor_parallelism` | Tensor parallel degree (TP) | +| `pipeline_parallelism` | Pipeline parallel degree (PP) | +| `recompute` | Activation recompute (`"0"` off, `"1"` on) | +| `fsdp` | Fully Sharded Data Parallel (`"0"` off, `"1"` on) | +| `precision` | Default precision; overridden by sweep (`"FP8"`, `"BF16"`, `"MXFP4"`, `"MXFP8"`) | + +### `container` block + +| Field | Description | +|---|---| +| `lifetime` | `"per_run"` — launched once per session, torn down after | +| `name` | Docker container name | +| `image` | **Required** — Docker image URI; replace `` | +| `runtime.args.volumes` | Host paths volume-mounted into the container | +| `runtime.args.devices` | Host devices exposed (`/dev/kfd`, `/dev/dri` for AMD GPUs) | + +Distributed configs additionally mount the Broadcom RDMA library and expose `/dev/infiniband/rdma_cm`: + +```json +"/usr/local/lib/libbnxt_re-rdmav34.so:/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so.host", +"/lib/libibverbs.d:/lib/libibverbs.d" +``` + +### `scaling_baseline` block — distributed only + +| Field | Description | +|---|---| +| `tokens_per_sec_total` | Single-node baseline total tok/s (`tok/s/GPU × 8`); `0.0` disables efficiency calculation | +| `num_nodes` | Number of nodes used for the baseline (typically `1`) | + +## Sweeps + +Each entry in `sweep.combinations` is one parametrized training run. `sweep.runs` is the ordered list of combo IDs to execute; omit it to run all combinations. + +```json +"sweep": { + "combinations": { + "llama3_1_8b-mi325-bs128-mbs4-fp8": { + "name": "llama3_1_8b_mbs4_gbs128_FP8", + "global_batch_size": "128", + "micro_batch_size": "4", + "precision": "FP8" + } + }, + "runs": ["llama3_1_8b-mi325-bs128-mbs4-fp8"] +} +``` + +| Combo field | Description | +|---|---| +| `name` | Human-readable label (used in reports) | +| `global_batch_size` | Global batch size for this combo | +| `micro_batch_size` | Micro-batch size for this combo | +| `precision` | Precision override (`"FP8"`, `"BF16"`, `"MXFP4"`, `"MXFP8"`) | + +Any key in a combo overrides the matching `model_params` field — adding a new sweep parameter (e.g. `tensor_parallelism`) requires only a config edit, no code change. + +## Threshold Files + +A threshold file maps each sweep combo (cell key) to per-metric pass/fail limits. A metric is gated only when `enforce_thresholds: true` and it has a numeric spec; otherwise it is recorded. + +Cell keys must match the format `MBS=,GBS=,PRECISION=`. + +Example cell: + +```json +"MBS=4,GBS=128,PRECISION=FP8": { + "training.throughput_per_gpu": { "kind": "min", "value": 100 }, + "training.tokens_per_gpu": { "kind": "min", "value": 1000 }, + "training.elapsed_time_per_iteration":{ "kind": "max", "value": 500 }, + "training.mem_usage": { "kind": "max", "value": 0.85 } +} +``` + +### Threshold kinds + +| Kind | Passes when | +|---|---| +| `min` | actual ≥ value | +| `max` | actual ≤ value | +| `min_ratio` | actual / reference ≥ value (needs `reference` key) | + +### Tracked metrics + +| Metric | Description | +|---|---| +| `training.throughput_per_gpu` | TFLOP/s per GPU | +| `training.tokens_per_gpu` | Tokens per GPU per second | +| `training.elapsed_time_per_iteration` | Wall time per training step (ms) | +| `training.mem_usage` | GPU memory usage | + +To start gating a metric: set a calibrated `value` and the appropriate `kind`. The cell key must match the combo's `MBS=`, `GBS=`, and `PRECISION=` values exactly, or the metric falls back to record-only. diff --git a/cvs/input/config_file/training/megatron/mi325x_megatron_deepseek-v2-lite_single.json b/cvs/input/config_file/training/megatron/mi325x_megatron_deepseek-v2-lite_single.json new file mode 100644 index 000000000..9c72c15d0 --- /dev/null +++ b/cvs/input/config_file/training/megatron/mi325x_megatron_deepseek-v2-lite_single.json @@ -0,0 +1,82 @@ +{ + "schema_version": 1, + "framework": "megatron_single", + "gpu_arch": "MI325X", + "enforce_thresholds": true, + "loss_curve": { + "sample_every": 10, + "milestone_steps": [100, 500, 1000, 5000], + "max_slope": 0.0, + "enforce": true + }, + "threshold_json": "mi325x_megatron_deepseek-v2-lite_single_threshold.json", + "config": { + "hf_token_file": "/home/{user-id}/.hf_token", + "log_dir": "/home/{user-id}/LOGS/megatron", + "scripts_dir": "/home/{user-id}/SCRIPTS/megatron", + "data_cache_dir": "/home/{user-id}/cache", + "rocm_dir": "", + "megatron_root": "/workspace/Megatron-LM", + "training_iterations": "10", + "nnodes": "1", + "nic_type": "thor2", + "nccl_socket_ifname": "ensf1np1", + "gloo_socket_ifname": "ensf1np1", + "nccl_ib_gid_index": "3", + "nccl_debug": "ERROR", + "master_address": "127.0.0.1", + "verify_network_errors": "False" + }, + "model_params": { + "model_name": "deepseek_v2_lite", + "tokenizer_model": "deepseek-ai/DeepSeek-V2-Lite", + "model_size": "16", + "sequence_length": "4096", + "recompute": "0", + "fsdp": "0", + "tensor_parallelism": "1", + "pipeline_parallelism": "1", + "precision": "BF16" + }, + "container": { + "lifetime": "per_run", + "name": "megatron_deepseek_v2_lite_single", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/dev/infiniband:/dev/infiniband" + ], + "devices": [ + "/dev/kfd", + "/dev/dri" + ] + } + } + }, + "sweep": { + "combinations": { + "deepseek_v2_lite-mi325x-bs128-mbs4-bf16": { + "name": "deepseek_v2_lite_mbs4_gbs128_BF16", + "global_batch_size": "128", + "micro_batch_size": "4", + "precision": "BF16" + }, + "deepseek_v2_lite-mi325x-bs128-mbs4-fp8": { + "name": "deepseek_v2_lite_mbs4_gbs128_FP8", + "global_batch_size": "128", + "micro_batch_size": "4", + "precision": "FP8" + } + }, + "runs": [ + "deepseek_v2_lite-mi325x-bs128-mbs4-bf16", + "deepseek_v2_lite-mi325x-bs128-mbs4-fp8" + ] + } +} diff --git a/cvs/input/config_file/training/megatron/mi325x_megatron_deepseek-v2-lite_single_threshold.json b/cvs/input/config_file/training/megatron/mi325x_megatron_deepseek-v2-lite_single_threshold.json new file mode 100644 index 000000000..e199fcc8d --- /dev/null +++ b/cvs/input/config_file/training/megatron/mi325x_megatron_deepseek-v2-lite_single_threshold.json @@ -0,0 +1,39 @@ +{ + "_comment": "DeepSeek V2 Lite single-node thresholds for MI325X Megatron. Cell keys must match MegatronVariantConfig.cell_key() format: MBS=,GBS=,PRECISION=.", + "MBS=4,GBS=128,PRECISION=BF16": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.elapsed_time_per_iteration": { + "kind": "max", + "value": 500 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 1000 + }, + "training.mem_usage": { + "kind": "max", + "value": 0.85 + } + }, + "MBS=4,GBS=128,PRECISION=FP8": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.elapsed_time_per_iteration": { + "kind": "max", + "value": 500 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 1000 + }, + "training.mem_usage": { + "kind": "max", + "value": 0.85 + } + } +} diff --git a/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.1-8b_single.json b/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.1-8b_single.json new file mode 100644 index 000000000..32d54690a --- /dev/null +++ b/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.1-8b_single.json @@ -0,0 +1,96 @@ +{ + "schema_version": 1, + "framework": "megatron_single", + "gpu_arch": "MI325X", + "enforce_thresholds": true, + "loss_curve": { + "sample_every": 10, + "milestone_steps": [100, 500, 1000, 5000], + "max_slope": 0.0, + "enforce": true + }, + "threshold_json": "mi325x_megatron_llama-3.1-8b_single_threshold.json", + "config": { + "hf_token_file": "/home/{user-id}/.hf_token", + "log_dir": "/home/{user-id}/LOGS/megatron", + "scripts_dir": "/home/{user-id}/SCRIPTS/megatron", + "data_cache_dir": "/home/{user-id}/cache", + "rocm_dir": "", + "megatron_root": "/workspace/Megatron-LM", + "training_iterations": "10", + "nnodes": "1", + "nic_type": "thor2", + "nccl_socket_ifname": "ensf1np1", + "gloo_socket_ifname": "ensf1np1", + "nccl_ib_gid_index": "3", + "nccl_debug": "ERROR", + "master_address": "127.0.0.1", + "verify_network_errors": "False" + }, + "model_params": { + "model_name": "llama3.1_8B", + "tokenizer_model": "meta-llama/Llama-3.1-8B", + "model_size": "8", + "sequence_length": "8192", + "recompute": "0", + "fsdp": "0", + "tensor_parallelism": "1", + "pipeline_parallelism": "1", + "precision": "FP8" + }, + "container": { + "lifetime": "per_run", + "name": "megatron_llama3_1_8b_single", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/dev/infiniband:/dev/infiniband" + ], + "devices": [ + "/dev/kfd", + "/dev/dri" + ] + } + } + }, + "sweep": { + "combinations": { + "llama3_1_8b-mi325-bs128-mbs4-fp8": { + "name": "llama3_1_8b_mbs4_gbs128_FP8", + "global_batch_size": "128", + "micro_batch_size": "4", + "precision": "FP8" + }, + "llama3_1_8b-mi325-bs128-mbs4-bf16": { + "name": "llama3_1_8b_mbs4_gbs128_BF16", + "global_batch_size": "128", + "micro_batch_size": "4", + "precision": "BF16" + }, + "llama3_1_8b-mi325-bs128-mbs4-mxfp4": { + "name": "llama3_1_8b_mbs4_gbs128_MXFP4", + "global_batch_size": "128", + "micro_batch_size": "4", + "precision": "MXFP4" + }, + "llama3_1_8b-mi325-bs128-mbs4-mxfp8": { + "name": "llama3_1_8b_mbs4_gbs128_MXFP8", + "global_batch_size": "128", + "micro_batch_size": "4", + "precision": "MXFP8" + } + }, + "runs": [ + "llama3_1_8b-mi325-bs128-mbs4-fp8", + "llama3_1_8b-mi325-bs128-mbs4-bf16", + "llama3_1_8b-mi325-bs128-mbs4-mxfp4", + "llama3_1_8b-mi325-bs128-mbs4-mxfp8" + ] + } +} diff --git a/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.1-8b_single_threshold.json b/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.1-8b_single_threshold.json new file mode 100644 index 000000000..ba32d6e67 --- /dev/null +++ b/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.1-8b_single_threshold.json @@ -0,0 +1,75 @@ +{ + "_comment": "Llama 3.1 8B single-node thresholds for MI325X Megatron. Cell keys must match MegatronVariantConfig.cell_key() format: MBS=,GBS=,PRECISION=.", + "MBS=4,GBS=128,PRECISION=FP8": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.elapsed_time_per_iteration": { + "kind": "max", + "value": 500 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 1000 + }, + "training.mem_usage": { + "kind": "max", + "value": 0.85 + } + }, + "MBS=4,GBS=128,PRECISION=BF16": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.elapsed_time_per_iteration": { + "kind": "max", + "value": 500 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 1000 + }, + "training.mem_usage": { + "kind": "max", + "value": 0.85 + } + }, + "MBS=4,GBS=128,PRECISION=MXFP4": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.elapsed_time_per_iteration": { + "kind": "max", + "value": 500 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 1000 + }, + "training.mem_usage": { + "kind": "max", + "value": 0.85 + } + }, + "MBS=4,GBS=128,PRECISION=MXFP8": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.elapsed_time_per_iteration": { + "kind": "max", + "value": 500 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 1000 + }, + "training.mem_usage": { + "kind": "max", + "value": 0.85 + } + } +} diff --git a/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.3-70b_distributed.json b/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.3-70b_distributed.json new file mode 100644 index 000000000..4a93829e9 --- /dev/null +++ b/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.3-70b_distributed.json @@ -0,0 +1,105 @@ +{ + "schema_version": 1, + "framework": "megatron_distributed", + "gpu_arch": "MI325X", + "enforce_thresholds": true, + "loss_curve": { + "sample_every": 10, + "milestone_steps": [100, 500, 1000, 5000], + "max_slope": 0.0, + "enforce": true + }, + "threshold_json": "mi325x_megatron_llama-3.3-70b_distributed_threshold.json", + "_scaling_baseline_comment": "Single-node baseline for scaling-efficiency %. tokens_per_sec_total=0.0 means disabled (record-only). Populate from this run's tok/s/GPU * 8 to use as reference for multi-node comparisons.", + "scaling_baseline": { + "tokens_per_sec_total": 0.0, + "num_nodes": 1 + }, + "config": { + "hf_token_file": "/home/{user-id}/.hf_token", + "log_dir": "/home/{user-id}/LOGS/megatron", + "scripts_dir": "/home/{user-id}/SCRIPTS/megatron", + "data_cache_dir": "/home/{user-id}/cache", + "rocm_dir": "", + "megatron_root": "/workspace/Megatron-LM", + "training_iterations": "10", + "nnodes": "", + "master_address": "", + "nic_type": "", + "nccl_ib_hca_list": "", + "nccl_ib_hca": "", + "nccl_socket_ifname": "", + "gloo_socket_ifname": "", + "hca_id_pattern": "bnxt_|rocep", + "nccl_ib_gid_index": "3", + "nccl_debug": "ERROR", + "verify_network_errors": "True" + }, + "model_params": { + "model_name": "llama3.3_70B", + "tokenizer_model": "meta-llama/Llama-3.3-70B-Instruct", + "model_size": "70", + "sequence_length": "8192", + "recompute": "0", + "fsdp": "0", + "tensor_parallelism": "8", + "pipeline_parallelism": "1", + "precision": "FP8" + }, + "container": { + "lifetime": "per_run", + "name": "megatron_llama3_3_70b_distributed", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/dev/infiniband:/dev/infiniband", + "/usr/local/lib/libbnxt_re-rdmav34.so:/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so.host", + "/lib/libibverbs.d:/lib/libibverbs.d" + ], + "devices": [ + "/dev/kfd", + "/dev/dri", + "/dev/infiniband/rdma_cm" + ] + } + } + }, + "sweep": { + "combinations": { + "llama3_3_70b-mi325-bs64-mbs1-fp8": { + "name": "llama3_3_70b_mbs1_gbs64_FP8", + "global_batch_size": "64", + "micro_batch_size": "1", + "precision": "FP8", + "result_dict": { + "throughput_per_gpu": "100", + "elapsed_time_per_iteration": "500", + "tokens_per_gpu": "1000", + "mem_usage": "0" + } + }, + "llama3_3_70b-mi325-bs64-mbs1-bf16": { + "name": "llama3_3_70b_mbs1_gbs64_BF16", + "global_batch_size": "64", + "micro_batch_size": "1", + "precision": "BF16", + "result_dict": { + "throughput_per_gpu": "100", + "elapsed_time_per_iteration": "500", + "tokens_per_gpu": "1000", + "mem_usage": "0" + } + } + }, + "runs": [ + "llama3_3_70b-mi325-bs64-mbs1-fp8", + "llama3_3_70b-mi325-bs64-mbs1-bf16" + ] + } +} diff --git a/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.3-70b_distributed_threshold.json b/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.3-70b_distributed_threshold.json new file mode 100644 index 000000000..e0ed8edd5 --- /dev/null +++ b/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.3-70b_distributed_threshold.json @@ -0,0 +1,47 @@ +{ + "_comment": "Llama 3.3 70B distributed thresholds for MI325X Megatron. Cell keys must match MegatronVariantConfig.cell_key() format: MBS=,GBS=,PRECISION=.", + "MBS=1,GBS=64,PRECISION=BF16": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.elapsed_time_per_iteration": { + "kind": "max", + "value": 500 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 1000 + }, + "training.mem_usage": { + "kind": "max", + "value": 0.85 + }, + "training.scaling_efficiency_pct": { + "kind": "min", + "value": 85.0 + } + }, + "MBS=1,GBS=64,PRECISION=FP8": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.elapsed_time_per_iteration": { + "kind": "max", + "value": 500 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 1000 + }, + "training.mem_usage": { + "kind": "max", + "value": 0.85 + }, + "training.scaling_efficiency_pct": { + "kind": "min", + "value": 85.0 + } + } +} diff --git a/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.3-70b_single.json b/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.3-70b_single.json new file mode 100644 index 000000000..1ca887b76 --- /dev/null +++ b/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.3-70b_single.json @@ -0,0 +1,82 @@ +{ + "schema_version": 1, + "framework": "megatron_single", + "gpu_arch": "MI325X", + "enforce_thresholds": true, + "loss_curve": { + "sample_every": 10, + "milestone_steps": [100, 500, 1000, 5000], + "max_slope": 0.0, + "enforce": true + }, + "threshold_json": "mi325x_megatron_llama-3.3-70b_single_threshold.json", + "config": { + "hf_token_file": "/home/{user-id}/.hf_token", + "log_dir": "/home/{user-id}/LOGS/megatron", + "scripts_dir": "/home/{user-id}/SCRIPTS/megatron", + "data_cache_dir": "/home/{user-id}/cache", + "rocm_dir": "", + "megatron_root": "/workspace/Megatron-LM", + "training_iterations": "10", + "nnodes": "1", + "nic_type": "thor2", + "nccl_socket_ifname": "ensf1np1", + "gloo_socket_ifname": "ensf1np1", + "nccl_ib_gid_index": "3", + "nccl_debug": "ERROR", + "master_address": "127.0.0.1", + "verify_network_errors": "False" + }, + "model_params": { + "model_name": "llama3.3_70B", + "tokenizer_model": "meta-llama/Llama-3.3-70B-Instruct", + "model_size": "70", + "sequence_length": "8192", + "recompute": "0", + "fsdp": "0", + "tensor_parallelism": "8", + "pipeline_parallelism": "1", + "precision": "FP8" + }, + "container": { + "lifetime": "per_run", + "name": "megatron_llama3_3_70b_single", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/dev/infiniband:/dev/infiniband" + ], + "devices": [ + "/dev/kfd", + "/dev/dri" + ] + } + } + }, + "sweep": { + "combinations": { + "llama3_3_70b-mi325-bs96-mbs3-fp8": { + "name": "llama3_3_70b_mbs3_gbs96_FP8", + "global_batch_size": "96", + "micro_batch_size": "3", + "precision": "FP8" + }, + "llama3_3_70b-mi325-bs96-mbs3-bf16": { + "name": "llama3_3_70b_mbs3_gbs96_BF16", + "global_batch_size": "96", + "micro_batch_size": "3", + "precision": "BF16" + } + }, + "runs": [ + "llama3_3_70b-mi325-bs96-mbs3-fp8", + "llama3_3_70b-mi325-bs96-mbs3-bf16" + ] + } +} diff --git a/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.3-70b_single_threshold.json b/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.3-70b_single_threshold.json new file mode 100644 index 000000000..c91bb4538 --- /dev/null +++ b/cvs/input/config_file/training/megatron/mi325x_megatron_llama-3.3-70b_single_threshold.json @@ -0,0 +1,39 @@ +{ + "_comment": "Llama 3.3 70B single-node thresholds for MI325X Megatron. Cell keys must match MegatronVariantConfig.cell_key() format: MBS=,GBS=,PRECISION=.", + "MBS=3,GBS=96,PRECISION=FP8": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.elapsed_time_per_iteration": { + "kind": "max", + "value": 500 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 1000 + }, + "training.mem_usage": { + "kind": "max", + "value": 0.85 + } + }, + "MBS=3,GBS=96,PRECISION=BF16": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.elapsed_time_per_iteration": { + "kind": "max", + "value": 500 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 1000 + }, + "training.mem_usage": { + "kind": "max", + "value": 0.85 + } + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_deepseek_v3_16b_single_config.json b/cvs/input/config_file/training/torchtitan/mi355_deepseek_v3_16b_single_config.json new file mode 100644 index 000000000..1545c2ab7 --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_deepseek_v3_16b_single_config.json @@ -0,0 +1,80 @@ +{ + "schema_version": 1, + "framework": "torchtitan_single", + "gpu_arch": "MI355", + "enforce_thresholds": false, + "threshold_json": "mi355_deepseek_v3_16b_single_threshold.json", + "scaling_baseline": { + "tokens_per_sec_total": 0.0, + "num_nodes": 1 + }, + "config": { + "hf_token_file": "/home/{user-id}/.hf_token", + "log_dir": "/home/{user-id}/LOGS/torchtitan", + "scripts_dir": "/home/{user-id}/SCRIPTS/torchtitan", + "data_cache_dir": "/home/{user-id}/cache", + "rocm_dir": "", + "torchtitan_root": "/workspace/Primus/third_party/torchtitan", + "training_iterations": "10", + "nnodes": "1", + "nic_type": "thor2", + "nccl_socket_ifname": "ensf1np1", + "gloo_socket_ifname": "ensf1np1", + "nccl_ib_gid_index": "3", + "nccl_debug": "ERROR", + "master_address": "127.0.0.1", + "verify_network_errors": "False", + "use_generated_config": "True" + }, + "model_params": { + "model_name": "deepseek_v3_16b", + "hf_model_name": "deepseek-ai/DeepSeek-V3-Base", + "sequence_length": "8192", + "dataset": "c4", + "lr": "1.5e-4", + "warmup_steps": "200", + "activation_checkpointing": "selective", + "compile": "false", + "tensor_parallel_degree": "1", + "pipeline_parallel_degree": "1", + "context_parallel_degree": "1", + "expert_parallel_degree": "1", + "enable_async_tensor_parallel": "false", + "precompute_float8_dynamic_scale_for_fsdp": "false", + "data_parallel_shard_degree": "8" + }, + "container": { + "lifetime": "per_run", + "name": "torchtitan_deepseek_v3_16b_single", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/dev/infiniband:/dev/infiniband" + ], + "devices": [ + "/dev/kfd", + "/dev/dri" + ] + } + } + }, + "sweep": { + "combinations": { + "deepseek_v3_16b-mi355-bs16-mbs1-bf16": { + "name": "deepseek_v3_16b_mbs1_gbs16_bf16", + "global_batch_size": "16", + "micro_batch_size": "1", + "precision": "bf16" + } + }, + "runs": [ + "deepseek_v3_16b-mi355-bs16-mbs1-bf16" + ] + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_deepseek_v3_16b_single_threshold.json b/cvs/input/config_file/training/torchtitan/mi355_deepseek_v3_16b_single_threshold.json new file mode 100644 index 000000000..ff738d3a2 --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_deepseek_v3_16b_single_threshold.json @@ -0,0 +1,20 @@ +{ + "_comment": "DeepSeek V3 16B single-node thresholds for MI355 TorchTitan.", + "_note": "Based on actual MI355 test results: BF16: 2,574 TPS. Threshold set at actual - 10%.", + "MBS=1,GBS=16,PRECISION=bf16": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 2317, + "_calculation": "2574 * 0.90 = 2316.6" + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 37072, + "_calculation": "2317 * 16 (batch size) = 37072" + }, + "training.loss": { + "kind": "max", + "value": 15.0 + } + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_llama3_1_405b_distributed_config.json b/cvs/input/config_file/training/torchtitan/mi355_llama3_1_405b_distributed_config.json new file mode 100644 index 000000000..d9f4c9599 --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_llama3_1_405b_distributed_config.json @@ -0,0 +1,81 @@ +{ + "schema_version": 1, + "framework": "torchtitan_distributed", + "gpu_arch": "MI355", + "enforce_thresholds": false, + "threshold_json": "mi355_llama3_1_405b_distributed_threshold.json", + "scaling_baseline": { + "tokens_per_sec_total": 0.0, + "num_nodes": 1 + }, + "config": { + "hf_token_file": "/home/{user-id}/.hf_token", + "log_dir": "/home/{user-id}/LOGS/torchtitan", + "scripts_dir": "/home/{user-id}/SCRIPTS/torchtitan", + "data_cache_dir": "/home/{user-id}/cache", + "rocm_dir": "", + "torchtitan_root": "/workspace/Primus/third_party/torchtitan", + "training_iterations": "10", + "nnodes": "8", + "nic_type": "thor2", + "nccl_socket_ifname": "ensf1np1", + "gloo_socket_ifname": "ensf1np1", + "nccl_ib_gid_index": "3", + "nccl_debug": "ERROR", + "master_address": "", + "verify_network_errors": "False", + "use_generated_config": "True" + }, + "model_params": { + "model_name": "llama3_1_405b", + "hf_model_name": "meta-llama/Llama-3.1-405B", + "sequence_length": "8192", + "dataset": "c4", + "lr": "1.5e-4", + "warmup_steps": "200", + "activation_checkpointing": "selective", + "compile": "false", + "tensor_parallel_degree": "8", + "pipeline_parallel_degree": "1", + "context_parallel_degree": "1", + "expert_parallel_degree": "1", + "enable_async_tensor_parallel": "false", + "precompute_float8_dynamic_scale_for_fsdp": "true", + "data_parallel_shard_degree": "8" + }, + "container": { + "lifetime": "per_run", + "name": "torchtitan_llama3_1_405b_distributed", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/dev/infiniband:/dev/infiniband" + ], + "devices": [ + "/dev/kfd", + "/dev/dri" + ] + } + } + }, + "sweep": { + "combinations": { + "llama3_1_405b-mi355-8n-bs256-mbs1-fp8": { + "name": "llama3_1_405b_8n_mbs1_gbs256_fp8", + "global_batch_size": "256", + "micro_batch_size": "1", + "precision": "fp8" + } + }, + "runs": [ + "llama3_1_405b-mi355-8n-bs256-mbs1-fp8" + ] + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_llama3_1_405b_distributed_threshold.json b/cvs/input/config_file/training/torchtitan/mi355_llama3_1_405b_distributed_threshold.json new file mode 100644 index 000000000..ae174e0d1 --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_llama3_1_405b_distributed_threshold.json @@ -0,0 +1,22 @@ +{ + "_comment": "Llama 3.1 405B 8-node distributed thresholds for MI355 TorchTitan.", + "_note": "Placeholder thresholds - update with actual 8-node test results.", + "MBS=1,GBS=256,PRECISION=fp8": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 25600 + }, + "training.loss": { + "kind": "max", + "value": 15.0 + }, + "training.scaling_efficiency_pct": { + "kind": "min", + "value": 80.0 + } + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_llama3_1_70b_distributed_config.json b/cvs/input/config_file/training/torchtitan/mi355_llama3_1_70b_distributed_config.json new file mode 100644 index 000000000..1fcc90cb7 --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_llama3_1_70b_distributed_config.json @@ -0,0 +1,88 @@ +{ + "schema_version": 1, + "framework": "torchtitan_distributed", + "gpu_arch": "MI355", + "enforce_thresholds": false, + "threshold_json": "mi355_llama3_1_70b_distributed_threshold.json", + "scaling_baseline": { + "tokens_per_sec_total": 0.0, + "num_nodes": 1 + }, + "config": { + "hf_token_file": "/home/{user-id}/.hf_token", + "log_dir": "/home/{user-id}/LOGS/torchtitan", + "scripts_dir": "/home/{user-id}/SCRIPTS/torchtitan", + "data_cache_dir": "/home/{user-id}/cache", + "rocm_dir": "", + "torchtitan_root": "/workspace/Primus/third_party/torchtitan", + "training_iterations": "10", + "nnodes": "4", + "nic_type": "thor2", + "nccl_socket_ifname": "ensf1np1", + "gloo_socket_ifname": "ensf1np1", + "nccl_ib_gid_index": "3", + "nccl_debug": "ERROR", + "master_address": "", + "verify_network_errors": "False", + "use_generated_config": "True" + }, + "model_params": { + "model_name": "llama3_1_70b", + "hf_model_name": "meta-llama/Llama-3.1-70B", + "sequence_length": "8192", + "dataset": "c4", + "lr": "3e-4", + "warmup_steps": "200", + "activation_checkpointing": "selective", + "compile": "false", + "tensor_parallel_degree": "4", + "pipeline_parallel_degree": "1", + "context_parallel_degree": "1", + "expert_parallel_degree": "1", + "enable_async_tensor_parallel": "false", + "precompute_float8_dynamic_scale_for_fsdp": "true", + "data_parallel_shard_degree": "8" + }, + "container": { + "lifetime": "per_run", + "name": "torchtitan_llama3_1_70b_distributed", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/dev/infiniband:/dev/infiniband" + ], + "devices": [ + "/dev/kfd", + "/dev/dri" + ] + } + } + }, + "sweep": { + "combinations": { + "llama3_1_70b-mi355-4n-bs128-mbs1-fp8": { + "name": "llama3_1_70b_4n_mbs1_gbs128_fp8", + "global_batch_size": "128", + "micro_batch_size": "1", + "precision": "fp8" + }, + "llama3_1_70b-mi355-4n-bs128-mbs1-bf16": { + "name": "llama3_1_70b_4n_mbs1_gbs128_bf16", + "global_batch_size": "128", + "micro_batch_size": "1", + "precision": "bf16" + } + }, + "runs": [ + "llama3_1_70b-mi355-4n-bs128-mbs1-fp8", + "llama3_1_70b-mi355-4n-bs128-mbs1-bf16" + ] + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_llama3_1_70b_distributed_threshold.json b/cvs/input/config_file/training/torchtitan/mi355_llama3_1_70b_distributed_threshold.json new file mode 100644 index 000000000..f9e141347 --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_llama3_1_70b_distributed_threshold.json @@ -0,0 +1,40 @@ +{ + "_comment": "Llama 3.1 70B 4-node distributed thresholds for MI355 TorchTitan.", + "_note": "Placeholder thresholds - update with actual 4-node test results.", + "MBS=1,GBS=128,PRECISION=fp8": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 12800 + }, + "training.loss": { + "kind": "max", + "value": 15.0 + }, + "training.scaling_efficiency_pct": { + "kind": "min", + "value": 85.0 + } + }, + "MBS=1,GBS=128,PRECISION=bf16": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 12800 + }, + "training.loss": { + "kind": "max", + "value": 15.0 + }, + "training.scaling_efficiency_pct": { + "kind": "min", + "value": 85.0 + } + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_llama3_1_8b_single_config.json b/cvs/input/config_file/training/torchtitan/mi355_llama3_1_8b_single_config.json new file mode 100644 index 000000000..b4b255b63 --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_llama3_1_8b_single_config.json @@ -0,0 +1,84 @@ +{ + "schema_version": 1, + "framework": "torchtitan_single", + "gpu_arch": "mi355", + "enforce_thresholds": false, + "config": { + "hf_token_file": "/home/{user-id}/.hf_token", + "log_dir": "/home/{user-id}/LOG_DIR", + "scripts_dir": "/home/{user-id}/SCRIPTS", + "data_cache_dir": "/home/{user-id}/cache", + "rocm_dir": "", + "torchtitan_root": "/workspace/Primus/third_party/torchtitan", + "training_iterations": "10", + "nnodes": "1", + "nic_type": "thor2", + "nccl_socket_ifname": "enp49s0f0np0", + "gloo_socket_ifname": "enp49s0f0np0", + "nccl_ib_gid_index": "3", + "nccl_debug": "ERROR", + "master_address": "127.0.0.1", + "verify_network_errors": "False", + "use_generated_config": "True" + }, + "model_params": { + "model_name": "llama3_1_8b", + "hf_model_name": "meta-llama/Llama-3.1-8B", + "sequence_length": "8192", + "dataset": "c4", + "lr": "3e-4", + "warmup_steps": "200", + "activation_checkpointing": "selective", + "compile": "false", + "tensor_parallel_degree": "1", + "pipeline_parallel_degree": "1", + "context_parallel_degree": "1", + "expert_parallel_degree": "1", + "enable_async_tensor_parallel": "false", + "precompute_float8_dynamic_scale_for_fsdp": "false", + "data_parallel_shard_degree": "8" + }, + "container": { + "lifetime": "per_run", + "name": "torchtitan_llama3_1_8b_bf16", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/dev/infiniband:/dev/infiniband" + ], + "devices": [ + "/dev/kfd", + "/dev/dri" + ] + } + } + }, + "sweep": { + "combinations": { + "llama3_1_8b-mi355-bs8-mbs1-bf16": { + "name": "llama3_1_8b_mbs1_gbs8_bf16", + "global_batch_size": "8", + "micro_batch_size": "1", + "precision": "bf16" + }, + "llama3_1_8b-mi355-bs8-mbs1-fp8": { + "name": "llama3_1_8b_mbs1_gbs8_fp8", + "global_batch_size": "8", + "micro_batch_size": "1", + "precision": "fp8" + } + }, + "runs": [ + "llama3_1_8b-mi355-bs8-mbs1-bf16", + "llama3_1_8b-mi355-bs8-mbs1-fp8" + ] + }, + "threshold_json": "mi355_llama3_1_8b_single_threshold.json" +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_llama3_1_8b_single_threshold.json b/cvs/input/config_file/training/torchtitan/mi355_llama3_1_8b_single_threshold.json new file mode 100644 index 000000000..af2dac586 --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_llama3_1_8b_single_threshold.json @@ -0,0 +1,38 @@ +{ + "_comment": "Llama 3.1 8B single-node thresholds for MI355 TorchTitan. Cell keys must match TorchTitanVariantConfig.cell_key() format: MBS=,GBS=,PRECISION=.", + "_note": "Based on actual MI355 test results: BF16: 11,767 TPS, FP8: 11,724 TPS. Thresholds set at actual - 10%.", + "MBS=1,GBS=8,PRECISION=bf16": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 10590, + "_calculation": "11767 * 0.90 = 10590.3" + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 84720, + "_calculation": "10590 * 8 (batch size) = 84720" + }, + "training.loss": { + "kind": "max", + "value": 12.77, + "_calculation": "10.64 * 1.20 = 12.768" + } + }, + "MBS=1,GBS=8,PRECISION=fp8": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 10552, + "_calculation": "11724 * 0.90 = 10551.6" + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 84416, + "_calculation": "10552 * 8 (batch size) = 84416" + }, + "training.loss": { + "kind": "max", + "value": 12.10, + "_calculation": "10.08 * 1.20 = 12.096" + } + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_llama3_3_70b_distributed_config.json b/cvs/input/config_file/training/torchtitan/mi355_llama3_3_70b_distributed_config.json new file mode 100644 index 000000000..d28e09167 --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_llama3_3_70b_distributed_config.json @@ -0,0 +1,88 @@ +{ + "schema_version": 1, + "framework": "torchtitan_distributed", + "gpu_arch": "MI355", + "enforce_thresholds": false, + "threshold_json": "mi355_llama3_3_70b_distributed_threshold.json", + "scaling_baseline": { + "tokens_per_sec_total": 0.0, + "num_nodes": 1 + }, + "config": { + "hf_token_file": "/home/{user-id}/.hf_token", + "log_dir": "/home/{user-id}/LOGS/torchtitan", + "scripts_dir": "/home/{user-id}/SCRIPTS/torchtitan", + "data_cache_dir": "/home/{user-id}/cache", + "rocm_dir": "", + "torchtitan_root": "/workspace/Primus/third_party/torchtitan", + "training_iterations": "10", + "nnodes": "4", + "nic_type": "thor2", + "nccl_socket_ifname": "ensf1np1", + "gloo_socket_ifname": "ensf1np1", + "nccl_ib_gid_index": "3", + "nccl_debug": "ERROR", + "master_address": "", + "verify_network_errors": "False", + "use_generated_config": "True" + }, + "model_params": { + "model_name": "llama3_3_70b", + "hf_model_name": "meta-llama/Llama-3.3-70B-Instruct", + "sequence_length": "8192", + "dataset": "c4", + "lr": "1.5e-4", + "warmup_steps": "200", + "activation_checkpointing": "selective", + "compile": "false", + "tensor_parallel_degree": "4", + "pipeline_parallel_degree": "1", + "context_parallel_degree": "1", + "expert_parallel_degree": "1", + "enable_async_tensor_parallel": "false", + "precompute_float8_dynamic_scale_for_fsdp": "true", + "data_parallel_shard_degree": "8" + }, + "container": { + "lifetime": "per_run", + "name": "torchtitan_llama3_3_70b_distributed", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "shm_size": "128G", + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/dev/infiniband:/dev/infiniband" + ], + "devices": [ + "/dev/kfd", + "/dev/dri" + ] + } + } + }, + "sweep": { + "combinations": { + "llama3_3_70b-mi355-4n-bs128-mbs1-fp8": { + "name": "llama3_3_70b_4n_mbs1_gbs128_fp8", + "global_batch_size": "128", + "micro_batch_size": "1", + "precision": "fp8" + }, + "llama3_3_70b-mi355-4n-bs128-mbs1-bf16": { + "name": "llama3_3_70b_4n_mbs1_gbs128_bf16", + "global_batch_size": "128", + "micro_batch_size": "1", + "precision": "bf16" + } + }, + "runs": [ + "llama3_3_70b-mi355-4n-bs128-mbs1-fp8", + "llama3_3_70b-mi355-4n-bs128-mbs1-bf16" + ] + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_llama3_3_70b_distributed_threshold.json b/cvs/input/config_file/training/torchtitan/mi355_llama3_3_70b_distributed_threshold.json new file mode 100644 index 000000000..1f13da610 --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_llama3_3_70b_distributed_threshold.json @@ -0,0 +1,40 @@ +{ + "_comment": "Llama 3.3 70B 4-node distributed thresholds for MI355 TorchTitan.", + "_note": "Placeholder thresholds - update with actual 4-node test results.", + "MBS=1,GBS=128,PRECISION=fp8": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 12800 + }, + "training.loss": { + "kind": "max", + "value": 15.0 + }, + "training.scaling_efficiency_pct": { + "kind": "min", + "value": 85.0 + } + }, + "MBS=1,GBS=128,PRECISION=bf16": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 100 + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 12800 + }, + "training.loss": { + "kind": "max", + "value": 15.0 + }, + "training.scaling_efficiency_pct": { + "kind": "min", + "value": 85.0 + } + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_llama3_3_70b_single_config.json b/cvs/input/config_file/training/torchtitan/mi355_llama3_3_70b_single_config.json new file mode 100644 index 000000000..a7b71ff7a --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_llama3_3_70b_single_config.json @@ -0,0 +1,88 @@ +{ + "schema_version": 1, + "framework": "torchtitan_single", + "gpu_arch": "MI355", + "enforce_thresholds": false, + "threshold_json": "mi355_llama3_3_70b_single_threshold.json", + "_scaling_baseline_comment": "Single-node baseline for scaling-efficiency %. tokens_per_sec_total=0.0 means disabled (record-only).", + "scaling_baseline": { + "tokens_per_sec_total": 0.0, + "num_nodes": 1 + }, + "config": { + "hf_token_file": "/home/{user-id}/.hf_token", + "log_dir": "/home/{user-id}/LOGS/torchtitan", + "scripts_dir": "/home/{user-id}/SCRIPTS/torchtitan", + "data_cache_dir": "/home/{user-id}/cache", + "rocm_dir": "", + "torchtitan_root": "/workspace/Primus/third_party/torchtitan", + "training_iterations": "10", + "nnodes": "1", + "nic_type": "thor2", + "nccl_socket_ifname": "ensf1np1", + "gloo_socket_ifname": "ensf1np1", + "nccl_ib_gid_index": "3", + "nccl_debug": "ERROR", + "master_address": "127.0.0.1", + "verify_network_errors": "False", + "use_generated_config": "True" + }, + "model_params": { + "model_name": "llama3_3_70b", + "hf_model_name": "meta-llama/Llama-3.3-70B-Instruct", + "sequence_length": "8192", + "dataset": "c4", + "lr": "1.5e-4", + "warmup_steps": "200", + "activation_checkpointing": "selective", + "compile": "false", + "tensor_parallel_degree": "4", + "pipeline_parallel_degree": "1", + "context_parallel_degree": "1", + "expert_parallel_degree": "1", + "enable_async_tensor_parallel": "false", + "precompute_float8_dynamic_scale_for_fsdp": "false", + "data_parallel_shard_degree": "2" + }, + "container": { + "lifetime": "per_run", + "name": "torchtitan_llama3_3_70b_single", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/dev/infiniband:/dev/infiniband" + ], + "devices": [ + "/dev/kfd", + "/dev/dri" + ] + } + } + }, + "sweep": { + "combinations": { + "llama3_3_70b-mi355-bs32-mbs1-bf16": { + "name": "llama3_3_70b_mbs1_gbs32_bf16", + "global_batch_size": "32", + "micro_batch_size": "1", + "precision": "bf16" + }, + "llama3_3_70b-mi355-bs32-mbs1-fp8": { + "name": "llama3_3_70b_mbs1_gbs32_fp8", + "global_batch_size": "32", + "micro_batch_size": "1", + "precision": "fp8" + } + }, + "runs": [ + "llama3_3_70b-mi355-bs32-mbs1-bf16", + "llama3_3_70b-mi355-bs32-mbs1-fp8" + ] + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_llama3_3_70b_single_threshold.json b/cvs/input/config_file/training/torchtitan/mi355_llama3_3_70b_single_threshold.json new file mode 100644 index 000000000..f0488ab8c --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_llama3_3_70b_single_threshold.json @@ -0,0 +1,38 @@ +{ + "_comment": "Llama 3.3 70B single-node thresholds for MI355 TorchTitan. Cell keys must match TorchTitanVariantConfig.cell_key() format: MBS=,GBS=,PRECISION=.", + "_note": "Based on actual MI355 test results with TP=4: BF16: 838 TPS, FP8: 838 TPS. Thresholds set at actual - 10%.", + "MBS=1,GBS=32,PRECISION=bf16": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 754, + "_calculation": "838 * 0.90 = 754.2" + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 24128, + "_calculation": "754 * 32 (batch size) = 24128" + }, + "training.loss": { + "kind": "max", + "value": 15.0, + "_note": "Placeholder - update with actual loss values" + } + }, + "MBS=1,GBS=32,PRECISION=fp8": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 754, + "_calculation": "838 * 0.90 = 754.2" + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 24128, + "_calculation": "754 * 32 (batch size) = 24128" + }, + "training.loss": { + "kind": "max", + "value": 15.0, + "_note": "Placeholder - update with actual loss values" + } + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_mixtral_8x22b_single_config.json b/cvs/input/config_file/training/torchtitan/mi355_mixtral_8x22b_single_config.json new file mode 100644 index 000000000..3e0023b6e --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_mixtral_8x22b_single_config.json @@ -0,0 +1,80 @@ +{ + "schema_version": 1, + "framework": "torchtitan_single", + "gpu_arch": "MI355", + "enforce_thresholds": false, + "threshold_json": "mi355_mixtral_8x22b_single_threshold.json", + "scaling_baseline": { + "tokens_per_sec_total": 0.0, + "num_nodes": 1 + }, + "config": { + "hf_token_file": "/home/{user-id}/.hf_token", + "log_dir": "/home/{user-id}/LOGS/torchtitan", + "scripts_dir": "/home/{user-id}/SCRIPTS/torchtitan", + "data_cache_dir": "/home/{user-id}/cache", + "rocm_dir": "", + "torchtitan_root": "/workspace/Primus/third_party/torchtitan", + "training_iterations": "10", + "nnodes": "1", + "nic_type": "thor2", + "nccl_socket_ifname": "ensf1np1", + "gloo_socket_ifname": "ensf1np1", + "nccl_ib_gid_index": "3", + "nccl_debug": "ERROR", + "master_address": "127.0.0.1", + "verify_network_errors": "False", + "use_generated_config": "True" + }, + "model_params": { + "model_name": "mixtral_8x22b", + "hf_model_name": "mistralai/Mixtral-8x22B-v0.1", + "sequence_length": "8192", + "dataset": "c4", + "lr": "1.5e-4", + "warmup_steps": "200", + "activation_checkpointing": "selective", + "compile": "false", + "tensor_parallel_degree": "4", + "pipeline_parallel_degree": "1", + "context_parallel_degree": "1", + "expert_parallel_degree": "1", + "enable_async_tensor_parallel": "false", + "precompute_float8_dynamic_scale_for_fsdp": "false", + "data_parallel_shard_degree": "2" + }, + "container": { + "lifetime": "per_run", + "name": "torchtitan_mixtral_8x22b_single", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/dev/infiniband:/dev/infiniband" + ], + "devices": [ + "/dev/kfd", + "/dev/dri" + ] + } + } + }, + "sweep": { + "combinations": { + "mixtral_8x22b-mi355-bs32-mbs1-bf16": { + "name": "mixtral_8x22b_mbs1_gbs32_bf16", + "global_batch_size": "32", + "micro_batch_size": "1", + "precision": "bf16" + } + }, + "runs": [ + "mixtral_8x22b-mi355-bs32-mbs1-bf16" + ] + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_mixtral_8x22b_single_threshold.json b/cvs/input/config_file/training/torchtitan/mi355_mixtral_8x22b_single_threshold.json new file mode 100644 index 000000000..268d07eea --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_mixtral_8x22b_single_threshold.json @@ -0,0 +1,20 @@ +{ + "_comment": "Mixtral 8x22B single-node thresholds for MI355 TorchTitan.", + "_note": "Based on actual MI355 test results: BF16: 838 TPS. Threshold set at actual - 10%.", + "MBS=1,GBS=32,PRECISION=bf16": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 754, + "_calculation": "838 * 0.90 = 754.2" + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 24128, + "_calculation": "754 * 32 (batch size) = 24128" + }, + "training.loss": { + "kind": "max", + "value": 15.0 + } + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_qwen3_32b_single_config.json b/cvs/input/config_file/training/torchtitan/mi355_qwen3_32b_single_config.json new file mode 100644 index 000000000..44ac525b5 --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_qwen3_32b_single_config.json @@ -0,0 +1,80 @@ +{ + "schema_version": 1, + "framework": "torchtitan_single", + "gpu_arch": "MI355", + "enforce_thresholds": false, + "threshold_json": "mi355_qwen3_32b_single_threshold.json", + "scaling_baseline": { + "tokens_per_sec_total": 0.0, + "num_nodes": 1 + }, + "config": { + "hf_token_file": "/home/{user-id}/.hf_token", + "log_dir": "/home/{user-id}/LOGS/torchtitan", + "scripts_dir": "/home/{user-id}/SCRIPTS/torchtitan", + "data_cache_dir": "/home/{user-id}/cache", + "rocm_dir": "", + "torchtitan_root": "/workspace/Primus/third_party/torchtitan", + "training_iterations": "10", + "nnodes": "1", + "nic_type": "thor2", + "nccl_socket_ifname": "ensf1np1", + "gloo_socket_ifname": "ensf1np1", + "nccl_ib_gid_index": "3", + "nccl_debug": "ERROR", + "master_address": "127.0.0.1", + "verify_network_errors": "False", + "use_generated_config": "True" + }, + "model_params": { + "model_name": "qwen3_32b", + "hf_model_name": "Qwen/Qwen2.5-32B", + "sequence_length": "8192", + "dataset": "c4", + "lr": "1.5e-4", + "warmup_steps": "200", + "activation_checkpointing": "selective", + "compile": "false", + "tensor_parallel_degree": "4", + "pipeline_parallel_degree": "1", + "context_parallel_degree": "1", + "expert_parallel_degree": "1", + "enable_async_tensor_parallel": "false", + "precompute_float8_dynamic_scale_for_fsdp": "false", + "data_parallel_shard_degree": "2" + }, + "container": { + "lifetime": "per_run", + "name": "torchtitan_qwen3_32b_single", + "image": "", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "/dev/infiniband:/dev/infiniband" + ], + "devices": [ + "/dev/kfd", + "/dev/dri" + ] + } + } + }, + "sweep": { + "combinations": { + "qwen3_32b-mi355-bs16-mbs1-bf16": { + "name": "qwen3_32b_mbs1_gbs16_bf16", + "global_batch_size": "16", + "micro_batch_size": "1", + "precision": "bf16" + } + }, + "runs": [ + "qwen3_32b-mi355-bs16-mbs1-bf16" + ] + } +} diff --git a/cvs/input/config_file/training/torchtitan/mi355_qwen3_32b_single_threshold.json b/cvs/input/config_file/training/torchtitan/mi355_qwen3_32b_single_threshold.json new file mode 100644 index 000000000..1929fe720 --- /dev/null +++ b/cvs/input/config_file/training/torchtitan/mi355_qwen3_32b_single_threshold.json @@ -0,0 +1,20 @@ +{ + "_comment": "Qwen3 32B single-node thresholds for MI355 TorchTitan.", + "_note": "Based on actual MI355 test results: BF16: 1,646 TPS. Threshold set at actual - 10%.", + "MBS=1,GBS=16,PRECISION=bf16": { + "training.throughput_per_gpu": { + "kind": "min", + "value": 1481, + "_calculation": "1646 * 0.90 = 1481.4" + }, + "training.tokens_per_gpu": { + "kind": "min", + "value": 23696, + "_calculation": "1481 * 16 (batch size) = 23696" + }, + "training.loss": { + "kind": "max", + "value": 15.0 + } + } +} diff --git a/cvs/lib/globals.py b/cvs/lib/globals.py index 397615c96..26b173281 100644 --- a/cvs/lib/globals.py +++ b/cvs/lib/globals.py @@ -9,6 +9,30 @@ log = logging.getLogger() + +# pssh's own host_logger emits every remote stdout/stderr line, tagged with the +# host (pssh/clients/base/single.py). Upstream keeps it quiet behind a +# NullHandler unless enable_host_logger() is called -- which CVS never does -- +# but `log` above is the ROOT logger, so propagation delivers those lines to +# CVS's handlers anyway. The result is that every line is logged twice: once by +# pssh, once by Pssh._process_output (cvs/lib/parallel/pssh.py). Dropping the +# pssh copy keeps the _process_output one, which is the one that honors +# print_console and can therefore be suppressed for bulk-data commands. +# +# A filter, not propagate=False: pytest's catching_logs attaches its capture +# handler to root AND to every non-propagating logger (_pytest/logging.py), so +# clearing propagate makes pytest attach directly and the duplicate survives. +# A filter drops the record before any handler is consulted, however attached. +# +# Named rather than a lambda so it is identifiable in +# logging.getLogger('pssh.host_logger').filters when someone is debugging log +# routing on a live node. +def _suppress_pssh_host_logger(_record): + return False + + +logging.getLogger('pssh.host_logger').addFilter(_suppress_pssh_host_logger) + error_list = [] diff --git a/cvs/lib/inference/ADDING_A_SUITE.md b/cvs/lib/inference/ADDING_A_SUITE.md new file mode 100644 index 000000000..94e90a35c --- /dev/null +++ b/cvs/lib/inference/ADDING_A_SUITE.md @@ -0,0 +1,660 @@ +# Adding a new DTNI suite + +Step-by-step guide for adding a new Distributed Training aNd Inference suite. +The `vllm_single` suite is the reference implementation throughout. +Follow this top to bottom; each step links to the authoritative contract at the +moment you need it. + +--- + +## The layer map (read this first) + +Every concern has exactly one home. Before writing any code, locate your work on +this table: + +| Layer | Directory | What belongs here | +|---|---|---| +| Framework-agnostic | `cvs/lib/utils/` | `BaseVariantConfig`, `substitute_config`, `Paths`, `ContainerSpec`, `evaluate_all` | +| Serving-generic | `cvs/lib/inference/utils/` | `Sweep`, `SeqCombo`, `GoodputSlo`, `Roles`, `validate_sweep_selector` | +| Framework-specific | `cvs/lib//utils/` | `VariantConfig` subclass, `Params`, `load_variant`, metric vocabulary | +| Test suite | `cvs/tests/inference//` | `conftest.py`, test module(s) | +| Input configs | `cvs/input/config_file/inference//` | `_config.json`, `_threshold.json` | + +Decision rule at every layer boundary: + +- "Does any other suite (now or plausibly soon) need this?" → move it up one layer. +- "Is this specific to my framework's CLI flags or artifact format?" → keep it here. + +When in doubt, push up. Code stranded too low gets copy-pasted into the next +suite; code pushed too high creates invisible coupling. Neither is free. + +--- + +## Step 1: Decide what is generic vs framework-specific + +Before writing a single class, answer these questions: + +**Is this a serving/inference suite?** + +Serving suites sweep sequence lengths (`isl`/`osl`) at concurrency levels. +The sweep machinery — `Sweep`, `SeqCombo`, `GoodputSlo`, `Roles`, +`validate_sweep_selector` — already lives in `cvs/lib/inference/utils/` and is +reusable unchanged. The only framework-specific piece is `Params`: the CLI flags +your benchmark tool accepts. + +See [The serving-generic / vllm-specific seam](utils/AGENTS.md#the-serving-generic--vllm-specific-seam) +for the second-framework checklist. + +**Is this a training suite?** + +Training suites typically sweep different dimensions: `batch_size`, `seq_len`, +`num_gpus`, or similar. Write your own sweep schema. Still subclass +`BaseVariantConfig` (the framework-agnostic skeleton is always your base). +Your `cell_key` format is your choice — it just has to match your +`threshold.json` top-level keys exactly. + +--- + +## Step 2: Subclass `BaseVariantConfig` + +Create `cvs/lib//utils/_config_loader.py`. + +**Minimal skeleton for a serving suite:** + +```python +from pydantic import model_validator +from typing_extensions import Literal + +from cvs.lib.utils.config_loader import BaseVariantConfig, _Forbid, substitute_config +from cvs.lib.inference.utils.inferencing_config_loader import ( + GoodputSlo, Roles, Run, Sweep, SeqCombo, validate_sweep_selector, +) +from cvs.lib..utils._parsing import GATED_METRICS + + +class Params(_Forbid): + # Your framework's CLI flags. All fields str (passed as CLI arguments). + tensor_parallelism: str = "1" + port_no: str = "8888" + # ... add your flags here + + +class VariantConfig(BaseVariantConfig): + framework: Literal["your_framework"] + gpu_arch: str + roles: Roles = Roles() + params: Params + sweep: Sweep + + def cell_key(self, isl, osl, concurrency) -> str: + """Single source of truth for the threshold key for one sweep cell.""" + return f"ISL={isl},OSL={osl},TP={self.params.tensor_parallelism},CONC={concurrency}" + + def expected_cells(self) -> list: + """Every cell key the sweep's runs selector picks.""" + by_name = {c.name: c for c in self.sweep.sequence_combinations} + return [ + self.cell_key(by_name[r.combo].isl, by_name[r.combo].osl, r.concurrency) + for r in self.sweep.runs + ] + + @model_validator(mode="after") + def _check_thresholds_cover_sweep(self): + # Copy the two-axis check from inferencing_config_loader.py: + # Axis 1: every sweep cell has a threshold entry; no key names a phantom cell. + # Axis 2: every present cell has a spec for every GATED_METRICS member. + # When enforce_thresholds=False: warn instead of raise. + ... + return self + + +def load_variant(config_path, cluster_dict) -> VariantConfig: + raw, thresholds = substitute_config(config_path, cluster_dict) + raw["thresholds"] = thresholds + return VariantConfig(**raw) +``` + +See [Subclassing BaseVariantConfig](../utils/AGENTS.md#subclassing-basevariantconfig) +for the full contract: what fields you must add, what methods you must implement, +and the validator ordering rules. + +Key points: + +- `cell_key` is the **single source of truth**. The loader's coverage check + (`_check_thresholds_cover_sweep`) calls it to build expected keys; `test_metric` + calls it to look up the threshold spec. Change the format in one place and both + paths move together. A space, field-order change, or separator difference silently + drops the cell (no threshold match, no verdict). +- `_check_thresholds_cover_sweep` must check **both axes**. Without axis 2, a gated + metric with no threshold spec falls through the record-only branch of `test_metric` + and reports a green PASS with zero assertions even when `enforce_thresholds=True`. +- `load_variant` must call `substitute_config` — never reimplement file-read or + placeholder substitution. See [substitute_config contract](../utils/AGENTS.md#config_loaderpy) + for what it returns and what it does not do (it does not validate or type-coerce). + +--- + +## Step 2b: Write the metric vocabulary module + +Create `cvs/lib//utils/_parsing.py`. + +Reference: `cvs/lib/inference/utils/vllm_parsing.py`. + +This module is a pure-transform layer with no I/O. It contains: + +1. **A pure transform function** that maps a raw benchmark artifact dict to a + namespaced `{"client.": value}` dict. This function accepts only + data structures (no `orch`, no file paths) so it can be unit-tested without + a running container. + +2. **`YOUR_METRICS: list[tuple[str, str]]`** — the display surface: a list of + `(short_name, unit)` pairs for every metric the suite surfaces. This list is + iterated by `pytest_generate_tests` to emit one `test_metric` row per metric + per cell. + +3. **`GATED_METRICS: set[str]`** — the asserted subset: the short names whose + threshold specs are required in `threshold.json` for every sweep cell when + `enforce_thresholds=True`. This set is imported by `VariantConfig`'s + `_check_thresholds_cover_sweep` to run the axis-2 coverage check at load time. + +4. **The gated-vs-record-only decision rule:** gate a metric (add it to + `GATED_METRICS`) when you have a calibrated baseline and a regression means a + real performance failure. Keep a metric record-only (in `YOUR_METRICS` but not + in `GATED_METRICS`) for diagnostic or informational metrics (e.g. percentiles + useful for debugging but not yet part of the SLO contract) or for metrics + whose baselines are not yet calibrated. Record-only metrics still appear in + the HTML results table; they simply do not trigger a FAIL. + +--- + +## Step 3: Write the job class + +Create `cvs/lib/inference/_job.py` (or a similarly named module). + +Reference: `cvs/lib/inference/vllm_single.py` (`VllmJob`). + +The job class owns the benchmark lifecycle for a single cell. It is deliberately +I/O-agnostic above the `orch.exec` boundary — all container/SSH plumbing belongs +to `orch`, which is injected. + +**Constructor:** accept `orch`, `variant` (your `VariantConfig`), and every +per-cell parameter (`isl`, `osl`, `concurrency`, etc.) as explicit arguments. +Pull all config values from `variant.params` and `variant.paths` here, not in +the methods, so the methods stay stateless and testable. + +**Required methods:** + +```python +class YourJob: + def build_server_cmd(self): + """Write the env script and create per-cell output directories inside the container.""" + + def start_server(self): + """Launch the server in the background inside the container.""" + + def is_ready(self) -> bool: + """Check readiness by scanning the server log (not a fixed tail).""" + + def wait_ready(self): + """Poll until is_ready() or raise on timeout.""" + + def stop_server(self): + """Kill the server process.""" + + def run_client(self): + """Launch the benchmark client in the background inside the container.""" + + def wait_client_complete(self): + """Poll the client log until completion, crash, or timeout.""" + + def parse_results(self) -> dict: + """Fetches the results artifact (the only method that reads output data); the metric + transform is delegated to the pure function in _parsing.py.""" +``` + +**Key patterns from `VllmJob` to carry forward:** + +- **Scan the whole server log for readiness**, not `tail -N`. The startup banner + scrolls out of a fixed tail once the server gets chatty. +- **Accumulate completion and failure states independently in each poll iteration, + then raise on failure before returning on completion.** The benchmark tool + always prints an explicit completion marker (`COMPLETION_RE`) — key off that + positive signal rather than the absence of an error line. +- **Per-cell output directories** keyed by `isl/osl/concurrency`. A multi-cell + sweep must not overwrite an earlier cell's artifact; without this, + `parse_results` may silently read stale data from a prior cell. +- **`parse_results` raises on empty/missing/unparseable artifacts.** The test + wraps it in `try/except ... raise`, so a hard failure here is the correct + behavior — it breaks the cell cleanly rather than recording a silently-green row. +- **`parse_results` delegates the transform** to the pure function in + `_parsing.py`. The fetch (I/O) lives in the job class because + artifact layout is job-specific; the metric math lives in `_parsing.py` so + other suite variants can reuse it. + +--- + +## Step 4: Write the conftest fixtures + +Create `cvs/tests/inference//conftest.py`. + +Reference: `cvs/tests/inference/vllm/conftest.py`. + +See [conftest fixtures](utils/AGENTS.md#conftest-fixtures) for the fixture +ownership table. + +**All fixtures must be `scope="module"`** so they are shared across the entire +parametrized test run. A `scope="function"` fixture would re-launch the container +for every single metric row. + +**Required fixtures:** + +```python +@pytest.fixture(scope="module") +def cluster_dict(pytestconfig): + cluster_file = pytestconfig.getoption("cluster_file") + if not cluster_file: + pytest.fail("--cluster_file is required") + with open(cluster_file) as fp: + d = json.load(fp) + return resolve_cluster_config_placeholders(d) + + +@pytest.fixture(scope="module") +def variant_config(pytestconfig, cluster_dict): + config_file = pytestconfig.getoption("config_file") + if not config_file: + pytest.fail("--config_file is required") + return load_variant(config_file, cluster_dict) + + +@pytest.fixture(scope="module") +def lifecycle(): + return _Lifecycle() # see _Lifecycle class below + + +@pytest.fixture(scope="module") +def orch(cluster_dict, variant_config, lifecycle): + container_block = _deep_merge( + cluster_dict.get("container", {}), + variant_config.container.model_dump(), + ) + testsuite_config = {"orchestrator": "container", "container": container_block} + cfg = OrchestratorConfig.from_configs(cluster_dict, testsuite_config) + o = OrchestratorFactory.create_orchestrator(log, cfg) + yield o + if not lifecycle.torn_down: + log.info("orch fixture leak-guard: tearing down container") + o.teardown_containers() + + +@pytest.fixture(scope="module") +def hf_token(variant_config): + path = variant_config.paths.hf_token_file + if not os.path.isfile(path): + pytest.skip(f"hf_token file missing: {path}") + with open(path) as fp: + return fp.read().strip() + + +@pytest.fixture(scope="module") +def inf_res_dict(): # or train_res_dict for a training suite + return {} +``` + +The `hf_token` fixture reads `variant_config.paths.hf_token_file` and calls +`pytest.skip` (not `pytest.fail`) when the file is missing. Using `skip` rather +than `fail` means suites that do not require an HF token can simply omit this +fixture from their conftest without breaking collection; the inference test that +accepts it as an argument will be skipped rather than erroring at fixture setup. + +**`_Lifecycle`** — cross-test state for the lifecycle-as-tests model. Copy from +`cvs/tests/inference/vllm/conftest.py`. It carries three fields: +- `failed: bool` — set when any stage fails; causes remaining stages to skip +- `torn_down: bool` — set when `test_teardown` succeeds; suppresses the + `orch` leak-guard finalizer so teardown never runs twice +- `report: dict` — maps `nodeid → [(label, value, unit)]`; populated by + `lifecycle.record(...)` and rendered by `pytest_runtest_makereport` + +**The `_deep_merge` pattern:** + +`OrchestratorConfig.from_configs` does a top-level `dict.update`, so a bare +variant container block wipes all cluster-set container settings. Deep-merge the +variant ONTO the cluster block so cluster-set keys (e.g. `shm_size`, env maps) +survive, with the variant winning on conflicts. Copy `_deep_merge` verbatim from +the vllm conftest. If your suite is the second to need it, extract it to +`cvs/lib/utils/` instead of copy-pasting again. + +**Required pytest hooks:** + +```python +def pytest_collection_modifyitems(items): + """Pin lifecycle order explicitly — never rely on definition order.""" + rank = { + "test_launch_container": 0, + "test_setup_sshd": 1, + "test_model_fetch": 2, + "test_": 3, + "test_metric": 4, + "test_print_results_table": 5, + "test_teardown": 6, + } + items.sort(key=lambda it: rank.get(it.originalname or it.name.split("[")[0], 99)) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """Attach this test's recorded rows to its HTML detail panel.""" + ... # copy from vllm conftest + + +def pytest_html_results_table_header(cells): + """Add Value + Unit columns.""" + cells.insert(-1, "Value") + cells.insert(-1, "Unit") + + +def pytest_html_results_table_row(report, cells): + """Populate Value + Unit from metric_value / metric_unit user properties.""" + ... # copy from vllm conftest +``` + +`pytest_collection_modifyitems` is not optional. `test_print_results_table` is +typically an imported function whose source line points into a shared module, so +default pytest ordering collects it first — which logs an empty table before any +cell ran. Explicit ranking fixes this. + +--- + +## Step 5: Wire up `pytest_generate_tests` + +Add `pytest_generate_tests` to your **test module** (not conftest), immediately +above the test functions. + +Reference: `cvs/tests/inference/vllm/vllm_single.py`. + +See [pytest_generate_tests mirror rule](utils/AGENTS.md#pytest_generate_tests-mirror-rule) +for why this must call `validate_sweep_selector`. + +```python +def pytest_generate_tests(metafunc): + """Parametrize the workload test and test_metric from the raw config sweep. + + Runs at collection time before fixtures exist — reads raw JSON directly. + """ + config_file = metafunc.config.getoption("config_file") + if not config_file or not os.path.isfile(config_file): + return + with open(config_file) as fp: + raw = json.load(fp) + sweep = raw.get("sweep", {}) + combos = sweep.get("sequence_combinations", []) + runs = sweep.get("runs", []) + + # Validate GoodputSlo dicts through the _Forbid model so a typo'd SLO key + # fails collection, not silently drops the gate on hardware. + for combo in combos: + if combo.get("goodput_slo") is not None: + GoodputSlo(**combo["goodput_slo"]) + + # Mirror the typed Sweep validator via the shared rule. + # If you add a check to Sweep, add it to validate_sweep_selector so both paths enforce it. + validate_sweep_selector([c["name"] for c in combos], [r["combo"] for r in runs]) + + by_name = {c["name"]: c for c in combos} + cases = [(by_name[r["combo"]], r["concurrency"]) for r in runs] + ids = [r["combo"] + "-conc" + str(r["concurrency"]) for r in runs] + + if "metric" in metafunc.fixturenames: + # test_metric: one case per (cell, metric) + metric_cases = [] + metric_ids = [] + for (combo, c), cid in zip(cases, ids): + for short, _unit in YOUR_METRICS: + metric_cases.append((combo, c, short)) + metric_ids.append(cid + "-" + short) + metafunc.parametrize("seq_combo,concurrency,metric", metric_cases, ids=metric_ids) + elif "seq_combo" in metafunc.fixturenames and "concurrency" in metafunc.fixturenames and cases: + # workload test: one case per cell + metafunc.parametrize("seq_combo,concurrency", cases, ids=ids) +``` + +**Why `validate_sweep_selector` is mandatory here:** +`pytest_generate_tests` reads raw JSON at collection time, before `load_variant` +and the typed `Sweep` validator have run. Without calling `validate_sweep_selector` +here, a duplicate combo name or a `run.combo` typo is a silently-dropped cell at +collection time — the sweep runs a different matrix than the config reads. + +--- + +## Step 6: Write the test module + +Create `cvs/tests/inference//.py`. + +Reference: `cvs/tests/inference/vllm/vllm_single.py`. + +**The lifecycle-as-tests model:** each stage is an independent pytest test, not +fixture body code. Each appears as a timed, independently pass/fail HTML row. + +**Standard lifecycle order** (must match the rank dict in `pytest_collection_modifyitems`): + +1. `test_launch_container` — calls `orch.setup_containers()`; asserts container is running +2. `test_setup_sshd` — calls `orch.setup_sshd()`; probes `:2224` for multinode +3. `test_model_fetch` — ensures model bytes present; polls/downloads if remote +4. `test_` — benchmark loop; stores results in `res_dict` +5. `test_metric` — one test per metric per cell; reads `res_dict`; asserts verdict +6. `test_print_results_table` — summary log; must run after all cells +7. `test_teardown` — calls `orch.teardown_containers()`; sets `lifecycle.torn_down` + +**Invariants — every suite must enforce these:** + +- Every test except `test_launch_container`, `test_print_results_table`, and + `test_teardown` checks `lifecycle.failed` and calls `pytest.skip(...)` if + `True`. This prevents cascading failures where a broken launch causes every + subsequent cell to re-fail instead of skipping cleanly. + `test_print_results_table` does not guard on `lifecycle.failed`; instead it + checks whether `inf_res_dict` is empty and logs whatever results were + recorded (even a partial sweep produces a useful table). This behavior is + implemented in `cvs/tests/inference/vllm/_shared.py`; verify the check there + if you are adapting the pattern for a new suite. +- `test_` wraps its entire body in `try/except`: on any exception, set + `lifecycle.failed = True` then re-raise. +- `test_teardown` **never skips** — it must run even when `lifecycle.failed` is + `True`. The container must be torn down regardless of what happened in the sweep. +- `test_teardown` sets `lifecycle.torn_down = True` after a successful teardown. + This suppresses the `orch` fixture's leak-guard finalizer so the container is + not torn down twice. + +**`test_metric` pattern:** + +```python +def test_metric(seq_combo, concurrency, metric, inf_res_dict, variant_config, lifecycle, request): + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + # Build the lookup key (must match what test_ stored) + isl, osl = seq_combo["isl"], seq_combo["osl"] + key = (variant_config.model.id, variant_config.gpu_arch, isl, osl, + seq_combo.get("name", "default"), concurrency) + if key not in inf_res_dict: + pytest.skip(f"no recorded results for cell {key!r}") + + host_dict = inf_res_dict[key] + _host, actuals = next(iter(host_dict.items())) + full = "client." + metric + value = actuals.get(full) + unit = YOUR_METRIC_UNITS.get(metric, "-") + + # Attach for HTML rendering (Value/Unit columns) + request.node.user_properties.append(("metric_value", value)) + request.node.user_properties.append(("metric_unit", unit)) + + if not variant_config.enforce_thresholds: + return # record-only + + cell = variant_config.cell_key(isl, osl, concurrency) + spec = (variant_config.thresholds.get(cell) or {}).get(full) + if spec is None: + return # record-only (no spec for this metric) + + # Pass the FULL per-cell actuals dict, not just this one metric's value. + # evaluate_all needs the full dict so a min_ratio spec can resolve its + # reference metric from the same actuals. + evaluate_all(actuals, {full: spec}) +``` + +See [evaluate_all contract](../utils/AGENTS.md#verdictpy) for why full cell +actuals are passed (needed for `min_ratio` reference resolution), and for the +behavior on `None` values and missing metrics. + +--- + +## Step 7: Write the config and threshold files + +**Config JSON** (`cvs/input/config_file/inference//_config.json`): + +```json +{ + "schema_version": 1, + "framework": "your_framework", + "gpu_arch": "mi300x", + "enforce_thresholds": false, + "threshold_json": "/absolute/path/to/your_threshold.json", + "paths": { + "shared_fs": "/mnt/data/{user-id}", + "models_dir": "{shared_fs}/models", + "log_dir": "{shared_fs}/logs", + "hf_token_file": "/home/{user-id}/.hf_token" + }, + "model": {"id": "meta-llama/Llama-3.1-70B-Instruct", "remote": 0}, + "container": { + "lifetime": "per_run", + "name": "your_suite_container", + "image": "your.registry/image:tag", + "runtime": {"name": "docker", "args": {}} + }, + "params": {"tensor_parallelism": "8"}, + "sweep": { + "sequence_combinations": [ + {"name": "isl1000_osl1000", "isl": "1000", "osl": "1000"} + ], + "runs": [ + {"combo": "isl1000_osl1000", "concurrency": 16} + ] + } +} +``` + +Start with `"enforce_thresholds": false` until you have calibrated baselines. +Flip to `true` once threshold values are established. + +`threshold_json` is a **literal absolute path** — not relative to the config +file, not a glob. No placeholder substitution of any kind is applied to +`threshold_json` — not cluster placeholders, not `{paths.*}`. It is read +verbatim before any substitution pass runs. If the threshold path must vary by +user, it must be pre-resolved before being written into the config file. See +[placeholder-substitution.md](../utils/docs/placeholder-substitution.md) for a +worked example. + +**Threshold JSON** (`_threshold.json`): + +```json +{ + "_comment": "keys starting with _ are stripped before the coverage check", + "ISL=1000,OSL=1000,TP=8,CONC=16": { + "client.total_token_throughput": {"kind": "min_tok_s", "value": 12000}, + "client.output_throughput": {"kind": "min_tok_s", "value": 1500}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 200}, + "client.success_rate": {"kind": "min", "value": 0.99}, + "client.failed": {"kind": "max", "value": 0} + } +} +``` + +Every top-level key must match `cell_key(...)` output exactly — same field +order, same separators, no spaces. See +[cell-key-format.md](utils/docs/cell-key-format.md) for the exact format spec +and common mistake patterns. + +Every `GATED_METRICS` member must have a spec for every present cell. Missing +specs are caught at load time by `_check_thresholds_cover_sweep` (axis 2) when +`enforce_thresholds=True`. + +See [threshold-kinds.md](../utils/docs/threshold-kinds.md) for the full kind +reference (`min`, `max`, `max_ms`, `within`, `min_tok_s`, `min_ratio`). + +--- + +## Step 8: Suite reports (optional, `--html`) + +**Full guide:** `cvs/lib/report/README.md` + +Add one preset at `cvs/lib/report/presets/.py`. Root `cvs/conftest.py` +auto-loads it when the stem matches `cvs run …` and writes HTML/JSON/viewer at session +end when `--html` is set. Reports are render-only and do not change pass/fail. + +Do **not** wire reports in suite `conftest.py`. **IX-atom reference:** +`presets/inferencex_atom.py` + `presets/inferencex_atom.py`. + +--- + +## Pre-PR checklist + +Walk this list against your suite before opening a PR. Each item is verifiable +in the existing code. + +**Config machinery** + +- [ ] `load_variant` calls `substitute_config` — does not reimplement file-read or substitution +- [ ] `VariantConfig` subclasses `BaseVariantConfig` +- [ ] `VariantConfig` declares `framework: Literal["your_framework"]`, `params`, and `sweep` +- [ ] `cell_key` is implemented and is the single source of truth used by both + `_check_thresholds_cover_sweep` and `test_metric` +- [ ] `expected_cells` is implemented and returns the full list of cell keys +- [ ] `_check_thresholds_cover_sweep` is present as a `@model_validator(mode="after")` + and checks both axes (cell coverage + gated-metric coverage) + +**Test fixtures** + +- [ ] All fixtures are `scope="module"` +- [ ] `orch` fixture has a leak-guard finalizer that calls `teardown_containers()` when + `lifecycle.torn_down` is `False` +- [ ] `_deep_merge` is used when building the container block (not bare `dict.update`) + +**Test lifecycle** + +- [ ] Lifecycle order is pinned explicitly in `pytest_collection_modifyitems` +- [ ] Every test except `test_launch_container`, `test_print_results_table`, and `test_teardown` checks `lifecycle.failed` and skips if `True` (`test_print_results_table` instead checks whether `inf_res_dict` is empty — see `_shared.py`) +- [ ] `test_` catches all exceptions, sets `lifecycle.failed = True`, re-raises +- [ ] `test_teardown` does NOT skip on `lifecycle.failed` +- [ ] `test_teardown` sets `lifecycle.torn_down = True` after successful teardown + +**`pytest_generate_tests`** + +- [ ] Calls `validate_sweep_selector` to mirror the typed `Sweep` validator — collection-time + and load-time paths enforce the same rules +- [ ] Validates `GoodputSlo` dicts through the `_Forbid` model if your sweep uses goodput SLOs + +**`test_metric`** + +- [ ] Passes the full per-cell actuals dict to `evaluate_all`, not just the single metric value +- [ ] Returns (record-only) when `enforce_thresholds` is `False` +- [ ] Returns (record-only) when `spec is None` for this cell+metric + +**Layer placement** + +- [ ] Things only this suite needs → `cvs/lib//utils/` (not pushed up) +- [ ] Things any serving suite needs → `cvs/lib/inference/utils/` (not kept local) +- [ ] Things any CVS suite needs → `cvs/lib/utils/` (not kept at the inference layer) + +**Config files** + +- [ ] `threshold_json` is a literal absolute path (not relative, not a glob) +- [ ] Every threshold key matches `cell_key(...)` output exactly +- [ ] Every `GATED_METRICS` member has a spec for every present cell +- [ ] `enforce_thresholds` starts as `false` until baselines are calibrated +- [ ] New gated metric → every existing `threshold.json` that covers cells where the + metric is defined has a spec for it + +**Suite reports (optional)** + +- [ ] Added `cvs/lib/report/presets/.py` when using `--html` (see [Step 8](#step-8-suite-reports-optional-html)) diff --git a/cvs/lib/inference/atom/__init__.py b/cvs/lib/inference/atom/__init__.py new file mode 100644 index 000000000..395823d63 --- /dev/null +++ b/cvs/lib/inference/atom/__init__.py @@ -0,0 +1 @@ +'''ATOM suite library (orchestrator, config loader, parsing).''' diff --git a/cvs/lib/inference/atom/atom_config_loader.py b/cvs/lib/inference/atom/atom_config_loader.py new file mode 100644 index 000000000..397e1a9e7 --- /dev/null +++ b/cvs/lib/inference/atom/atom_config_loader.py @@ -0,0 +1,358 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +ATOM suite config schema (``atom``). + +Generic paths/model/container/threshold plumbing lives in +:mod:`cvs.lib.utils.config_loader`. Sweep selector types are shared with +:mod:`cvs.lib.inference.utils.inferencing_config_loader`. +''' + +from __future__ import annotations + +import re +from typing import Any, Dict, List, Union + +from pydantic import field_validator, model_validator +from typing_extensions import Literal + +from cvs.lib.inference.utils.inferencing_config_loader import ( + RoleServer, + Sweep, + validate_sweep_selector, + validate_thresholds_cover_sweep, +) +from cvs.lib.inference.atom.atom_parsing import GATED_METRICS +from cvs.lib.utils.config_loader import BaseVariantConfig, _Forbid, substitute_config +from cvs.lib import globals + +ATOM_DRIVERS = ("atom", "vllm", "vllm_atom", "sglang") +ATOM_PP_DRIVERS = ("vllm", "vllm_atom", "sglang") + +log = globals.log + +# Written by test_discover_topology / resolve_multinode_fabric — not user env. +_ORCH_MANAGED_NETWORK_ENV = frozenset({"NCCL_SOCKET_IFNAME", "GLOO_SOCKET_IFNAME", "TP_SOCKET_IFNAME", "NCCL_IB_HCA"}) +_IB_HCA_NETDEV_RE = re.compile(r"^mlx5_\d+$", re.I) + + +class AtomRoleServer(RoleServer): + # Extra CLI tokens for ``python -m atom.entrypoints.openai_server`` after + # ``--model`` / ``--server-port`` (e.g. ``-tp``, ``--kv_cache_dtype``). + atom_args: List[str] = [] + # Extra CLI tokens appended to ``python3 -m sglang.launch_server`` (driver=sglang). + sglang_args: List[str] = [] + # IB HCA devices for NCCL_IB_HCA (multinode only). + # absent or "auto" -> use whatever ibv_devinfo -l reports (test_discover_topology). + # explicit list -> validated at preflight against ibv_devinfo output. + ib_hca_devices: Union[Literal["auto"], List[str], None] = None + # Linux netdev for NCCL_SOCKET_IFNAME / GLOO_SOCKET_IFNAME on multinode PP runs. + # absent or "auto" -> resolved at runtime by test_discover_topology from cluster IPs. + ib_netdev: Union[Literal["auto"], str, None] = None + + @field_validator("ib_netdev", mode="after") + @classmethod + def _normalize_ib_netdev(cls, v): + raw = (v or "").strip() + if raw and raw.lower() != "auto" and _IB_HCA_NETDEV_RE.match(raw): + log.warning( + "roles.server.ib_netdev=%r looks like an IB HCA name; coercing to 'auto' " + "(socket netdev is discovered from cluster IPs at runtime)", + raw, + ) + return "auto" + return v + + @model_validator(mode="after") + def _strip_orchestrator_managed_network_env(self): + if not self.env: + return self + dropped = sorted(k for k in self.env if k in _ORCH_MANAGED_NETWORK_ENV) + if not dropped: + return self + log.warning( + "roles.server.env drops orchestrator-managed keys %s " + "(set by test_discover_topology / build_server_cmd instead)", + dropped, + ) + self.env = {k: v for k, v in self.env.items() if k not in _ORCH_MANAGED_NETWORK_ENV} + return self + + +class AtomRoles(_Forbid): + server: AtomRoleServer = AtomRoleServer() + + +class AtomParams(_Forbid): + # ``atom`` = standalone ATOM openai_server + benchmark_serving. + # ``vllm_atom`` = vLLM coordinator + ATOM local kernels (true multinode PP). + # ``vllm`` = interim ROCm vLLM uplift (vllm serve + vllm bench serve). + # ``sglang`` = SGLang coordinator (launch_server + bench_serving) for PP runs. + driver: Literal["atom", "vllm", "vllm_atom", "sglang"] = "vllm" + backend: str = "vllm" + base_url: str = "http://0.0.0.0" + port_no: str = "8000" + dataset_name: str = "random" + burstiness: str = "1.0" + seed: str = "0" + request_rate: str = "inf" + random_range_ratio: str = "0.8" + random_prefix_len: str = "0" + tensor_parallelism: str = "8" + tokenizer_mode: str = "auto" + percentile_metrics: str = "ttft,tpot,itl,e2el" + metric_percentiles: str = "95,99" + num_prompts: str = "1000" + max_model_length: str = "8192" + client_poll_count: str = "50" + client_poll_wait_time: str = "60" + client_initial_wait_s: str = "120" + server_precheck_wait_s: str = "30" + server_warmup_wait_s: str = "330" + server_poll_count: str = "60" + server_poll_wait_time: str = "60" + reuse_server_across_sweep: str = "false" + bench_max_failed_requests: str = "0" + bench_extra_args: str = "" + result_filename: str = "results" + # Multinode (M5): omit or set nnodes=1 for single-node runs. When nnodes>1, + # cluster node_dict must list the same number of hosts and test_setup_sshd runs. + nnodes: str = "1" + pipeline_parallel_size: str = "1" + master_addr: str = "" + master_port: str = "29501" + # Optional single-node reference output_throughput for scaling.efficiency_pct. + scaling_baseline_output_throughput: str = "" + + +class AtomRunCard(_Forbid): + upstream_run_url: str = "" + atom_image_pin: str = "" + notes: str = "" + + +ATOM_FRAMEWORKS = ("atom",) + + +class AtomVariantConfig(BaseVariantConfig): + framework: Literal["atom"] + + gpu_arch: str + run_card: AtomRunCard = AtomRunCard() + roles: AtomRoles = AtomRoles() + params: AtomParams + sweep: Sweep + + def cell_key(self, isl, osl, concurrency): + p = self.params + key = f"ISL={isl},OSL={osl},TP={p.tensor_parallelism}" + nnodes = int(p.nnodes) + pp = int(p.pipeline_parallel_size) + if p.driver == "atom": + if nnodes > 1: + key += f",DP={nnodes},NNODES={nnodes}" + elif p.driver in ATOM_PP_DRIVERS: + if pp > 1 or nnodes > 1: + key += f",PP={p.pipeline_parallel_size}" + if nnodes > 1: + key += f",NNODES={p.nnodes}" + return f"{key},CONC={concurrency}" + + def expected_cells(self) -> List[str]: + by_name = {c.name: c for c in self.sweep.sequence_combinations} + return [self.cell_key(by_name[r.combo].isl, by_name[r.combo].osl, r.concurrency) for r in self.sweep.runs] + + @model_validator(mode="after") + def _check_thresholds_cover_sweep(self): + validate_thresholds_cover_sweep( + expected_cells=self.expected_cells(), + thresholds=self.thresholds, + enforce_thresholds=self.enforce_thresholds, + gated_metrics=GATED_METRICS, + ) + if int(self.params.nnodes) > 1 and (self.params.scaling_baseline_output_throughput or "").strip(): + missing = [] + for cell in self.expected_cells(): + specs = self.thresholds.get(cell) or {} + if "scaling.efficiency_pct" not in specs: + missing.append(cell) + if missing: + msg = ( + "multinode variant with scaling_baseline_output_throughput requires " + f"scaling.efficiency_pct in every cell; missing: {missing}" + ) + if self.enforce_thresholds: + raise ValueError(msg) + import warnings + + warnings.warn(f"{msg} (enforce_thresholds=false -> record-only)", stacklevel=2) + return self + + @model_validator(mode="after") + def _atom_multinode_uses_dp_not_pp(self): + if self.params.driver == "atom" and int(self.params.nnodes) > 1: + if int(self.params.pipeline_parallel_size) > 1: + raise ValueError( + "params.driver='atom' with nnodes>1 uses ATOM SPMD data parallel (-dp); " + "standalone ATOM cannot execute pipeline parallel. For true PP>1 use " + "params.driver='vllm_atom' or 'sglang'." + ) + return self + + @model_validator(mode="after") + def _pp_driver_distributed_consistency(self): + driver = self.params.driver + if driver not in ATOM_PP_DRIVERS: + return self + nn = int(self.params.nnodes) + pp = int(self.params.pipeline_parallel_size) + is_ray = self.roles.server.serve_args.get("distributed-executor-backend") == "ray" + if nn > 1 and pp == 1 and not is_ray: + raise ValueError( + f"params.driver={driver!r} with nnodes={nn} requires pipeline_parallel_size>1 " + f"(got pp={pp}) for multinode pipeline parallel" + ) + if pp > 1 and nn == 1: + raise ValueError( + f"pipeline_parallel_size={pp} > 1 requires nnodes > 1 (got nnodes={nn}) for params.driver={driver!r}" + ) + return self + + @model_validator(mode="after") + def _atom_driver_requires_inline_server_args(self): + if self.params.driver == "atom" and not self.roles.server.atom_args: + raise ValueError( + "params.driver='atom' requires roles.server.atom_args " + "(inline ATOM openai_server CLI tokens, vLLM-style)" + ) + return self + + +def expand_sweep(sweep): + """Expand a sweep into ``(cases, ids)`` for pytest parametrization.""" + if hasattr(sweep, "sequence_combinations"): + combos = [c.model_dump() for c in sweep.sequence_combinations] + runs = [r.model_dump() for r in sweep.runs] + else: + combos = sweep.get("sequence_combinations", []) + runs = sweep.get("runs", []) + validate_sweep_selector([c["name"] for c in combos], [r["combo"] for r in runs]) + by_name = {c["name"]: c for c in combos} + cases = [] + ids = [] + for run in runs: + combo = by_name[run["combo"]] + conc = run["concurrency"] + cases.append((combo, conc)) + ids.append(f"{run['combo']}-conc{conc}") + return cases, ids + + +def reuse_server_flag(params) -> bool: + """Return True when ``params.reuse_server_across_sweep`` is a truthy string.""" + raw = str(getattr(params, "reuse_server_across_sweep", "false")).strip().lower() + return raw in ("true", "1", "yes") + + +def server_session_key(variant_config, isl, osl): + """Stable key for server reuse across sweep cells with identical model/shape.""" + p = variant_config.params + roles = variant_config.roles.server + if p.driver == "atom": + server_tokens = tuple(roles.atom_args) + elif p.driver == "sglang": + server_tokens = tuple(roles.sglang_args) + else: + server_tokens = tuple(sorted(roles.serve_args.items())) + return ( + variant_config.model.id, + p.driver, + str(isl), + str(osl), + server_tokens, + p.tensor_parallelism, + p.nnodes, + p.pipeline_parallel_size, + p.master_addr, + p.master_port, + ) + + +def expand_sweep_parametrize(sweep, fixturenames): + """Build pytest parametrize args for inference or metric-tier collection.""" + from cvs.lib.inference.atom.atom_parsing import METRIC_TIER_ORDER + + cases, ids = expand_sweep(sweep) + if "metric_tier" in fixturenames: + if not cases: + return None + tier_cases = [] + tier_ids = [] + for (combo, c), cid in zip(cases, ids): + for tier in METRIC_TIER_ORDER: + tier_cases.append((combo, c, tier)) + tier_ids.append(f"{cid}-{tier}") + return ("seq_combo,concurrency,metric_tier", tier_cases, tier_ids) + if "seq_combo" in fixturenames and "concurrency" in fixturenames and cases: + return ("seq_combo,concurrency", cases, ids) + return None + + +def load_variant(config_path, cluster_dict) -> AtomVariantConfig: + raw, thresholds = substitute_config(config_path, cluster_dict) + raw["thresholds"] = thresholds + return AtomVariantConfig(**raw) + + +def placeholder_gated_threshold_cell( + *, + output_throughput_min: float = 0, + total_token_throughput_min: float = 0, + per_gpu_throughput_min: float = 0, + output_tput_per_gpu_min: float = 0, + mean_ttft_max_ms: float = 1_000_000, + p99_ttft_max_ms: float = 1_000_000, + mean_tpot_max_ms: float = 1_000_000, + p95_tpot_max_ms: float = 1_000_000, + failed_max: int = 1_000_000_000, + success_rate_min: float = 0, +) -> Dict[str, Any]: + """Return one sweep cell's ``client.*`` specs covering every ``GATED_METRICS`` member.""" + loose_ms = {"kind": "max_ms", "value": 1_000_000} + return { + "client.total_token_throughput": {"kind": "min_tok_s", "value": total_token_throughput_min}, + "client.output_throughput": {"kind": "min_tok_s", "value": output_throughput_min}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": per_gpu_throughput_min}, + "client.output_tput_per_gpu": {"kind": "min_tok_s", "value": output_tput_per_gpu_min}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": mean_ttft_max_ms}, + "client.median_ttft_ms": loose_ms, + "client.p90_ttft_ms": loose_ms, + "client.p95_ttft_ms": loose_ms, + "client.p99_ttft_ms": {"kind": "max_ms", "value": p99_ttft_max_ms}, + "client.mean_tpot_ms": {"kind": "max_ms", "value": mean_tpot_max_ms}, + "client.median_tpot_ms": loose_ms, + "client.p90_tpot_ms": loose_ms, + "client.p95_tpot_ms": {"kind": "max_ms", "value": p95_tpot_max_ms}, + "client.p99_tpot_ms": loose_ms, + "client.mean_itl_ms": loose_ms, + "client.median_itl_ms": loose_ms, + "client.p95_itl_ms": loose_ms, + "client.p99_itl_ms": loose_ms, + "client.mean_e2el_ms": loose_ms, + "client.median_e2el_ms": loose_ms, + "client.p90_e2el_ms": loose_ms, + "client.p95_e2el_ms": loose_ms, + "client.p99_e2el_ms": loose_ms, + "client.success_rate": {"kind": "min", "value": success_rate_min}, + "client.failed": {"kind": "max", "value": failed_max}, + } + + +def orchestrator_container_from_variant(variant: AtomVariantConfig) -> Dict[str, Any]: + """``container`` block for :class:`OrchestratorConfig` (includes server env).""" + block = variant.container.model_dump() + server_env = variant.roles.server.env + if server_env: + block = {**block, "env": dict(server_env)} + return block diff --git a/cvs/lib/inference/atom/atom_orch.py b/cvs/lib/inference/atom/atom_orch.py new file mode 100644 index 000000000..c4305470f --- /dev/null +++ b/cvs/lib/inference/atom/atom_orch.py @@ -0,0 +1,880 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +ATOM job driven by a ContainerOrchestrator (single- or multi-node). + +``params.driver=atom`` (target): ``atom.entrypoints.openai_server`` + +``atom.benchmarks.benchmark_serving`` with ATOM JSON artifacts. Standalone ATOM +has no native pipeline parallel; multinode ``atom`` uses SPMD data parallel +(``-dp`` + ``ATOM_DP_*``) when scale-out is needed. + +``params.driver=vllm_atom``: ``vllm serve`` + ``vllm bench serve`` with vLLM as +the multinode coordinator (``--pipeline-parallel-size``, ``--node-rank``, …) +while ATOM accelerates local kernels via ROCm vLLM env flags. + +``params.driver=sglang``: ``sglang.launch_server`` + ``sglang.bench_serving`` with +SGLang PP flags (``--pp-size``, ``--nnodes``, ``--dist-init-addr``). + +``params.driver=vllm`` (interim uplift): same coordinator path as ``vllm_atom`` +without the ATOM-specific ROCm env block. + +Does NOT subclass :class:`cvs.lib.inference.base.InferenceBaseJob`. +''' + +from __future__ import annotations + +import json +import re +import shlex +import time + +from cvs.lib import globals +from cvs.lib.inference.atom.atom_parsing import to_client_metrics + +log = globals.log + + +class AtomJob: + """ATOM benchmark job driven by an injected ContainerOrchestrator.""" + + READINESS_RE = re.compile(r"Application startup complete|Uvicorn running|Started server", re.I) + COMPLETION_RE = re.compile(r"Serving Benchmark Result", re.I) + FAILED_REQUESTS_RE = re.compile(r"Failed requests:\s+([0-9]+)", re.I) + CLIENT_CRASH_RE = re.compile(r"Traceback \(most recent call last\)", re.I) + CLIENT_LAUNCH_FAIL_RE = re.compile( + r"unrecognized arguments|invalid choice|error: argument |command not found|: No such file or directory", + re.I, + ) + EARLY_FAILURE_RE = re.compile( + r"no such file or directory|command not found|cannot access|failed to start" + r"|unrecognized arguments|invalid choice|error: argument " + r"|Free memory on device.*less than desired" + r"|Engine core initialization failed" + r"|WorkerProc failed to start", + re.I, + ) + FATAL_LOG_RE = re.compile( + r"Free memory on device.{0,80}less than desired" + r"|Engine core initialization failed" + r"|RuntimeError:.*[Ee]ngine", + re.I, + ) + + _DEFAULT_SERVE_ARGS = { + "block-size": 64, + "no-enable-prefix-caching": True, + } + + # vLLM multinode flags; ATOM openai_server rejects these (use ATOM_DP_* / -dp instead). + _VLLM_DISTRIBUTED_FLAGS = frozenset( + { + "--node-rank", + "--master-addr", + "--master-port", + "--nnodes", + "--pipeline-parallel-size", + "--distributed-executor-backend", + } + ) + + def __init__( + self, + orch, + variant, + hf_token, + isl, + osl, + concurrency, + num_prompts, + log_subdir="atom", + server_precheck_wait_s=30, + server_warmup_wait_s=330, + server_poll_count=60, + server_poll_wait_s=60, + client_initial_wait_s=120, + client_poll_count=50, + client_poll_wait_s=60, + bench_max_failed_requests=0, + ib_hcas=None, + ib_netdev=None, + ): + self.orch = orch + self.variant = variant + self.hf_token = hf_token + self.isl = str(isl) + self.osl = str(osl) + self.concurrency = str(concurrency) + self.num_prompts = str(num_prompts) + self.log_subdir = log_subdir + + p = variant.params + self.driver = str(p.driver or "atom").strip().lower() + self.tp = p.tensor_parallelism + self.pp = p.pipeline_parallel_size + self.nnodes = int(p.nnodes) + self.distributed = self.nnodes > 1 + raw_master = (p.master_addr or "").strip() + self.master_addr = raw_master or (orch.hosts[0] if getattr(orch, "hosts", None) else "localhost") + self.master_port = p.master_port + self.port_no = p.port_no + self.random_range_ratio = p.random_range_ratio + self.random_prefix_len = p.random_prefix_len + self.burstiness = p.burstiness + self.seed = p.seed + self.request_rate = p.request_rate + self.tokenizer_mode = p.tokenizer_mode + self.percentile_metrics = p.percentile_metrics + self.metric_percentiles = p.metric_percentiles + self.base_url = p.base_url + self.dataset_name = p.dataset_name + self.backend = p.backend + self.max_model_length = str(p.max_model_length) + self.bench_extra_args = (p.bench_extra_args or "").strip() + self.result_stem = (p.result_filename or "results").removesuffix(".json") + raw_baseline = (p.scaling_baseline_output_throughput or "").strip() + self._scaling_baseline = float(raw_baseline) if raw_baseline else None + + self.model_id = variant.model.id + self.log_dir = variant.paths.log_dir + self.models_dir = variant.paths.models_dir + self.serve_args = self._merged_serve_args(variant) + self.atom_server_args = list(variant.roles.server.atom_args) + self.sglang_server_args = list(variant.roles.server.sglang_args) + self.server_env = dict(variant.roles.server.env) + configured_netdev = (getattr(variant.roles.server, "ib_netdev", None) or "").strip() + if ib_netdev: + self.ib_netdev = str(ib_netdev).strip() + elif configured_netdev and configured_netdev.lower() != "auto": + self.ib_netdev = configured_netdev + else: + self.ib_netdev = "" + # Discovered HCA names for NCCL_IB_HCA (multinode only). Prefilled by + # test_discover_topology; build_server_cmd can resolve lazily if omitted. + self.ib_hcas = ib_hcas or [] + + self.out_dir = self._node_out_dir(0) + self.server_log = self._rank_server_log(0) + self.client_log = f"{self.out_dir}/client.log" + self._result_artifact = ( + f"{self.out_dir}/{self.result_stem}.json" if self.driver == "atom" else f"{self.out_dir}/{self.result_stem}" + ) + + self._precheck_wait = server_precheck_wait_s + self._warmup_wait = server_warmup_wait_s + self._server_poll_count = server_poll_count + self._server_poll_wait = server_poll_wait_s + self._client_initial_wait = client_initial_wait_s + self._client_poll_count = client_poll_count + self._client_poll_wait = client_poll_wait_s + self._bench_max_failed_requests = int(bench_max_failed_requests) + + @classmethod + def from_variant(cls, orch, variant, hf_token, isl, osl, concurrency, **overrides): + """Construct a job with server/client timing from ``variant.params``.""" + p = variant.params + + def _int_attr(name, default): + raw = getattr(p, name, None) + if raw is None or str(raw).strip() == "": + return default + try: + return int(raw) + except (TypeError, ValueError): + return default + + kw = dict( + orch=orch, + variant=variant, + hf_token=hf_token, + isl=isl, + osl=osl, + concurrency=concurrency, + num_prompts=p.num_prompts, + server_precheck_wait_s=_int_attr("server_precheck_wait_s", 30), + server_warmup_wait_s=_int_attr("server_warmup_wait_s", 330), + server_poll_count=_int_attr("server_poll_count", 60), + server_poll_wait_s=_int_attr("server_poll_wait_time", 60), + client_initial_wait_s=_int_attr("client_initial_wait_s", 120), + client_poll_count=_int_attr("client_poll_count", 50), + client_poll_wait_s=_int_attr("client_poll_wait_time", 60), + bench_max_failed_requests=_int_attr("bench_max_failed_requests", 0), + ) + kw.update(overrides) + return cls(**kw) + + def _node_out_dir(self, rank): + return f"{self.log_dir}/{self.log_subdir}/out-node{rank}/isl{self.isl}_osl{self.osl}_conc{self.concurrency}" + + def _uses_vllm_serve(self): + return self.driver in ("vllm", "vllm_atom") + + def _uses_sglang_serve(self): + return self.driver == "sglang" + + def _framework_coordinator_label(self): + if self.driver == "atom": + return "atom" + if self._uses_sglang_serve(): + return "sglang" + return "vllm" + + def _rank_server_log_name(self): + if self.driver == "atom": + return "atom_server.log" + if self._uses_sglang_serve(): + return "sglang_server.log" + return "vllm_serve_server.log" + + def _rank_server_log(self, rank): + base = self._node_out_dir(rank) + return f"{base}/{self._rank_server_log_name()}" + + def _exec_all(self, cmd, **kwargs): + return self.orch.exec(cmd, **kwargs) + + def _exec_head(self, cmd, **kwargs): + if self.distributed: + return self.orch.exec_on_head(cmd, **kwargs) + return self.orch.exec(cmd, **kwargs) + + def prepare_cell_out_dir(self): + """Create per-cell output directory without touching server env or cache.""" + if self.distributed: + for rank in range(self.nnodes): + self._exec_all(f"mkdir -p {shlex.quote(self._node_out_dir(rank))}") + else: + self._exec_all(f"mkdir -p {shlex.quote(self.out_dir)}") + + @classmethod + def _merged_serve_args(cls, variant): + merged = dict(cls._DEFAULT_SERVE_ARGS) + merged.update(variant.roles.server.serve_args) + env = variant.roles.server.env + gpu_mem = env.get("CVS_GPU_MEMORY_UTIL") or env.get("VLLM_GPU_MEMORY_UTIL") + if gpu_mem is not None and "gpu-memory-utilization" not in merged: + merged["gpu-memory-utilization"] = str(gpu_mem) + if "enforce-eager" not in merged: + merged["enforce-eager"] = True + return merged + + @staticmethod + def _flatten_serve_args(mapping): + argv = [] + for flag, value in mapping.items(): + opt = f"--{flag}" + if value is True: + argv.append(opt) + elif isinstance(value, (list, tuple)): + for v in value: + argv.extend([opt, str(v)]) + else: + argv.extend([opt, str(value)]) + return argv + + @staticmethod + def _argv_has_flag(argv, *names): + for tok in argv: + if tok in names: + return True + return False + + def _without_vllm_distributed_flags(self, argv): + """Strip vLLM multinode tokens from ``roles.server.atom_args`` if present.""" + out = [] + skip_next = False + for tok in argv: + if skip_next: + skip_next = False + continue + if tok in self._VLLM_DISTRIBUTED_FLAGS: + skip_next = True + continue + out.append(tok) + return out + + def _atom_spmd_dp_enabled(self): + """True when CVS should wire multinode ATOM SPMD data parallel (``-dp`` + ``ATOM_DP_*``).""" + if self.driver != "atom" or not self.distributed: + return False + atom_argv = self._without_vllm_distributed_flags(self.atom_server_args) + if self._argv_has_flag(atom_argv, "-dp", "--data-parallel-size"): + return False + return True + + def _atom_spmd_dp_cli(self): + """Return ``-dp nnodes`` for coupled multinode ATOM replicas (one DP rank per host).""" + if not self._atom_spmd_dp_enabled(): + return [] + tp = int(self.tp) + if tp > 8: + raise RuntimeError( + f"params.tensor_parallelism={tp} exceeds ATOM local TP limit (8); " + "multinode SPMD runs one TP group per node" + ) + return ["-dp", str(self.nnodes)] + + def _atom_multinode_argv(self): + """ATOM-only multinode CLI tokens (never vLLM ``--node-rank`` / ``--pipeline-parallel-size``).""" + return self._atom_spmd_dp_cli() + + def _atom_spmd_env_exports(self, rank): + if not self._atom_spmd_dp_enabled(): + return [] + return [ + f"export ATOM_DP_RANK={rank}", + f"export ATOM_DP_SIZE={self.nnodes}", + "export ATOM_DP_RANK_LOCAL=0", + f"export ATOM_DP_MASTER_IP={shlex.quote(self.master_addr)}", + f"export ATOM_DP_MASTER_PORT={self.master_port}", + ] + + def _vllm_distributed_argv(self, rank): + if not self.distributed: + return [] + argv = [ + "--node-rank", + str(rank), + "--master-addr", + str(self.master_addr), + "--master-port", + str(self.master_port), + "--nnodes", + str(self.nnodes), + "--pipeline-parallel-size", + str(self.pp), + "--distributed-executor-backend", + "mp", + ] + if rank > 0: + argv.append("--headless") + return argv + + def _ensure_multinode_topology(self): + if not self.distributed: + return + if self.ib_hcas and self.ib_netdev: + return + from cvs.lib.utils.ib_discovery import resolve_multinode_fabric + + roles = self.variant.roles.server + hcas, netdev = resolve_multinode_fabric( + self.orch, + ib_hca_devices=getattr(roles, "ib_hca_devices", None), + ib_netdev=self.ib_netdev or getattr(roles, "ib_netdev", None), + master_addr=self.master_addr, + ) + if not self.ib_hcas: + self.ib_hcas = hcas + if not self.ib_netdev: + self.ib_netdev = netdev + log.info( + "multinode fabric resolved: netdev=%s HCAs=%s", + self.ib_netdev, + self.ib_hcas, + ) + + def build_server_cmd(self, *, clear_atom_cache=True): + self._ensure_multinode_topology() + env_lines = [ + f"export HF_TOKEN={shlex.quote(self.hf_token)}", + f"export HF_HUB_CACHE={shlex.quote(self.models_dir)}", + ] + if self._uses_vllm_serve(): + env_lines.extend( + [ + "export VLLM_USE_AITER_UNIFIED_ATTENTION=1", + "export VLLM_ROCM_USE_AITER_MHA=0", + "export VLLM_ROCM_USE_AITER_FUSED_MOE_A16W4=1", + ] + ) + elif self._uses_sglang_serve(): + env_lines.append("export SGLANG_USE_AITER=1") + if self.ib_hcas: + env_lines.append(f"export NCCL_IB_HCA={shlex.quote(','.join(self.ib_hcas))}") + if self.distributed and not self.ib_netdev: + raise RuntimeError( + "multinode run has no socket netdev after topology resolution " + "(set roles.server.ib_netdev or fix cluster IP discovery)" + ) + if self.distributed and self.ib_netdev: + env_lines.append(f"export NCCL_SOCKET_IFNAME={shlex.quote(self.ib_netdev)}") + env_lines.append(f"export GLOO_SOCKET_IFNAME={shlex.quote(self.ib_netdev)}") + env_lines.append(f"export TP_SOCKET_IFNAME={shlex.quote(self.ib_netdev)}") + for k, v in self.server_env.items(): + if k in ( + "CVS_GPU_MEMORY_UTIL", + "VLLM_GPU_MEMORY_UTIL", + "VLLM_ENFORCE_EAGER", + "NCCL_SOCKET_IFNAME", + "GLOO_SOCKET_IFNAME", + "TP_SOCKET_IFNAME", + "NCCL_IB_HCA", + ): + continue + env_lines.append(f"export {k}={shlex.quote(str(v))}") + env_script = "\n".join(env_lines) + "\n" + self._exec_all("bash -c " + shlex.quote(f"printf '%s' {shlex.quote(env_script)} > /tmp/server_env_script.sh")) + if self.distributed: + for rank in range(self.nnodes): + self._exec_all(f"mkdir -p {shlex.quote(self._node_out_dir(rank))}") + else: + self._exec_all(f"mkdir -p {shlex.quote(self.out_dir)}") + if self.driver == "atom" and clear_atom_cache: + self._exec_all("bash -c 'rm -rf ~/.cache/atom/* 2>/dev/null || true'") + + def _server_argv(self, rank=0): + argv = [ + "vllm", + "serve", + self.model_id, + "--host", + "0.0.0.0", + "--tensor-parallel-size", + str(self.tp), + "--max-model-len", + self.max_model_length, + "--port", + str(self.port_no), + ] + argv.extend(self._vllm_distributed_argv(rank)) + argv.extend(self._flatten_serve_args(self.serve_args)) + return argv + + def _sglang_server_argv(self, rank=0): + argv = [ + "python3", + "-m", + "sglang.launch_server", + "--model-path", + self.model_id, + "--host", + "0.0.0.0", + "--port", + str(self.port_no), + "--tp", + str(self.tp), + ] + if self.distributed: + dist_init = f"{self.master_addr}:{self.master_port}" + argv.extend( + [ + "--pp-size", + str(self.pp), + "--nnodes", + str(self.nnodes), + "--node-rank", + str(rank), + "--dist-init-addr", + dist_init, + ] + ) + argv.extend(self.sglang_server_args) + return argv + + def _server_argv_for_driver(self, rank=0): + if self.driver == "atom": + return self._atom_server_argv(rank) + if self._uses_vllm_serve(): + return self._server_argv(rank) + if self._uses_sglang_serve(): + return self._sglang_server_argv(rank) + raise RuntimeError( + f"unsupported params.driver={self.driver!r}; expected 'atom', 'vllm', 'vllm_atom', or 'sglang'" + ) + + def _atom_server_argv(self, rank=0): + argv = [ + "python", + "-m", + "atom.entrypoints.openai_server", + "--model", + self.model_id, + "--server-port", + str(self.port_no), + ] + argv.extend(self._without_vllm_distributed_flags(self.atom_server_args)) + argv.extend(self._atom_multinode_argv()) + return argv + + def start_server(self): + hosts = list(getattr(self.orch, "hosts", []) or ["node0"]) + if self.distributed and len(hosts) != self.nnodes: + raise RuntimeError( + f"params.nnodes={self.nnodes} but cluster has {len(hosts)} host(s); " + "align cluster node_dict with params.nnodes" + ) + label = self._framework_coordinator_label() + launch_hosts = enumerate(hosts) if self.distributed else [(0, hosts[0])] + for rank, host in launch_hosts: + argv = self._server_argv_for_driver(rank) + serve_cmd = " ".join(shlex.quote(str(a)) for a in argv) + rank_log = self._rank_server_log(rank) + rank_env = " && ".join(self._atom_spmd_env_exports(rank)) + env_prefix = f"{rank_env} && " if rank_env else "" + inner = ( + f"source /tmp/server_env_script.sh && {env_prefix}nohup {serve_cmd} > {shlex.quote(rank_log)} 2>&1 &" + ) + if self.distributed: + out = self._exec_all("bash -c " + shlex.quote(inner), hosts=[host]) + else: + out = self._exec_all("bash -c " + shlex.quote(inner)) + for h, output in out.items(): + if self.EARLY_FAILURE_RE.search(output or ""): + raise RuntimeError(f"{label} server failed to launch on {h} (rank {rank}): {output[-500:]}") + + def _atom_health_ok(self): + url = f"http://localhost:{self.port_no}/health" + probe = f"curl -sf {shlex.quote(url)} -o /dev/null && echo OK || echo NO" + if self.distributed: + out = self._exec_all("bash -c " + shlex.quote(probe)) + else: + out = self._exec_head("bash -c " + shlex.quote(probe)) + return bool(out) and all("OK" in (v or "") for v in out.values()) + + def _atom_warmup_ok(self): + payload = json.dumps( + {"model": self.model_id, "prompt": "hi", "max_tokens": 1}, + separators=(",", ":"), + ) + url = f"http://localhost:{self.port_no}/v1/completions" + inner = ( + f"curl -sf {shlex.quote(url)} -H 'Content-Type: application/json' " + f"-d {shlex.quote(payload)} -o /dev/null --max-time 120 && echo OK || echo NO" + ) + out = self._exec_head("bash -c " + shlex.quote(inner)) + return bool(out) and all("OK" in (v or "") for v in out.values()) + + def is_ready(self): + if self.driver == "atom": + return self._atom_health_ok() + pattern = self.READINESS_RE.pattern + for rank, host in enumerate(self.orch.hosts): + # Headless workers (rank > 0) never log Uvicorn startup; only the head + # API server does. Match vllm_job.is_ready() multinode behaviour. + if rank > 0 and self.nnodes > 1: + continue + rank_log = self._rank_server_log(rank) if self.distributed else self.server_log + out = self.orch.exec( + f"grep -qiE {shlex.quote(pattern)} {shlex.quote(rank_log)}", + detailed=True, + hosts=[host], + ) + if not out or not all(r["exit_code"] == 0 for r in out.values()): + return False + return True + + def _check_coordinator_early_failure(self, emit_tail: bool = False): + """Tail/grep per-rank server logs on each host for fatal startup errors.""" + label = self._framework_coordinator_label() + for rank, host in enumerate(self.orch.hosts): + rank_log = self._rank_server_log(rank) if self.distributed else self.server_log + out = self.orch.exec(f"tail -30 {shlex.quote(rank_log)}", hosts=[host]) + for h, output in (out or {}).items(): + if emit_tail: + for line in (output or "").splitlines(): + log.info("[%s rank%d server.log] %s", h, rank, line) + if self.EARLY_FAILURE_RE.search(output or ""): + raise RuntimeError(f"{label} server early failure on {h} (rank {rank}): {(output or '')[-500:]}") + out = self.orch.exec( + f"grep -m1 -iE {shlex.quote(self.FATAL_LOG_RE.pattern)} {shlex.quote(rank_log)}", + detailed=True, + hosts=[host], + ) + for h, r in (out or {}).items(): + if r.get("exit_code") == 0 and r.get("output", "").strip(): + raise RuntimeError(f"{label} server fatal error on {h} (rank {rank}): {r['output'].strip()[-500:]}") + + def _tail_server_logs(self, lines=30): + if self.distributed: + out = {} + for rank in range(self.nnodes): + chunk = self._exec_all(f"tail -{lines} {shlex.quote(self._rank_server_log(rank))}") + out.update(chunk or {}) + return out + return self._exec_all(f"tail -{lines} {shlex.quote(self.server_log)}") + + def wait_ready(self): + log.info("waiting %ds for server log to materialise", self._precheck_wait) + time.sleep(self._precheck_wait) + + if self.driver == "atom": + out = self._tail_server_logs(30) + for host, output in out.items(): + if self.EARLY_FAILURE_RE.search(output or ""): + raise RuntimeError(f"atom server early failure on {host}: {output[-500:]}") + else: + self._check_coordinator_early_failure(emit_tail=True) + + log.info("warmup wait %ds", self._warmup_wait) + time.sleep(self._warmup_wait) + + if self.driver == "atom": + out = self._tail_server_logs(30) + for host, output in out.items(): + if self.EARLY_FAILURE_RE.search(output or ""): + raise RuntimeError(f"atom server early failure on {host}: {output[-500:]}") + else: + self._check_coordinator_early_failure(emit_tail=True) + + for it in range(self._server_poll_count): + log.info("readiness poll iter=%d/%d", it, self._server_poll_count - 1) + if self.is_ready(): + log.info("server health ready (iter=%d)", it) + break + if self.driver == "atom": + poll_out = self._tail_server_logs(30) + for host, output in poll_out.items(): + if self.EARLY_FAILURE_RE.search(output or ""): + raise RuntimeError(f"atom server early failure on {host}: {output[-500:]}") + else: + self._check_coordinator_early_failure() + time.sleep(self._server_poll_wait) + else: + raise RuntimeError("server did not become ready before timeout") + + if self.driver == "atom": + for it in range(10): + if self._atom_warmup_ok(): + log.info("server warmup complete (iter=%d)", it) + return + time.sleep(30) + raise RuntimeError("atom server warmup did not complete before timeout") + + def stop_server(self): + if self.driver == "atom": + log.info("stopping atom server") + self._exec_all( + "bash -c " + + shlex.quote("pkill -f 'atom.entrypoints.openai_server' || pkill -f 'openai_server' || true") + ) + elif self._uses_sglang_serve(): + log.info("stopping sglang server") + self._exec_all("bash -c 'pkill -f \"sglang.launch_server\" || true'") + else: + log.info("stopping vllm server") + self._exec_all("bash -c 'pkill -f \"vllm serve\" || true'") + time.sleep(5) + + def _atom_client_argv(self): + warmups = int(self.concurrency) * 2 + argv = [ + "python", + "-m", + "atom.benchmarks.benchmark_serving", + "--model", + self.model_id, + "--backend", + "vllm", + "--base-url", + f"http://localhost:{self.port_no}", + "--dataset-name", + self.dataset_name, + "--random-input-len", + self.isl, + "--random-output-len", + self.osl, + "--random-range-ratio", + self.random_range_ratio, + "--max-concurrency", + self.concurrency, + "--num-prompts", + self.num_prompts, + "--trust-remote-code", + "--num-warmups", + str(warmups), + "--request-rate", + self.request_rate, + "--ignore-eos", + "--save-result", + "--percentile-metrics", + self.percentile_metrics, + "--result-dir", + self.out_dir, + "--result-filename", + f"{self.result_stem}.json", + ] + if self.bench_extra_args: + argv.extend(shlex.split(self.bench_extra_args)) + return argv + + def _sglang_client_argv(self): + return [ + "python3", + "-m", + "sglang.bench_serving", + "--backend", + "sglang", + "--host", + "0.0.0.0", + "--port", + str(self.port_no), + "--dataset-name", + self.dataset_name, + "--num-prompts", + self.num_prompts, + "--random-input", + self.isl, + "--random-output", + self.osl, + "--random-range-ratio", + self.random_range_ratio, + "--max-concurrency", + self.concurrency, + "--request-rate", + self.request_rate, + ] + + def _client_argv(self): + if self.driver == "atom": + return self._atom_client_argv() + if self._uses_sglang_serve(): + return self._sglang_client_argv() + return self._vllm_client_argv() + + def _vllm_client_argv(self): + return [ + "vllm", + "bench", + "serve", + "--model", + self.model_id, + "--backend", + self.backend, + "--base-url", + f"{self.base_url}:{self.port_no}", + "--dataset-name", + self.dataset_name, + "--num-prompts", + self.num_prompts, + "--random-input-len", + self.isl, + "--random-output-len", + self.osl, + "--max-concurrency", + self.concurrency, + "--request-rate", + self.request_rate, + "--burstiness", + self.burstiness, + "--tokenizer-mode", + self.tokenizer_mode, + "--seed", + self.seed, + "--random-range-ratio", + self.random_range_ratio, + "--random-prefix-len", + self.random_prefix_len, + "--percentile-metrics", + self.percentile_metrics, + "--metric-percentiles", + self.metric_percentiles, + "--ignore-eos", + "--save-result", + "--result-dir", + self.out_dir, + "--result-filename", + self.result_stem, + ] + + def _clear_stale_result_artifact(self): + """Remove a prior run's result file so poll logic cannot treat it as complete.""" + artifact = shlex.quote(self._result_artifact) + self._exec_head(f"rm -f {artifact}") + + def run_client(self): + self._clear_stale_result_artifact() + args = self._client_argv() + bench_cmd = " ".join(shlex.quote(str(a)) for a in args) + client_cmd = f"source /tmp/server_env_script.sh && {bench_cmd} > {shlex.quote(self.client_log)} 2>&1 &" + self._exec_head("bash -c " + shlex.quote(client_cmd)) + + def _atom_result_ready(self): + out = self._exec_head(f"test -s {shlex.quote(self._result_artifact)} && echo OK || echo NO") + return bool(out) and all("OK" in (v or "") for v in out.values()) + + def _client_log_failures(self, tail_lines=2000): + out = self._exec_head(f"tail -{tail_lines} {shlex.quote(self.client_log)}") + failed = [] + for host, output in out.items(): + txt = output or "" + if self.CLIENT_CRASH_RE.search(txt) or self.CLIENT_LAUNCH_FAIL_RE.search(txt): + failed.append((host, txt[-500:])) + continue + fm = self.FAILED_REQUESTS_RE.search(txt) + if fm: + fc = int(fm.group(1)) + cap = self._bench_max_failed_requests + if fc > cap: + failed.append((host, f"Failed requests: {fc} (cap {cap}) -- {txt[-500:]}")) + elif fc > 0: + log.warning( + "client on %s completed with %d failed requests (allowed up to %d)", + host, + fc, + cap, + ) + return failed + + def wait_client_complete(self): + if self.driver == "atom": + log.info( + "client initial wait (atom: polling for result artifact, up to %ds)", + self._client_initial_wait, + ) + deadline = time.monotonic() + self._client_initial_wait + poll_s = 15 + while time.monotonic() < deadline: + failed = self._client_log_failures(tail_lines=500) + if failed: + raise RuntimeError("client failed: " + "; ".join(f"{h}: {m}" for h, m in failed)) + if self._atom_result_ready(): + log.info("client result artifact ready during initial wait") + return + remaining = deadline - time.monotonic() + if remaining <= 0: + break + time.sleep(min(poll_s, remaining)) + else: + log.info("client initial wait %ds", self._client_initial_wait) + time.sleep(self._client_initial_wait) + + for it in range(self._client_poll_count): + failed = self._client_log_failures() + if failed: + raise RuntimeError("client failed: " + "; ".join(f"{h}: {m}" for h, m in failed)) + if self.driver == "atom": + if self._atom_result_ready(): + log.info("client complete (iter=%d)", it) + return + else: + out = self._exec_head(f"tail -2000 {shlex.quote(self.client_log)}") + done = [bool(self.COMPLETION_RE.search(txt or "")) for txt in out.values()] + if done and all(done): + log.info("client complete (iter=%d)", it) + return + time.sleep(self._client_poll_wait) + raise RuntimeError("client did not complete before poll cap") + + def parse_results(self): + out = self._exec_head(f"cat {shlex.quote(self._result_artifact)}") + results = {} + for host, text in out.items(): + text = (text or "").strip() + if not text: + raise RuntimeError(f"empty/missing results artifact on {host}: {self._result_artifact}") + try: + raw = json.loads(text) + except (json.JSONDecodeError, ValueError) as e: + raise RuntimeError(f"unparseable results artifact on {host}: {self._result_artifact}: {e}") from e + if self.driver == "atom": + raw.setdefault("random_input_len", int(self.isl)) + raw.setdefault("random_output_len", int(self.osl)) + results[host] = to_client_metrics( + raw, + tp=self.tp, + isl=self.isl, + scaling_baseline_output_throughput=self._scaling_baseline, + nnodes=self.nnodes, + ) + return results diff --git a/cvs/lib/inference/atom/atom_parsing.py b/cvs/lib/inference/atom/atom_parsing.py new file mode 100644 index 000000000..d9f69e208 --- /dev/null +++ b/cvs/lib/inference/atom/atom_parsing.py @@ -0,0 +1,117 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +ATOM metric vocabulary and parsers. + +ATOM ``benchmark_serving`` emits the same JSON scalar keys as stock vLLM bench, +so base parsing reuses :func:`vllm_parsing.to_client_metrics`. W1 gates +(``per_gpu_throughput``, ``output_tput_per_gpu``, tail percentiles) live here — +not in the vLLM single-node ``GATED_METRICS`` set (vLLM parity is a separate track). +''' + +from __future__ import annotations + +from cvs.lib.inference.utils.vllm_parsing import ( + CLIENT_METRICS as _VLLM_CLIENT_METRICS, + GATED_METRICS as _VLLM_GATED_METRICS, + _safe_div, + to_client_metrics as _vllm_to_client_metrics, +) + +_METRIC_INSERT = ("output_tput_per_gpu", "tok/s") +_idx = next( + (i for i, (n, _) in enumerate(_VLLM_CLIENT_METRICS) if n == "per_gpu_throughput"), + None, +) +CLIENT_METRICS = list(_VLLM_CLIENT_METRICS) +CLIENT_METRICS.insert( + (_idx + 1) if _idx is not None else len(CLIENT_METRICS), + _METRIC_INSERT, +) +CLIENT_METRIC_UNITS = dict(CLIENT_METRICS) + +# W1 perf gates: vLLM baseline set plus per-GPU throughput derivations. +GATED_METRICS = frozenset(_VLLM_GATED_METRICS) | { + "per_gpu_throughput", + "output_tput_per_gpu", +} + +# W1 metric tiers for ``test_cell_metrics`` (one parent row per cell × tier). +METRIC_TIERS: dict[str, tuple[str, ...]] = { + "throughput": ( + "total_token_throughput", + "output_throughput", + "per_gpu_throughput", + "output_tput_per_gpu", + ), + "ttft": ( + "mean_ttft_ms", + "p99_ttft_ms", + ), + "tpot": ( + "mean_tpot_ms", + "p99_tpot_ms", + ), + "health": ( + "success_rate", + "failed", + ), + "scaling": ("efficiency_pct",), +} + +SCALING_METRICS: tuple[str, ...] = METRIC_TIERS["scaling"] +SCALING_METRIC_UNITS: dict[str, str] = {"efficiency_pct": "%"} + +METRIC_TIER_ORDER: tuple[str, ...] = tuple(METRIC_TIERS.keys()) + ("record",) + +_tiered = {m for names in METRIC_TIERS.values() for m in names} +RECORD_METRICS: tuple[str, ...] = tuple(short for short, _unit in CLIENT_METRICS if short not in _tiered) + +ENFORCED_METRICS = frozenset(_tiered) + + +def scaling_efficiency_pct(actual_output_throughput, *, baseline_single_node, nnodes): + """Linear scaling efficiency: actual / (single-node baseline × nnodes).""" + denom = _safe_div(baseline_single_node, 1) + if denom is None or int(nnodes) < 1: + return None + ideal = denom * int(nnodes) + if ideal <= 0: + return None + return _safe_div(actual_output_throughput, ideal) + + +def to_client_metrics(raw, *, tp, isl, scaling_baseline_output_throughput=None, nnodes=1): + """Map an ATOM ``results.json`` dict to the ``client.*`` namespace.""" + m = _vllm_to_client_metrics(raw, tp=tp, isl=isl) + m["client.output_tput_per_gpu"] = _safe_div(raw.get("output_throughput"), tp) + if scaling_baseline_output_throughput is not None: + eff = scaling_efficiency_pct( + raw.get("output_throughput"), + baseline_single_node=scaling_baseline_output_throughput, + nnodes=nnodes, + ) + if eff is not None: + m["scaling.efficiency_pct"] = eff * 100.0 + return m + + +def tier_metric_specs(thresholds_cell: dict, tier: str) -> dict[str, dict]: + """Return threshold specs for one tier in a sweep cell.""" + if tier == "record": + names = RECORD_METRICS + prefix = "client." + elif tier == "scaling": + names = SCALING_METRICS + prefix = "scaling." + else: + names = METRIC_TIERS.get(tier, ()) + prefix = "client." + specs = {} + for short in names: + full = f"{prefix}{short}" + spec = thresholds_cell.get(full) + if spec is not None: + specs[full] = spec + return specs diff --git a/cvs/lib/inference/base.py b/cvs/lib/inference/base.py index d735f1658..7c2bf0c22 100644 --- a/cvs/lib/inference/base.py +++ b/cvs/lib/inference/base.py @@ -7,9 +7,14 @@ import os import re +import shlex import time from cvs.lib import globals +from cvs.lib.inference.utils.vllm_benchmark_scripts import ( + bash_export_bench_script_from_vllm_install, + clamped_bench_random_range_ratio_str, +) from cvs.lib.utils_lib import * from cvs.lib.verify_lib import * from cvs.lib import linux_utils @@ -45,7 +50,9 @@ def __init__( hf_token, gpu_type='mi300', distributed_inference=False, - server_launch_poll_count=20, + # 60 * 60s polls after warmup matches VllmJob: large HF model cache + weight load + # on MI300 can exceed 20min with little log churn before Uvicorn prints ready. + server_launch_poll_count=60, ): # Client instance phdl self.c_phdl = c_phdl @@ -75,6 +82,7 @@ def __init__( self.rdma_stats_dict_after = {} self.inference_start_time = s_phdl.exec('date +"%a %b %e %H:%M"') self.inference_end_time = None + self.inference_results_dict = {} self.home_dir = os.path.expanduser("~") self.if_dict.setdefault('container_image', 'rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1') @@ -90,7 +98,6 @@ def __init__( self.if_dict.setdefault('nccl_debug', 'ERROR') self.if_dict.setdefault('data_cache_dir', f'{self.home_dir}/cache') self.if_dict.setdefault('log_dir', f'{self.home_dir}/LOG_DIR') - self.if_dict.setdefault('benchmark_script_repo', 'https://github.com/kimbochen/bench_serving.git') log.info('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%') log.info(f'inference_dict = {self.if_dict}') @@ -138,8 +145,6 @@ def __init__( 'failed to start|no such file or directory|command not found|cannot access', re.I ) self.default_client_wait_time = 120 - self.default_client_poll_count = 20 - self.default_client_poll_wait_time = 60 # Regex/parse defaults that derived classes may override self.readiness_pattern = re.compile('Application startup complete|Uvicorn running|Started server', re.I) @@ -157,13 +162,32 @@ def __init__( self.bp_dict.setdefault('seed', '0') self.bp_dict.setdefault('request_rate', 'inf') self.bp_dict.setdefault('max_model_length', '9216') - self.bp_dict.setdefault('random_range_ration', '1.0') + self.bp_dict.setdefault('random_range_ratio', '1.0') self.bp_dict.setdefault('random_prefix_len', '0') self.bp_dict.setdefault('tensor_parallelism', '1') self.bp_dict.setdefault('port_no', '8000') self.bp_dict.setdefault('tokenizer_mode', 'auto') self.bp_dict.setdefault('percentile_metrics', 'ttft,tpot,itl,e2el') self.bp_dict.setdefault('metric_percentiles', '99') + # Bench client can exceed 20min for large num_prompts × long ISL/OSL; budget is + # default_client_wait_time + client_poll_count * client_poll_wait_time. + self.bp_dict.setdefault('client_poll_count', '50') + self.bp_dict.setdefault('client_poll_wait_time', '60') + self.bp_dict.setdefault('bench_max_failed_requests', '0') + try: + self.default_client_poll_count = max(1, int(float(str(self.bp_dict['client_poll_count']).strip()))) + except (TypeError, ValueError): + self.default_client_poll_count = 50 + try: + self.default_client_poll_wait_time = max(1, int(float(str(self.bp_dict['client_poll_wait_time']).strip()))) + except (TypeError, ValueError): + self.default_client_poll_wait_time = 60 + try: + self.bench_max_failed_requests_cap = max( + 0, int(float(str(self.bp_dict['bench_max_failed_requests']).strip())) + ) + except (TypeError, ValueError): + self.bench_max_failed_requests_cap = 0 # Set server and client scripts self.server_script = self.bp_dict['server_script'] @@ -247,6 +271,9 @@ def exec_nic_setup_scripts( def build_server_inference_job_cmd( self, ): + eager_line = ( + "\n export VLLM_ENFORCE_EAGER=1" if self.if_dict.get("vllm_enforce_eager") else "" + ) s_cmd = f'''docker exec {self.container_name} /bin/bash -c "echo ' export MODEL={self.bp_dict['model']} export ISL={self.bp_dict['input_sequence_length']} @@ -258,7 +285,7 @@ def build_server_inference_job_cmd( export HF_TOKEN={self.hf_token} export VLLM_USE_AITER_UNIFIED_ATTENTION=1 export VLLM_ROCM_USE_AITER_MHA=0 - export VLLM_ROCM_USE_AITER_FUSED_MOE_A16W4=1 + export VLLM_ROCM_USE_AITER_FUSED_MOE_A16W4=1{eager_line} export RESULT_FILENAME=results export PORT={self.bp_dict['port_no']}' > /tmp/server_env_script.sh" ''' @@ -298,14 +325,10 @@ def build_server_inference_job_cmd( self.s_phdl.exec_cmd_list(cmd_list) def clone_bench_serving_repo(self, clone_dir): - """Clone bench_serving repository for client benchmarks.""" - cmd = f'''docker exec {self.container_name} /bin/bash -c "cd {clone_dir}; git clone {self.if_dict['benchmark_script_repo']}" ''' - out_dict = self.c_phdl.exec(cmd) - for node in out_dict.keys(): - # Ignore "already exists" error - repo was cloned in previous test - if re.search('error|fail', out_dict[node], re.I) and not re.search('already exists', out_dict[node], re.I): - fail_test('Errors or failures seen in pulling bench_serving repo from Github, pls check') - time.sleep(3) + """No-op: client benchmarks use the installed ``vllm`` package ``benchmarks/``.""" + log.info( + "clone_bench_serving_repo skipped; using vLLM-shipped benchmarks/ (no third-party bench_serving clone)" + ) def launch_server(self): """Launch inference server.""" @@ -352,10 +375,26 @@ def is_server_ready(self, out_dict, readiness_pattern): node_ready = {node: bool(readiness_pattern.search(output or '')) for node, output in out_dict.items()} return bool(node_ready) and all(node_ready.values()) + def _readiness_grep_cmd_list(self, log_file: str) -> list[str]: + """Remote bash lines: print CVS_SERVER_READY if the full server log matches readiness. + + Uses grep on the whole file (not tail) so the marker is not lost once vLLM logs scroll. + """ + pat = self.readiness_pattern.pattern + cmd_list: list[str] = [] + for i in range(0, int(self.nnodes)): + path = f'{self.log_dir}/{self.get_log_subdir()}/out-node{i}/{log_file}'.replace('\\', '/') + inner = f'grep -qiE {shlex.quote(pat)} {shlex.quote(path)} && echo CVS_SERVER_READY || true' + cmd_list.append(f'bash -c {shlex.quote(inner)}') + return cmd_list + + @staticmethod + def _grep_readiness_outputs_ok(out_dict: dict) -> bool: + return bool(out_dict) and all('CVS_SERVER_READY' in (output or '') for output in out_dict.values()) + def poll_server_startup(self): """Poll for server startup completion.""" log_file = f'{self.server_script}_server.log' - readiness_pattern = self.readiness_pattern # Do an early check for fast failures before the long wait log.info(f'Waiting {self.default_server_precheck_wait_time} secs for server to start writing logs...') @@ -381,11 +420,10 @@ def poll_server_startup(self): for j in range(0, self.default_server_poll_count): log.info(f'Polling for application startup complete on all nodes, iteration {j}') - cmd_list = [] + tail_cmds = [] for i in range(0, int(self.nnodes)): - cmd = f'tail -30 {self.log_dir}/{self.get_log_subdir()}/out-node{i}/{log_file}' - cmd_list.append(cmd) - out_dict = self.s_phdl.exec_cmd_list(cmd_list) + tail_cmds.append(f'tail -30 {self.log_dir}/{self.get_log_subdir()}/out-node{i}/{log_file}') + out_dict = self.s_phdl.exec_cmd_list(tail_cmds) for node in out_dict.keys(): if self.default_server_error_pattern_poll.search(out_dict[node] or ''): @@ -393,7 +431,8 @@ def poll_server_startup(self): fail_test(error_msg) raise Exception(error_msg) - if self.is_server_ready(out_dict, readiness_pattern): + grep_out = self.s_phdl.exec_cmd_list(self._readiness_grep_cmd_list(log_file)) + if self._grep_readiness_outputs_ok(grep_out): log.info('Server startup confirmed on all nodes') return @@ -410,11 +449,30 @@ def launch_client(self): backend = self.bp_dict['backend'] result_filename = self.get_result_filename() + export_bench = bash_export_bench_script_from_vllm_install(self.bench_serv_script) + + rr_str, rr_clamped = clamped_bench_random_range_ratio_str( + self.bp_dict["random_range_ratio"], + self.bp_dict["input_sequence_length"], + self.bp_dict["output_sequence_length"], + self.bp_dict["max_model_length"], + ) + if rr_clamped: + log.info( + "CVS: clamped --random-range-ratio from %s to %s so peak random (ISL+OSL)*(1+r) " + "fits max_model_length=%s (ISL=%s OSL=%s)", + self.bp_dict["random_range_ratio"], + rr_str, + self.bp_dict["max_model_length"], + self.bp_dict["input_sequence_length"], + self.bp_dict["output_sequence_length"], + ) + # Launch client benchmark cmd_list = [] for i in range(0, int(self.nnodes)): - client_cmd = f'''source /tmp/server_env_script.sh; cd {clone_dir}; \ - python3 bench_serving/{self.bench_serv_script} \ + client_cmd = f'''source /tmp/server_env_script.sh; {export_bench}; cd {clone_dir}; \ + _cvs_run_bench \ --model {self.bp_dict['model']} \ --backend {backend} \ --base-url {self.bp_dict['base_url']}:{self.bp_dict['port_no']} \ @@ -427,15 +485,17 @@ def launch_client(self): --burstiness {self.bp_dict['burstiness']} \ --tokenizer-mode {self.bp_dict['tokenizer_mode']} \ --seed {self.bp_dict['seed']} \ - --random-range-ratio {self.bp_dict['random_range_ratio']} \ + --random-range-ratio {rr_str} \ --random-prefix-len {self.bp_dict['random_prefix_len']} \ --percentile-metrics {self.bp_dict['percentile_metrics']} \ + --metric-percentiles {self.bp_dict['metric_percentiles']} \ + --temperature 0 \ --ignore-eos \ --save-result \ --result-dir {self.log_dir}/{self.get_log_subdir()}/out-node{i} \ --result-filename {result_filename} \ > {self.log_dir}/{self.get_log_subdir()}/out-node{i}/bench_serv_script.log 2>&1 &''' - cmd = f'''docker exec {self.container_name} /bin/bash -c "{client_cmd}" ''' + cmd = f"docker exec {shlex.quote(str(self.container_name))} /bin/bash -c {shlex.quote(client_cmd)}" cmd_list.append(cmd) self.c_phdl.exec_cmd_list(cmd_list) @@ -450,13 +510,35 @@ def poll_client_completion(self): cmd = f'tail -30 {self.log_dir}/{self.get_log_subdir()}/out-node{i}/bench_serv_script.log' cmd_list.append(cmd) out_dict = self.c_phdl.exec_cmd_list(cmd_list) + done = [] for node in out_dict.keys(): - if re.search('Failed', out_dict[node], re.I): + log_tail = out_dict[node] or '' + if re.search(r"can't open file|No such file or directory", log_tail, re.I): + fail_test( + f'Benchmark script missing or unreadable on node {node} ' + f'(see bench_serv_script.log); install vllm[bench] or use an image with benchmarks/.' + ) + return + if re.search('Failed', log_tail, re.I): fail_test(f'Failed to run benchmark script on node {node}') return - if not re.search('End-to-end Latency', out_dict[node], re.I): - log.info(f'Waiting {self.default_client_poll_wait_time} secs for next poll') - time.sleep(self.default_client_poll_wait_time) + done.append( + bool( + re.search( + r'Serving Benchmark Result|End-to-end Latency', + log_tail, + re.I, + ) + ) + ) + if done and all(done): + log.info('Benchmark client complete on all nodes (iter=%d)', j) + return + log.info(f'Waiting {self.default_client_poll_wait_time} secs for next poll') + time.sleep(self.default_client_poll_wait_time) + msg = 'client did not complete before poll cap' + fail_test(msg) + raise Exception(msg) def start_inference_server_job( self, @@ -471,7 +553,7 @@ def start_inference_client_job( ): log.info('Start Client side benchmark script on all Nodes') - # Clone bench_serving repo to /app + # Resolve benchmark driver from the installed vllm package (see inference.utils.vllm_benchmark_scripts) self.clone_bench_serving_repo('/app') if self.distributed_inference: @@ -498,7 +580,7 @@ def get_inference_results_dict(self, out_dict): self.inference_results_dict[node]['total_input_tokens'] = match.group(1) if re.search('Total generated tokens:', out_dict[node], re.I): match = re.search('Total generated tokens:\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['Total generated tokens:'] = match.group(1) + self.inference_results_dict[node]['total_generated_tokens'] = match.group(1) if re.search('Request throughput \(req/s\):', out_dict[node], re.I): match = re.search('Request throughput \(req/s\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['request_throughput_per_sec'] = match.group(1) @@ -511,20 +593,20 @@ def get_inference_results_dict(self, out_dict): if re.search('Mean TTFT \(ms\):', out_dict[node], re.I): match = re.search('Mean TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['mean_ttft_ms'] = match.group(1) - if re.search('Median TTFT (ms):', out_dict[node], re.I): - match = re.search('Median TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) + if re.search(r'Median TTFT \(ms\):', out_dict[node], re.I): + match = re.search(r'Median TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['median_ttft_ms'] = match.group(1) - if re.search('P99 TTFT (ms):', out_dict[node], re.I): - match = re.search('P99 TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) + if re.search(r'P99 TTFT \(ms\):', out_dict[node], re.I): + match = re.search(r'P99 TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['p99_ttft_ms'] = match.group(1) if re.search('Mean TPOT \(ms\)', out_dict[node], re.I): match = re.search('Mean TPOT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['mean_tpot_ms'] = match.group(1) if re.search('Median TPOT \(ms\):', out_dict[node], re.I): - match = re.search('Median TPOT \(ms\):\s+([0-9]+)', out_dict[node], re.I) + match = re.search('Median TPOT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['median_tpot_ms'] = match.group(1) - if re.search('P99 TPOT (ms):', out_dict[node], re.I): - match = re.search('P99 TPOT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) + if re.search(r'P99 TPOT \(ms\):', out_dict[node], re.I): + match = re.search(r'P99 TPOT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) self.inference_results_dict[node]['p99_tpot_ms'] = match.group(1) if re.search('Mean ITL \(ms\):', out_dict[node], re.I): match = re.search('Mean ITL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) diff --git a/cvs/lib/inference/inference_max.py b/cvs/lib/inference/inference_max.py deleted file mode 100644 index 61f91844d..000000000 --- a/cvs/lib/inference/inference_max.py +++ /dev/null @@ -1,56 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import re -import time - -from cvs.lib.inference.base import InferenceBaseJob -from cvs.lib.verify_lib import fail_test - - -class InferenceMaxJob(InferenceBaseJob): - """InferenceMAX-specific implementation.""" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.if_dict.setdefault('inferencemax_repo', 'https://github.com/InferenceMAX/InferenceMAX.git') - - def get_server_script_directory(self): - """InferenceMAX scripts are in the cloned repo.""" - return '/app/InferenceX' - - def get_server_script_path(self): - """InferenceMAX scripts are in the cloned repo.""" - base = "single_node" if int(self.nnodes) == 1 else "multi_node" - return f'benchmarks/{base}/{self.server_script}' - - def get_result_filename(self): - """InferenceMAX result filename.""" - return 'inferencemax_test_result.json' - - def get_completion_pattern(self): - """InferenceMAX completion pattern.""" - return re.compile('Serving Benchmark Result', re.I) - - def get_log_subdir(self): - """InferenceMAX uses 'inference-max' log subdirectory.""" - return 'inference-max' - - def clone_inferencemax_repo(self): - """Clone InferenceMAX repository.""" - cmd = f'''docker exec {self.container_name} /bin/bash -c "git clone {self.if_dict['inferencemax_repo']}" ''' - out_dict = self.s_phdl.exec(cmd) - for node in out_dict.keys(): - if re.search('error|fail', out_dict[node], re.I): - fail_test('Errors or failures seen in pulling InferenceMAX repo from Github, pls check') - time.sleep(3) - self.s_phdl.exec(f'''docker exec {self.container_name} /bin/bash -c "ls -ld /app/InferenceX" ''') - - def start_inference_server_job(self): - """Start InferenceMAX server - clone repo, then call base implementation.""" - self.clone_inferencemax_repo() - super().start_inference_server_job() diff --git a/cvs/lib/inference/sglang/__init__.py b/cvs/lib/inference/sglang/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cvs/lib/inference/sglang/sglang_common.py b/cvs/lib/inference/sglang/sglang_common.py new file mode 100644 index 000000000..6108fd362 --- /dev/null +++ b/cvs/lib/inference/sglang/sglang_common.py @@ -0,0 +1,310 @@ +'''Shared helpers for SGLang single-node and disaggregated inference libs.''' + +from __future__ import annotations + +import json +import re +import shlex +from typing import Any, Mapping, Callable + +from cvs.lib import globals + +log = globals.log + +DEFAULT_GPU_MEM_THRESHOLD_MB = 5000 +AMD_SMI_METRIC_CMD = "sudo amd-smi metric --json" + +_SERVER_READY_RE = re.compile( + r"server is fired up and ready to roll", + re.I, +) + + +def textwrap_for_yml(msg_string: str) -> str: + return '\n'.join([m.lstrip() for m in msg_string.split('\n')]) + + +def as_node_list(value) -> list: + """Normalize cluster JSON node field to a list of host strings.""" + if isinstance(value, str): + return [value] + return list(value) + + +def resolve_server_node_list(inf_dict: Mapping[str, Any]) -> list[str]: + """Hosts for unified multi-node SGLang (not PD disagg). + + Resolution order: + 1. ``server_node_list`` when set. + 2. Union of ``prefill_node_list`` and ``decode_node_list`` (stable order). + """ + explicit = inf_dict.get('server_node_list') + if explicit: + return as_node_list(explicit) + seen: list[str] = [] + for key in ('prefill_node_list', 'decode_node_list'): + for host in as_node_list(inf_dict.get(key) or []): + if host not in seen: + seen.append(host) + if not seen: + raise ValueError( + 'sglang_distributed requires server_node_list or at least one of ' + 'prefill_node_list / decode_node_list in the inference config' + ) + return seen + + +def resolve_distributed_client_host( + inf_dict: Mapping[str, Any], + *, + rank0_node: str, + benchmark_serv_node: str, +) -> str: + """HTTP target for bench/smoke/lm-eval when the unified server spans multiple nodes.""" + explicit = inf_dict.get('client_host') + if explicit: + return str(explicit) + if benchmark_serv_node == rank0_node: + return '127.0.0.1' + return rank0_node + + +def resolve_client_host(inf_dict: Mapping[str, Any], *, unified_server: bool = False) -> str: + """HTTP target for smoke/bench/lm-eval clients running inside a container.""" + explicit = inf_dict.get('client_host') + if explicit: + return str(explicit) + if unified_server: + return '127.0.0.1' + proxy = as_node_list(inf_dict['proxy_router_node'])[0] + bench = as_node_list(inf_dict['benchmark_serv_node'])[0] + if proxy == bench: + return '127.0.0.1' + return proxy + + +def _normalize_key_value_list(raw: Any, field_name: str) -> list[str]: + """Normalize ``add_export_env`` entries to ``KEY=VALUE`` strings.""" + if raw is None: + return [] + if isinstance(raw, dict): + return [f'{k}={v}' for k, v in raw.items()] + if isinstance(raw, list): + out: list[str] = [] + for item in raw: + line = str(item).strip() + if not line: + continue + if line.startswith('export '): + line = line[7:].strip() + out.append(line) + return out + raise ValueError(f'{field_name} must be a list or dict, got {type(raw).__name__}') + + +def _normalize_cli_flags(raw: Any) -> list[str]: + """Normalize ``add_flags`` entries to extra ``launch_server`` CLI tokens.""" + if raw is None: + return [] + if isinstance(raw, str): + line = raw.strip() + return [line] if line else [] + if isinstance(raw, list): + return [str(item).strip() for item in raw if str(item).strip()] + raise ValueError(f'add_flags must be a list or str, got {type(raw).__name__}') + + +def add_export_env_block(bp_dict: Mapping[str, Any], indent: str = ' ') -> str: + """Shell ``export`` lines from ``bp_dict['add_export_env']``.""" + env = _normalize_key_value_list(bp_dict.get('add_export_env'), 'add_export_env') + return '\n'.join(f'{indent}export {entry}' for entry in env) + + +def add_cli_flags_block(bp_dict: Mapping[str, Any], indent: str = ' ') -> str: + """Extra ``launch_server`` CLI flag lines from ``bp_dict['add_flags']``.""" + flags = _normalize_cli_flags(bp_dict.get('add_flags')) + if not flags: + return '' + return '\n'.join(f'{indent}{flag} \\' for flag in flags) + + +def first_float(pattern: str, text: str): + m = re.search(pattern, text, re.I) + return m.group(1) if m else None + + +def _is_sglang_latency_metric(metric_name: str) -> bool: + name = metric_name.lower() + return 'ms' in name or 'latency' in name + + +def _is_sglang_higher_is_better_metric(metric_name: str) -> bool: + if _is_sglang_latency_metric(metric_name): + return False + name = metric_name.lower() + return any( + token in name + for token in ( + 'throughput', + 'goodput', + 'mfu', + 'request_throughput', + ) + ) + + +def normalize_sglang_threshold_spec(metric_name: str, spec: Any) -> dict[str, Any]: + """Map threshold JSON specs (or legacy flat floats) to evaluate_all kinds.""" + if isinstance(spec, dict) and spec.get('kind'): + return spec + value = float(spec['value'] if isinstance(spec, dict) and 'value' in spec else spec) + if _is_sglang_latency_metric(metric_name): + return {'kind': 'max_ms', 'value': value} + if _is_sglang_higher_is_better_metric(metric_name): + kind = 'min_tok_s' if 'throughput' in metric_name.lower() else 'min' + return {'kind': kind, 'value': value} + return {'kind': 'min', 'value': value} + + +def coerce_sglang_actual(value: Any) -> float | None: + if value is None: + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +def build_log_dir_cleanup_cmd(log_dir: str, user: str) -> str: + """Shell command: rm -rf, recreate, chown (host namespace, not in-container).""" + if not log_dir or not str(log_dir).strip(): + raise ValueError("log_dir must be a non-empty path") + log_dir = str(log_dir).strip() + quser = shlex.quote(str(user)) + qdir = shlex.quote(log_dir) + return f"sudo rm -rf {qdir} && sudo mkdir -p {qdir} && sudo chown -R {quser}:{quser} {qdir}" + + +def cleanup_sglang_log_dir( + orch: Any, + log_dir: str, + *, + all_nodes: bool | None = None, + timeout: int = 60, +) -> None: + """Reset log root on cluster hosts via baremetal SSH (``orch.head`` / ``orch.all``).""" + if all_nodes is None: + all_nodes = len(orch.hosts) > 1 + cmd = build_log_dir_cleanup_cmd(log_dir, orch.user) + if all_nodes: + orch.all.exec(cmd, timeout=timeout) + else: + orch.head.exec(cmd, timeout=timeout) + + +LM_EVAL_SPECS = { + 'lm_eval_hellaswag': { + 'display': 'HellaSwag', + 'default_metric': 'acc_norm', + 'default_metric_key': 'acc_norm,none', + 'default_num_concurrent': '1', + }, + 'lm_eval_gsm8k': { + 'display': 'GSM8K', + 'default_metric': 'exact_match', + 'default_metric_key': 'exact_match,flexible-extract', + 'default_num_concurrent': '4', + }, +} + + +def _parse_amd_smi_gpu_entries(payload: str | None) -> list[dict]: + """Unwrap amd-smi --json (list or {"gpu_data": [...]}) -> GPU entry list.""" + try: + entries = json.loads((payload or "").strip()) + except (json.JSONDecodeError, AttributeError, TypeError): + return [] + if isinstance(entries, dict): + entries = entries.get("gpu_data", []) + return entries if isinstance(entries, list) else [] + + +def count_occupied_gpus_on_node( + payload: str | None, + *, + mem_threshold_mb: int = DEFAULT_GPU_MEM_THRESHOLD_MB, +) -> int: + count = 0 + for g in _parse_amd_smi_gpu_entries(payload): + used_mb = g.get("mem_usage", {}).get("used_vram", {}).get("value", 0) + if used_mb > mem_threshold_mb: + count += 1 + return count + + +def count_occupied_gpus_per_node( + out_dict: Mapping[str, str | None], + *, + mem_threshold_mb: int = DEFAULT_GPU_MEM_THRESHOLD_MB, +) -> dict[str, int]: + per_node: dict[str, int] = {} + for node, payload in out_dict.items(): + if payload is None: + log.warning("No amd-smi output on node %s", node) + per_node[node] = 0 + continue + try: + per_node[node] = count_occupied_gpus_on_node(payload, mem_threshold_mb=mem_threshold_mb) + except (TypeError, ValueError, AttributeError): + log.warning("Failed to parse amd-smi JSON on node %s", node) + per_node[node] = 0 + return per_node + + +def collect_sglang_gpu_topology( + host_exec: Callable[..., dict[str, str | None]], + groups: Mapping[str, list[str]], + *, + mem_threshold_mb: int = DEFAULT_GPU_MEM_THRESHOLD_MB, + amd_smi_cmd: str = AMD_SMI_METRIC_CMD, + timeout: int | None = None, +) -> dict[str, Any]: + """ + groups: e.g. {"server": [...]} or {"prefill": [...], "decode": [...]} + host_exec: suite _host_exec(cmd, hosts=..., timeout=...) + """ + group_stats: dict[str, dict[str, Any]] = {} + total = 0 + + for name, hosts in groups.items(): + if not hosts: + group_stats[name] = {"per_node": {}, "total": 0} + continue + per_node = count_occupied_gpus_per_node( + host_exec(amd_smi_cmd, hosts=hosts, timeout=timeout), + mem_threshold_mb=mem_threshold_mb, + ) + group_total = sum(per_node.values()) + group_stats[name] = {"per_node": per_node, "total": group_total} + total += group_total + + return {"groups": group_stats, "total_occupied_gpus": total} + + +def format_sglang_gpu_topology_lines( + *, + configured_tp: int, + configured_pp: int, + groups: Mapping[str, dict[str, Any]], + configured_nnodes: int | None = None, +) -> list[str]: + lines = ["", f"Configured TP: {configured_tp}", f"Configured PP: {configured_pp}"] + if configured_nnodes is not None: + lines.append(f"Configured nnodes: {configured_nnodes}") + for label, stats in groups.items(): + lines.extend(["", f"{label.title()}:"]) + for node, count in stats["per_node"].items(): + lines.append(f" {node}: {count} occupied GPUs") + lines.append(f" Total: {stats['total']} occupied GPUs") + lines.extend(["", "Total hardware GPUs consumed:", f" {sum(s['total'] for s in groups.values())}"]) + return lines diff --git a/cvs/lib/inference/sglang/sglang_config_loader.py b/cvs/lib/inference/sglang/sglang_config_loader.py new file mode 100644 index 000000000..5e267172b --- /dev/null +++ b/cvs/lib/inference/sglang/sglang_config_loader.py @@ -0,0 +1,453 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +SGLang single-node config loader for ContainerOrchestrator suites. + +Supports two on-disk layouts: + +1. **Legacy** (existing ``mi30x_sglang_*.json``): + top-level ``config`` + ``benchmark_params`` + per-variant ``threshold_file``. + +2. **Unified** (vLLM-style, optional future configs): + ``schema_version: 1``, ``framework: "sglang_single"``, ``paths`` / ``container`` / + ``model`` / ``threshold_json``. + +``load_variant()`` is the single entry point for ``sglang_single`` conftest and +produces both: +- typed fields for ``OrchestratorFactory`` (``container``, ``paths``, ``model``) +- legacy dicts (``inference``, ``benchmark_params``) for ``SglangSingle`` +''' + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from typing import Any, Dict, Mapping + +from pydantic import Field, model_validator +from typing_extensions import Literal + +from cvs.lib import globals +from cvs.lib.utils.config_loader import ( + BaseVariantConfig, + _Forbid, + substitute_config, +) +from cvs.lib.utils_lib import resolve_test_config_placeholders + +log = globals.log + +_LEGACY_FRAMEWORK = "sglang_single" +_UNIFIED_FRAMEWORK = "sglang_single" + +_PERF_CELL_RE = re.compile(r"^ISL=(?P\d+),OSL=(?P\d+),TP=(?P\d+),PP=(?P\d+),CONC=(?P\d+)$") + + +# ---------- threshold / variant helpers (moved out of conftest) ---------- + + +def resolve_benchmark_variant_key(root: Mapping[str, Any], config_path: str) -> str: + """Pick which ``benchmark_params`` entry to run.""" + env_key = (os.environ.get("SGLANG_BENCHMARK_KEY") or "").strip() + bp = root.get("benchmark_params") or {} + if not isinstance(bp, dict) or not bp: + raise ValueError(f"benchmark_params missing or empty in {config_path!r}") + + if env_key: + if env_key not in bp: + raise ValueError( + f"SGLANG_BENCHMARK_KEY={env_key!r} not found in benchmark_params ({config_path}); valid: {sorted(bp)!r}" + ) + log.info("Using benchmark variant from env SGLANG_BENCHMARK_KEY=%r", env_key) + return env_key + + explicit = root.get("active_benchmark") + if explicit is not None: + if explicit not in bp: + raise ValueError( + f"active_benchmark={explicit!r} not found in benchmark_params ({config_path}); valid: {sorted(bp)!r}" + ) + log.info("Using benchmark variant from active_benchmark=%r", explicit) + return str(explicit) + + if len(bp) == 1: + only = next(iter(bp)) + log.info("Single benchmark_params entry; using %r", only) + return str(only) + + raise ValueError( + f"Multiple benchmark_params keys in {config_path!r}: {sorted(bp)!r}. " + 'Set top-level "active_benchmark" to one of them, or export SGLANG_BENCHMARK_KEY.' + ) + + +def flat_expected_from_specs(specs: Mapping[str, Any]) -> dict[str, float]: + out: dict[str, float] = {} + for metric, spec in specs.items(): + if isinstance(spec, dict) and "value" in spec: + out[metric] = float(spec["value"]) + else: + out[metric] = float(spec) + return out + + +def perf_cell_key(bp_dict: Mapping[str, Any]) -> str: + bench = (bp_dict.get("inference_tests") or {}).get("bench_serv_random") or {} + return ( + f"ISL={bench.get('input_length', '-')}," + f"OSL={bench.get('output_length', '-')}," + f"TP={bp_dict.get('tensor_parallelism', '8')}," + f"PP={bp_dict.get('pipeline_parallelism', '1')}," + f"CONC={bp_dict.get('max_concurrency', '-')}" + ) + + +def bench_cell_key(bench_name: str) -> str: + return f"BENCH={bench_name}" + + +def perf_cells_from_thresholds(thresholds: Mapping[str, Any]) -> list[dict[str, Any]]: + cells = [] + for cell_key, specs in thresholds.items(): + if str(cell_key).startswith("_") or str(cell_key).startswith("BENCH="): + continue + m = _PERF_CELL_RE.match(str(cell_key)) + if not m: + continue + cells.append( + { + "cell_key": cell_key, + "isl": m.group("isl"), + "osl": m.group("osl"), + "tp": m.group("tp"), + "conc": m.group("conc"), + "specs": specs, + } + ) + cells.sort(key=lambda c: (int(c["isl"]), int(c["osl"]), int(c["conc"]))) + return cells + + +def _resolve_threshold_path(threshold_path: str, *, config_path: Path) -> Path: + path = Path(threshold_path) + if path.is_absolute(): + return path + # Legacy configs often use repo-relative paths like cvs/input/... + cwd_candidate = (Path.cwd() / path).resolve() + if cwd_candidate.is_file(): + return cwd_candidate + return (config_path.parent / path).resolve() + + +def _load_thresholds_file(path: Path) -> dict[str, Any]: + with open(path, encoding="utf-8") as fp: + raw = json.load(fp) + if not isinstance(raw, dict): + raise ValueError(f"threshold file must be a JSON object: {path}") + return {k: v for k, v in raw.items() if not str(k).startswith("_")} + + +def _threshold_file_path(bp_dict: Mapping[str, Any]) -> str | None: + path = bp_dict.get("threshold_file") + return str(path).strip() if path else None + + +def _inject_thresholds_into_bp_dict(bp_dict: dict[str, Any], thresholds: Mapping[str, Any]) -> None: + inference_tests = bp_dict.setdefault("inference_tests", {}) + + perf_key = perf_cell_key(bp_dict) + perf_specs = thresholds.get(perf_key) + if perf_specs: + bench = inference_tests.setdefault("bench_serv_random", {}) + expected = bench.setdefault("expected_results", {}) + expected["auto"] = flat_expected_from_specs(perf_specs) + log.info("Loaded performance thresholds from cell %r", perf_key) + else: + log.warning("No performance thresholds for cell %r in threshold file", perf_key) + + for bench_name in ("lm_eval_hellaswag", "lm_eval_gsm8k"): + cell = bench_cell_key(bench_name) + acc_specs = thresholds.get(cell) + if not acc_specs: + continue + bench = inference_tests.setdefault(bench_name, {}) + expected = bench.setdefault("expected_results", {}) + task_key = bench_name.removeprefix("lm_eval_") + expected[task_key] = flat_expected_from_specs(acc_specs) + log.info("Loaded accuracy thresholds from cell %r", cell) + + +def load_perf_cells_for_collection(config_file: str) -> list[dict[str, Any]]: + """Collection-time loader (no fixtures yet).""" + variant = load_variant(config_file, cluster_dict={}) + cells = perf_cells_from_thresholds(variant.thresholds) + if not cells: + raise ValueError(f"No ISL=... performance cells in thresholds for {config_file!r}") + return cells + + +# ---------- legacy → ContainerOrchestrator bridge ---------- + + +def _volume_dict_to_mounts(volume_dict: Mapping[str, Any]) -> list[str]: + mounts: list[str] = [] + for host, container in volume_dict.items(): + mounts.append(f"{host}:{container}") + return mounts + + +def _infer_models_dir(inference: Mapping[str, Any]) -> str: + volume_dict = (inference.get("container_config") or {}).get("volume_dict") or {} + for host, container in volume_dict.items(): + host_s, container_s = str(host), str(container) + if "models" in host_s.lower() or "models" in container_s.lower(): + return host_s + # Fallback: sibling of log_dir + log_dir = str(inference.get("log_dir") or "").rstrip("/") + if log_dir: + return str(Path(log_dir).parent / "models") + raise ValueError( + "cannot infer models_dir from legacy config; add a models volume mount or migrate to unified paths.models_dir" + ) + + +def _infer_shared_fs(inference: Mapping[str, Any]) -> str: + log_dir = str(inference.get("log_dir") or "").rstrip("/") + if log_dir: + return str(Path(log_dir).parent) + token = str(inference.get("hf_token_file") or "") + if token: + return str(Path(token).parent.parent) + raise ValueError("cannot infer shared_fs from legacy config") + + +def _legacy_server_env(inference: Mapping[str, Any], bp: Mapping[str, Any]) -> dict[str, str]: + """NCCL / runtime env merged into container env for ContainerOrchestrator.""" + env: dict[str, str] = {} + + def _put(key: str, src_key: str) -> None: + val = inference.get(src_key) + if val is not None and str(val).strip(): + env[key] = str(val) + + _put("NCCL_DEBUG", "nccl_debug") + _put("NCCL_IB_HCA", "nccl_ib_hca") + _put("NCCL_IB_GID_INDEX", "nccl_ib_gid_index") + _put("NCCL_SOCKET_IFNAME", "nccl_socket_ifname") + _put("GLOO_SOCKET_IFNAME", "gloo_socket_ifname") + _put("GLOO_TCP_IFNAME", "gloo_tcp_ifname") + + cc_env = (inference.get("container_config") or {}).get("env_dict") or {} + for k, v in cc_env.items(): + if v is not None: + env[str(k)] = str(v) + + for entry in bp.get("add_export_env") or []: + line = str(entry).strip() + if not line: + continue + if line.startswith("export "): + line = line[7:].strip() + if "=" in line: + k, v = line.split("=", 1) + env[k.strip()] = v.strip() + + return env + + +def legacy_container_block_from_inference(inference: Mapping[str, Any]) -> dict[str, Any]: + """Build a ``ContainerSpec``-compatible dict from legacy ``config``.""" + cc = inference.get("container_config") or {} + runtime_args: dict[str, Any] = { + "network": "host", + "ipc": "host", + "privileged": True, + "volumes": _volume_dict_to_mounts(cc.get("volume_dict") or {}), + "devices": list(cc.get("device_list") or []), + } + shm = inference.get("shm_size") + if shm: + runtime_args["shm_size"] = str(shm) + + return { + "lifetime": inference.get("container_lifetime", "per_run"), + "name": inference["container_name"], + "image": inference["container_image"], + "runtime": { + "name": "docker", + "args": runtime_args, + }, + } + + +def legacy_paths_from_inference(inference: Mapping[str, Any]) -> dict[str, str]: + shared_fs = _infer_shared_fs(inference) + return { + "shared_fs": shared_fs, + "models_dir": _infer_models_dir(inference), + "log_dir": str(inference["log_dir"]), + "hf_token_file": str(inference["hf_token_file"]), + } + + +def _is_legacy_root(raw: Mapping[str, Any]) -> bool: + return "benchmark_params" in raw and ("config" in raw or "container_image" in raw) + + +# ---------- typed config ---------- + + +class SglangRoleServer(_Forbid): + env: Dict[str, str] = Field(default_factory=dict) + serve_port: str = "8000" + + +class SglangRoles(_Forbid): + server: SglangRoleServer = Field(default_factory=SglangRoleServer) + + +class SglangSingleVariantConfig(BaseVariantConfig): + """Typed config for ``sglang_single`` + ContainerOrchestrator.""" + + framework: Literal["sglang_single"] + gpu_arch: str + variant_key: str = "" + config_path: str = "" + + # Legacy blocks kept for ``SglangSingle`` until that lib is refactored. + inference: Dict[str, Any] = Field(default_factory=dict) + benchmark_params: Dict[str, Any] = Field(default_factory=dict) + + roles: SglangRoles = Field(default_factory=SglangRoles) + + def cell_key(self, isl, osl, concurrency) -> str: + tp = self.benchmark_params.get("tensor_parallelism", "-") + pp = self.benchmark_params.get("pipeline_parallelism", "-") + return f"ISL={isl},OSL={osl},TP={tp},PP={pp},CONC={concurrency}" + + def perf_cell_key(self) -> str: + return perf_cell_key(self.benchmark_params) + + @property + def hf_token_file(self) -> str: + return self.paths.hf_token_file + + @model_validator(mode="after") + def _sync_legacy_inference_container_name(self): + """Keep legacy inference dict aligned with orchestrator container name.""" + if self.inference and self.container.name: + self.inference["container_name"] = self.container.name + self.inference["container_image"] = self.container.image + return self + + +# ---------- public API ---------- + + +def orchestrator_container_from_variant(variant: SglangSingleVariantConfig) -> Dict[str, Any]: + """``container`` block for ``OrchestratorConfig`` (includes server env).""" + block = variant.container.model_dump() + server_env = variant.roles.server.env + if server_env: + block = {**block, "env": dict(server_env)} + return block + + +def _load_legacy_variant(config_path: str, cluster_dict: Mapping[str, Any]) -> SglangSingleVariantConfig: + path = Path(config_path) + with open(path, encoding="utf-8") as fp: + root = json.load(fp) + + variant_key = resolve_benchmark_variant_key(root, config_path) + cfg = root["config"] if isinstance(root.get("config"), dict) else root + + inference = resolve_test_config_placeholders(cfg, cluster_dict) + bp_all = resolve_test_config_placeholders(root["benchmark_params"], cluster_dict) + bp = dict(bp_all[variant_key]) + + threshold_path_str = _threshold_file_path(bp) + if not threshold_path_str: + raise ValueError(f"benchmark_params[{variant_key!r}] missing 'threshold_file' in {config_path!r}") + + threshold_path = _resolve_threshold_path(threshold_path_str, config_path=path) + thresholds = _load_thresholds_file(threshold_path) + log.info("Loaded thresholds from %s (%d cells)", threshold_path, len(thresholds)) + _inject_thresholds_into_bp_dict(bp, thresholds) + + container_raw = legacy_container_block_from_inference(inference) + paths_raw = legacy_paths_from_inference(inference) + server_env = _legacy_server_env(inference, bp) + + raw: dict[str, Any] = { + "schema_version": 1, + "framework": _LEGACY_FRAMEWORK, + "gpu_arch": str(root.get("gpu_arch") or "mi30x"), + "enforce_thresholds": bool(root.get("enforce_thresholds", True)), + "threshold_json": str(threshold_path), + "paths": paths_raw, + "model": { + "id": str(bp["model"]), + "remote": int(root.get("model_remote", bp.get("model_remote", 0))), + }, + "container": container_raw, + "thresholds": thresholds, + "variant_key": variant_key, + "config_path": str(path.resolve()), + "inference": dict(inference), + "benchmark_params": bp, + "roles": { + "server": { + "env": server_env, + "serve_port": str( + inference.get("proxy_router_serv_port") or inference.get("proxy_router_port") or "8000" + ), + } + }, + } + return SglangSingleVariantConfig(**raw) + + +def _load_unified_variant(config_path: str, cluster_dict: Mapping[str, Any]) -> SglangSingleVariantConfig: + raw, thresholds = substitute_config(config_path, cluster_dict) + raw["thresholds"] = thresholds + raw["config_path"] = str(Path(config_path).resolve()) + + if not raw.get("variant_key"): + if "benchmark_params" in raw: + raw["variant_key"] = resolve_benchmark_variant_key(raw, config_path) + else: + raw["variant_key"] = raw.get("active_benchmark") or "default" + + # Optional embedded legacy blocks in unified configs. + if "config" in raw and not raw.get("inference"): + inference = resolve_test_config_placeholders(raw["config"], cluster_dict) + raw["inference"] = dict(inference) + if "benchmark_params" in raw and not raw.get("benchmark_params"): + bp_all = resolve_test_config_placeholders(raw["benchmark_params"], cluster_dict) + raw["benchmark_params"] = dict(bp_all[raw["variant_key"]]) + + known = {k: v for k, v in raw.items() if k in SglangSingleVariantConfig.model_fields} + return SglangSingleVariantConfig(**known) + + +def load_variant(config_path: str, cluster_dict: Mapping[str, Any]) -> SglangSingleVariantConfig: + """Load and validate an ``sglang_single`` variant config + thresholds.""" + path = Path(config_path) + if not path.is_file(): + raise FileNotFoundError(f"variant config not found: {path}") + + with open(path, encoding="utf-8") as fp: + peek = json.load(fp) + + if _is_legacy_root(peek): + return _load_legacy_variant(config_path, cluster_dict) + + if peek.get("framework") not in (None, _UNIFIED_FRAMEWORK): + raise ValueError( + f"unsupported framework {peek.get('framework')!r} in {config_path!r}; expected {_UNIFIED_FRAMEWORK!r}" + ) + + return _load_unified_variant(config_path, cluster_dict) diff --git a/cvs/lib/inference/sglang/sglang_disagg_lib.py b/cvs/lib/inference/sglang/sglang_disagg_lib.py new file mode 100644 index 000000000..1f930c1c1 --- /dev/null +++ b/cvs/lib/inference/sglang/sglang_disagg_lib.py @@ -0,0 +1,1341 @@ +''' +Copyright 2026 Advanced Micro Devices, Inc. +All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. +The year included in the foregoing notice is the year of creation of the work. +All code contained here is Property of Advanced Micro Devices, Inc. + +Disaggregated Prefill/Decode (PD) SGLang inference controller. + +Prefill, decode, proxy-router, and benchmark workloads run inside containers +on their respective cluster nodes via ``ContainerOrchestrator`` (``orch=``). +Bare-metal SSH (``orch.head`` / ``orch.all``) is used for ``amd-smi`` and +``dmesg`` verification. +''' + +from __future__ import annotations + +import base64 +import json +import os +import re +import shlex +import time +from typing import Any, Optional + +from cvs.lib import globals +from cvs.core.orchestrators.baremetal import BaremetalOrchestrator +from cvs.lib.inference.sglang.sglang_common import ( + LM_EVAL_SPECS, + add_cli_flags_block, + add_export_env_block, + as_node_list, + coerce_sglang_actual, + first_float, + normalize_sglang_threshold_spec, + resolve_client_host, + collect_sglang_gpu_topology, + format_sglang_gpu_topology_lines, + _SERVER_READY_RE, +) +from cvs.lib.utils.model_query_lib import LmEvalBenchmark, LongContextNiahBenchmark, OpenAIProbe +from cvs.lib.utils_lib import fail_test +from cvs.lib.utils.verdict import ThresholdViolation, evaluate_all +from cvs.lib.verify_lib import verify_dmesg_for_errors + +log = globals.log + +inference_err_dict = { + 'NCCL ERROR': 'NCCL ERROR|NCCL timeout|local work queue catastrophic error', + 'GPU HW ERROR': 'HW Exception by GPU|GPU Hang|Uncorrectable error|GPU Reset', + 'AssertionError': 'AssertionError|ValueError:|During handling of the above exception|triggered the following exception|RuntimeError|Python error: Aborted', + 'rocm Err': 'FAILED_PRECONDITION: No visible GPU devices|failed call to hipInit: HIP_ERROR_NoDevice|librocm reported version is: NOT_FOUND', + 'python err': 'ModuleNotFoundError: No module named|Fatal Python error:', + 'resource': 'RESOURCE_EXHAUSTED: Out of memory|failed: RESOURCE_EXHAUSTED|urllib.error.URLError|ConnectionRefusedError,HSA_STATUS_ERROR_OUT_OF_RESOURCES', + 'app_err': 'Service Unavailable|No decode workers available|No prefill workers available|Please check if decode servers are configured and healthy|Please check if prefill servers are configured and healthy|Cannot access gated repo|You must have access to it and be authenticated', +} + +err_counters_pattern = 'err|retransmit|drop|discard|naks|invalid|oflow|out_of_buffer|reset|fail' + + +class SglangDisaggPD: + """Disaggregated Prefill/Decode SGLang controller via ``ContainerOrchestrator``.""" + + def __init__( + self, + model_name, + inference_config_dict, + benchmark_params_dict, + hf_token, + orch=None, + gpu_type='mi300', + user_name=None, + priv_key_file=None, + ): + """ + Initialize a Disaggregated Prefill/Decode (PD) inference controller + for SGLang. + + This class encapsulates: + - Cluster topology (prefill, decode, proxy, benchmark nodes) + - Container execution via ``ContainerOrchestrator`` (``orch=``) + - Inference configuration (networking, containers, env vars) + - Benchmark configuration (load, concurrency, prompt sizes) + + Args: + model_name (str): HuggingFace or local model identifier + inference_config_dict (dict): Cluster and runtime configuration + benchmark_params_dict (dict): Benchmark workload parameters + hf_token (str): HuggingFace access token + orch: Required ``ContainerOrchestrator`` instance + gpu_type (str): GPU type (e.g., mi300, mi325) + user_name (str): SSH username for remote nodes (optional override) + priv_key_file (str): SSH private key file (optional override) + """ + if orch is None: + raise ValueError("SglangDisaggPD requires orch= (ContainerOrchestrator)") + + self.orch = orch + self.user_name = user_name + self.priv_key_file = priv_key_file + self.model_name = model_name + self.hf_token = hf_token + self.gpu_type = gpu_type + + self.inf_dict = inference_config_dict + self.bp_dict = benchmark_params_dict + + self.mount_vol = self.inf_dict.get( + 'mount_vol', + '/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so', + ) + + self.prefill_node_list = self._normalize_hosts(self.inf_dict['prefill_node_list']) + self.decode_node_list = self._normalize_hosts(self.inf_dict['decode_node_list']) + self.prefill_nnodes = len(self.prefill_node_list) + self.decode_nnodes = len(self.decode_node_list) + + self.proxy_node = self._normalize_hosts(self.inf_dict['proxy_router_node']) + self.benchmark_serv_node = self._normalize_hosts(self.inf_dict['benchmark_serv_node']) + + self.job_cmd = '' + self.job_cmd_list = [] + self.inference_results_dict = {} + log.info("%s", self.gpu_type) + + self.rdma_stats_dict_before = {} + self.ethtool_stats_dict_before = {} + self.rdma_stats_dict_after = {} + self.home_dir = os.path.expanduser("~") + self._apply_inf_defaults() + self._apply_bp_defaults() + + self.container_name = self.inf_dict['container_name'] + self.nic_type = self.inf_dict['nic_type'] + self.nccl_ib_hca_list = self.inf_dict['nccl_ib_hca_list'] + self.nccl_ib_hca = self.inf_dict['nccl_ib_hca'] + self.nccl_socket_ifname = self.inf_dict['nccl_socket_ifname'] + self.gloo_socket_ifname = self.inf_dict['gloo_socket_ifname'] + self.nccl_ib_gid_index = self.inf_dict['nccl_ib_gid_index'] + self.nccl_debug = self.inf_dict['nccl_debug'] + self.data_cache_dir = self.inf_dict['data_cache_dir'] + self.log_dir = self.inf_dict['log_dir'] + self.hca_id_prefix = str(self.inf_dict['hca_id_prefix']).strip() + self.inference_poll_iterations = self.bp_dict['inference_poll_iterations'] + + self.inference_start_time = self._host_exec('date +"%a %b %e %H:%M"') + self.inference_end_time = None + + log.info('disagg inference_dict = %s', self.inf_dict) + log.info('disagg benchmark_params_dict = %s', self.bp_dict) + log.info( + 'disagg client_host=%s router_serv_port=%s head=%s', + self.client_host, + self.router_serv_port, + self._head_host, + ) + + @property + def _head_host(self) -> str: + return self.orch.head_node + + @property + def router_serv_port(self) -> str: + """Client-facing proxy router port (bench/smoke/lm-eval).""" + return str(self.inf_dict['proxy_router_serv_port']) + + @property + def client_host(self) -> str: + return resolve_client_host(self.inf_dict, unified_server=False) + + @staticmethod + def _first_output(out_dict: dict) -> str: + if not out_dict: + return "" + return next(iter(out_dict.values())) or "" + + @staticmethod + def _normalize_hosts(hosts) -> list[str]: + """Normalize cluster JSON node field to a list of host strings.""" + if hosts is None: + return [] + return as_node_list(hosts) + + def _container_exec( + self, + cmd: str, + *, + hosts=None, + timeout: int | None = None, + ) -> dict: + """Run ``cmd`` inside the container on ``hosts`` (default: all orch hosts).""" + normalized = self._normalize_hosts(hosts) if hosts is not None else None + return self.orch.exec(cmd, hosts=normalized, timeout=timeout) + + def _container_exec_text( + self, + cmd: str, + *, + hosts=None, + timeout: int | None = None, + ) -> str: + return self._first_output(self._container_exec(cmd, hosts=hosts, timeout=timeout)) + + def _host_exec( + self, + cmd: str, + *, + hosts=None, + timeout: int | None = None, + ) -> dict: + """Run ``cmd`` on baremetal (``orch.head`` / ``orch.all``), e.g. amd-smi / dmesg.""" + if hosts is None: + return self.orch.head.exec(cmd, timeout=timeout) + normalized = self._normalize_hosts(hosts) + if not normalized: + return {} + if len(normalized) == 1 and normalized[0] == self._head_host: + return self.orch.head.exec(cmd, timeout=timeout) + if set(normalized) == set(self.orch.hosts): + return self.orch.all.exec(cmd, timeout=timeout) + return BaremetalOrchestrator.exec(self.orch, cmd, hosts=normalized, timeout=timeout) + + def _host_exec_text( + self, + cmd: str, + *, + hosts=None, + timeout: int | None = None, + ) -> str: + return self._first_output(self._host_exec(cmd, hosts=hosts, timeout=timeout)) + + def _apply_inf_defaults(self) -> None: + self.inf_dict.setdefault('container_image', 'lmsysorg/sglang:dev') + self.inf_dict.setdefault('container_name', 'sglang_container') + self.inf_dict.setdefault('nic_type', 'ainic') + self.inf_dict.setdefault('nccl_ib_hca_list', 'rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7') + self.inf_dict.setdefault('nccl_ib_hca', 'rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7') + self.inf_dict.setdefault('hca_id_prefix', 'bnxt_') + self.inf_dict.setdefault('nccl_socket_ifname', 'eno0') + self.inf_dict.setdefault('gloo_socket_ifname', 'eno0') + self.inf_dict.setdefault('nccl_ib_gid_index', '1') + self.inf_dict.setdefault('nccl_debug', 'ERROR') + self.inf_dict.setdefault('data_cache_dir', f'{self.home_dir}/cache') + self.inf_dict.setdefault('log_dir', f'{self.home_dir}/LOG_DIR') + self.inf_dict.setdefault('log_level', 'info') + self.inf_dict.setdefault('prefill_serv_port', '30001') + self.inf_dict.setdefault('decode_serv_port', '30002') + self.inf_dict.setdefault('proxy_router_port', '8000') + self.inf_dict.setdefault('proxy_router_serv_port', '8000') + self.inf_dict.setdefault('max_concurrent_requests', '-1') + self.inf_dict.setdefault('queue_size', '100') + self.inf_dict.setdefault('queue_timeout_secs', '60') + self.inf_dict.setdefault('max_retries', '5') + + def _apply_bp_defaults(self) -> None: + self.bp_dict.setdefault('backend', 'sglang') + self.bp_dict.setdefault('dataset_name', 'sharegpt') + self.bp_dict.setdefault('max_concurrency', '64') + self.bp_dict.setdefault('model', 'openai/gpt-oss-120b') + self.bp_dict.setdefault('num_prompts', '1000') + self.bp_dict.setdefault('input_sequence_length', '8192') + self.bp_dict.setdefault('burstiness', '1.0') + self.bp_dict.setdefault('seed', '0') + self.bp_dict.setdefault('request_rate', 'inf') + self.bp_dict.setdefault('random_range_ration', '1.0') + self.bp_dict.setdefault('random_prefix_len', '0') + self.bp_dict.setdefault('tensor_parallelism', '8') + self.bp_dict.setdefault('pipeline_parallelism', '1') + self.bp_dict.setdefault('context_length', '131072') + self.bp_dict.setdefault('port_no', '8000') + self.bp_dict.setdefault('tokenizer_mode', 'auto') + self.bp_dict.setdefault('percentile_metrics', 'ttft,tpot,itl,e2el') + self.bp_dict.setdefault('metric_percentiles', '99') + self.bp_dict.setdefault('inference_poll_iterations', '16') + self.bp_dict.setdefault('memory_fraction', '0.85') + + def install_container_packages( + self, + ): + """ + Install required system networking utilities inside inference containers. + + Purpose: + -------- + This method prepares the container environment for distributed inference + by installing basic networking and diagnostic tools that are commonly + needed for: + - Connectivity validation between nodes + - Debugging network paths (ping, ip route, ifconfig) + - Verifying NIC and routing configuration + - Troubleshooting NCCL/Gloo/RDMA-related issues + + These tools are installed inside the running container on: + - Prefill nodes + - Decode nodes + - Proxy/router nodes + """ + log.info('Run pre inference tasks') + cmd = "bash -c " + shlex.quote("sudo apt -y update && sudo apt install -y iputils-ping iproute2 net-tools") + for hosts in (self.prefill_node_list, self.decode_node_list, self.proxy_node): + self._container_exec(cmd, hosts=hosts) + + def exec_nic_setup_scripts( + self, + ): + """ + Execute NIC-related setup steps inside the inference container. + + Behavior: + - Only runs for distributed inference. + - If NIC type appears to be Broadcom/Thor, applies a temporary workaround: + * Copies the bnxt RDMA library from the host-named file to the container's expected path. + * Verifies that ibv_devinfo shows a bnxt_ HCA (to confirm RDMA is wired correctly). + - Forces NCCL GID index to 3 for Broadcom/Thor (common requirement). + + Assumptions: + - sudo is non-interactive within the container. + - The bnxt library file paths exist in the container base image. + """ + if re.search('broadcom|thor', self.nic_type, re.I): + self.nccl_ib_gid_index = 3 + cmd = "bash -c " + shlex.quote(f"cp {self.mount_vol}.host {self.mount_vol}; sleep 2; ibv_devinfo; sleep 2;") + hca_id_regex = rf'hca_id:\s+{re.escape(self.hca_id_prefix)}' + for hosts in (self.prefill_node_list, self.decode_node_list): + out_dict = self._container_exec(cmd, hosts=hosts) + for node, out in out_dict.items(): + if not re.search(hca_id_regex, out or '', re.I): + log.info("%s", out) + fail_test(f'Broadcom libbnxt rdma driver is not properly copied on node {node}') + + def check_ibv_devices( + self, + ): + """ + Verify that InfiniBand / RDMA devices are visible inside the container + on all relevant nodes. + + Purpose: + -------- + This method ensures that RDMA-capable devices (e.g., InfiniBand HCAs) + are correctly exposed inside the container environment. This is a + critical prerequisite for: + - NCCL / RCCL over RDMA + - High-performance distributed inference + - Low-latency, high-bandwidth GPU communication + + The check is performed on: + - Prefill nodes + - Decode nodes + + Proxy and benchmark nodes typically do not require RDMA access. + """ + for hosts in (self.prefill_node_list, self.decode_node_list): + out_dict = self._container_exec("ibv_devinfo", hosts=hosts) + for node, out in out_dict.items(): + if re.search('No IB devices found', out or '', re.I): + fail_test(f'IB devices not seen inside the container for node {node}') + + def setup_prefill_container_env( + self, + ): + """Write and source ``/tmp/prefill_env_script.sh`` on prefill nodes.""" + env_body = ( + "export LD_LIBRARY_PATH=/usr/local/lib:/sgl-workspace/Mooncake/build/mooncake-common/etcd:/opt/rocm/lib:$LD_LIBRARY_PATH\n" + f"export NCCL_DEBUG={self.inf_dict['nccl_debug']}\n" + f"export NCCL_IB_HCA={self.inf_dict['nccl_ib_hca']}\n" + f"export NCCL_IB_GID_INDEX={self.inf_dict['nccl_ib_gid_index']}\n" + f"export NCCL_SOCKET_IFNAME={self.inf_dict['nccl_socket_ifname']}\n" + f"export GLOO_SOCKET_IFNAME={self.inf_dict['gloo_socket_ifname']}\n" + f"export GLOO_TCP_IFNAME={self.inf_dict['gloo_socket_ifname']}\n" + f"export HSA_FORCE_FINE_GRAIN_PCIE=1\n" + f"export MASTER_PREFILL_ADDR={self.inf_dict['prefill_coordinator_addr']}\n" + f"export MASTER_PREFILL_PORT={self.inf_dict['prefill_coordinator_port']}\n" + f"export MODEL={self.bp_dict['model']}\n" + f"export TP={self.bp_dict['tensor_parallelism']}\n" + f"export PP={self.bp_dict['pipeline_parallelism']}\n" + f"export HF_TOKEN={self.hf_token}\n" + f"{add_export_env_block(self.bp_dict, indent='')}\n" + ) + write_cmd = "bash -c " + shlex.quote( + f"cat > /tmp/prefill_env_script.sh <<'EOF'\n{env_body}EOF\n" + "chmod 755 /tmp/prefill_env_script.sh && /tmp/prefill_env_script.sh" + ) + time.sleep(3) + self._container_exec(write_cmd, hosts=self.prefill_node_list) + + def setup_decode_container_env( + self, + ): + """Write and source ``/tmp/decode_env_script.sh`` on decode nodes.""" + env_body = ( + "export LD_LIBRARY_PATH=/usr/local/lib:/sgl-workspace/Mooncake/build/mooncake-common/etcd:/opt/rocm/lib:$LD_LIBRARY_PATH\n" + f"export NCCL_DEBUG={self.inf_dict['nccl_debug']}\n" + f"export NCCL_IB_HCA={self.inf_dict['nccl_ib_hca']}\n" + f"export NCCL_IB_GID_INDEX={self.inf_dict['nccl_ib_gid_index']}\n" + f"export NCCL_SOCKET_IFNAME={self.inf_dict['nccl_socket_ifname']}\n" + f"export GLOO_SOCKET_IFNAME={self.inf_dict['gloo_socket_ifname']}\n" + f"export GLOO_TCP_IFNAME={self.inf_dict['gloo_socket_ifname']}\n" + f"export HSA_FORCE_FINE_GRAIN_PCIE=1\n" + f"export MASTER_DECODE_ADDR={self.inf_dict['decode_coordinator_addr']}\n" + f"export MASTER_DECODE_PORT={self.inf_dict['decode_coordinator_port']}\n" + f"export MODEL={self.bp_dict['model']}\n" + f"export TP={self.bp_dict['tensor_parallelism']}\n" + f"export PP={self.bp_dict['pipeline_parallelism']}\n" + f"export HF_TOKEN={self.hf_token}\n" + f"{add_export_env_block(self.bp_dict, indent='')}\n" + ) + write_cmd = "bash -c " + shlex.quote( + f"cat > /tmp/decode_env_script.sh <<'EOF'\n{env_body}EOF\n" + "chmod 755 /tmp/decode_env_script.sh && /tmp/decode_env_script.sh" + ) + time.sleep(3) + self._container_exec(write_cmd, hosts=self.decode_node_list) + + def setup_proxy_router_container_env( + self, + ): + """Write and source ``/tmp/router_env_script.sh`` on proxy/router nodes.""" + env_body = ( + "export LD_LIBRARY_PATH=/usr/local/lib:/sgl-workspace/Mooncake/build/mooncake-common/etcd:/opt/rocm/lib:$LD_LIBRARY_PATH\n" + f"export NCCL_DEBUG={self.inf_dict['nccl_debug']}\n" + f"export NCCL_IB_HCA={self.inf_dict['nccl_ib_hca']}\n" + f"export NCCL_IB_GID_INDEX={self.inf_dict['nccl_ib_gid_index']}\n" + f"export NCCL_SOCKET_IFNAME={self.inf_dict['nccl_socket_ifname']}\n" + f"export GLOO_SOCKET_IFNAME={self.inf_dict['gloo_socket_ifname']}\n" + f"export GLOO_TCP_IFNAME={self.inf_dict['gloo_socket_ifname']}\n" + f"export HSA_FORCE_FINE_GRAIN_PCIE=1\n" + f"export HF_TOKEN={self.hf_token}\n" + ) + write_cmd = "bash -c " + shlex.quote( + f"cat > /tmp/router_env_script.sh <<'EOF'\n{env_body}EOF\n" + "chmod 755 /tmp/router_env_script.sh && /tmp/router_env_script.sh" + ) + time.sleep(3) + self._container_exec(write_cmd, hosts=self.proxy_node) + + def setup_benchmark_serv_container_env( + self, + ): + """Write and source ``/tmp/benchmark_env_script.sh`` on benchmark nodes.""" + env_body = ( + "export LD_LIBRARY_PATH=/usr/local/lib:/sgl-workspace/Mooncake/build/mooncake-common/etcd:/opt/rocm/lib:$LD_LIBRARY_PATH\n" + f"export NCCL_DEBUG={self.inf_dict['nccl_debug']}\n" + f"export NCCL_IB_HCA={self.inf_dict['nccl_ib_hca']}\n" + f"export NCCL_IB_GID_INDEX={self.inf_dict['nccl_ib_gid_index']}\n" + f"export NCCL_SOCKET_IFNAME={self.inf_dict['nccl_socket_ifname']}\n" + f"export GLOO_SOCKET_IFNAME={self.inf_dict['gloo_socket_ifname']}\n" + f"export GLOO_TCP_IFNAME={self.inf_dict['gloo_socket_ifname']}\n" + f"export HSA_FORCE_FINE_GRAIN_PCIE=1\n" + f"export HF_TOKEN={self.hf_token}\n" + ) + write_cmd = "bash -c " + shlex.quote( + f"cat > /tmp/benchmark_env_script.sh <<'EOF'\n{env_body}EOF\n" + "chmod 755 /tmp/benchmark_env_script.sh && /tmp/benchmark_env_script.sh" + ) + time.sleep(3) + self._container_exec(write_cmd, hosts=self.benchmark_serv_node) + time.sleep(5) + + def run_test_rmsnorm(self, max_jobs=192): + """ + Run RMSNorm 2D operator tests inside the SGLang container across + relevant nodes and validate correctness. + + Purpose: + -------- + This method executes the AITER RMSNorm 2D operator test, which validates: + - Correctness of RMSNorm kernel implementation + - Stability under high parallel job execution + - GPU kernel behavior under concurrent workloads + + The test is executed on: + - Prefill nodes + - Decode nodes + - Proxy/router nodes + + Args: + max_jobs (int): Maximum number of concurrent jobs to launch within + the RMSNorm test to stress the kernel. + """ + log.info('#================ * * * =========================#') + log.info('Run rmsnorm2d') + log.info('#================ * * * =========================#') + cmd = "bash -c " + shlex.quote( + f"MAX_JOBS={max_jobs} python /sgl-workspace/aiter/op_tests/test_rmsnorm2d.py > /tmp/rsmnorm_test.log 2>&1 &" + ) + for hosts in (self.prefill_node_list, self.decode_node_list, self.proxy_node): + self._container_exec(cmd, hosts=hosts) + log.info('Wait 180 secs for tests to complete') + time.sleep(180) + for hosts in (self.prefill_node_list, self.decode_node_list, self.proxy_node): + out_dict = self._container_exec( + "bash -c " + shlex.quote("cat /tmp/rsmnorm_test.log"), + hosts=hosts, + ) + for node, out in out_dict.items(): + if re.search('fail', out or '', re.I): + log.warning(f'Some failures observed in test rmsnorm on node {node}') + fail_test(f'Some failures observed in test rmsnorm on node {node}') + + def launch_prefill_servers(self, dtype='auto', kv_cache_dtype='auto'): + """ + Generate and stage Prefill server launch scripts on all Prefill nodes + for SGLang disaggregated inference. + + Purpose: + -------- + This method prepares the launch script for SGLang Prefill servers. + In disaggregated PD (Prefill / Decode) mode: + - Prefill servers are responsible for processing input prompts + - They generate KV cache entries + - KV cache is later consumed by Decode servers + + This method: + - Creates one launch script per Prefill node + - Sets distributed environment variables (NNODES, NODE_RANK) + - Configures SGLang for Prefill-only execution + - Does NOT start the servers yet; it stages the script for later execution + + Args: + dtype (str): Model compute datatype (e.g., fp16, bf16, auto) + kv_cache_dtype (str): KV cache datatype (e.g., fp16, bf16, auto) + """ + log.info('#================ * * * =========================#') + log.info('Create Prefill launch script on Prefill nodes') + log.info('#================ * * * =========================#') + + prefill_node_list = self.prefill_node_list + log.info('%%%% self.prefill_nnodes {}'.format(self.prefill_nnodes)) + dist_init_addr = f"{self.inf_dict['prefill_coordinator_addr']}:{self.inf_dict['prefill_coordinator_port']}" + flags_block = add_cli_flags_block(self.bp_dict, indent=' ') + + for i in range(0, int(self.prefill_nnodes)): + node = prefill_node_list[i] + launch_body = ( + f"export NNODES={self.prefill_nnodes}\n" + f"export NODE_RANK={i}\n" + f"python3 -m sglang.launch_server --model {self.bp_dict['model']} \\\n" + f" --disaggregation-mode prefill \\\n" + f" --disaggregation-ib-device {self.inf_dict['nccl_ib_hca']} \\\n" + f" --host {node} \\\n" + f" --port {self.inf_dict['prefill_serv_port']} \\\n" + f" --dtype {dtype} \\\n" + f" --kv-cache-dtype {kv_cache_dtype} \\\n" + f" --trust-remote-code \\\n" + f" --tp-size {self.bp_dict['tensor_parallelism']} \\\n" + f" --pp-size {self.bp_dict['pipeline_parallelism']} \\\n" + f" --nnodes {self.prefill_nnodes} \\\n" + f" --node-rank {i} \\\n" + f" --dist-init-addr {dist_init_addr} \\\n" + f" --disable-radix-cache --disable-cuda-graph \\\n" + f" --mem-fraction-static {self.bp_dict['memory_fraction']} \\\n" + f"{flags_block}\n" + f" --log-level {self.inf_dict['log_level']}\n" + ) + write_cmd = "bash -c " + shlex.quote(f"cat > /tmp/prefill_launch_script.sh <<'EOF'\n{launch_body}EOF") + self._container_exec(write_cmd, hosts=[node]) + + log.info('#================ * * * =========================#') + log.info('Launching Prefill servers on Prefill nodes') + log.info('#================ * * * =========================#') + for i in range(0, int(self.prefill_nnodes)): + node = prefill_node_list[i] + start_cmd = "bash -c " + shlex.quote( + f"chmod 755 /tmp/prefill_launch_script.sh\n" + f"mkdir -p {self.log_dir}/prefill_node{i}\n" + f"source /tmp/prefill_env_script.sh\n" + f"nohup /tmp/prefill_launch_script.sh > " + f"{self.log_dir}/prefill_node{i}/prefill_server.log 2>&1 &" + ) + self._container_exec(start_cmd, hosts=[node]) + time.sleep(5) + + def launch_decode_servers(self, dtype='auto', kv_cache_dtype='auto'): + """ + Generate and deploy Decode server launch scripts on all Decode nodes + for SGLang disaggregated inference. + + Purpose: + -------- + In disaggregated PD (Prefill / Decode) inference: + - Decode servers are responsible for token generation + - They consume KV cache generated by Prefill servers + - They perform the latency- and throughput-critical decode loop + + This method: + - Creates one Decode launch script per Decode node + - Sets distributed environment variables (NNODES, NODE_RANK) + - Configures SGLang for Decode-only execution + - Deploys the scripts to Decode nodes for later execution + + Args: + dtype (str): Model compute datatype (e.g., fp16, bf16, auto) + kv_cache_dtype (str): KV cache datatype (e.g., fp16, bf16, auto) + """ + log.info('#================ * * * =========================#') + log.info('Create Decode launch script on Decode nodes') + log.info('#================ * * * =========================#') + + decode_node_list = self.decode_node_list + log.info('%%%% self.decode_nnodes {}'.format(self.decode_nnodes)) + dist_init_addr = f"{self.inf_dict['decode_coordinator_addr']}:{self.inf_dict['decode_coordinator_port']}" + flags_block = add_cli_flags_block(self.bp_dict, indent=' ') + + for i in range(0, int(self.decode_nnodes)): + node = decode_node_list[i] + launch_body = ( + f"export NNODES={self.decode_nnodes}\n" + f"export NODE_RANK={i}\n" + f"python3 -m sglang.launch_server --model {self.bp_dict['model']} \\\n" + f" --disaggregation-mode decode \\\n" + f" --disaggregation-ib-device {self.inf_dict['nccl_ib_hca']} \\\n" + f" --host {node} \\\n" + f" --port {self.inf_dict['decode_serv_port']} \\\n" + f" --trust-remote-code \\\n" + f" --dtype {dtype} \\\n" + f" --kv-cache-dtype {kv_cache_dtype} \\\n" + f" --tp-size {self.bp_dict['tensor_parallelism']} \\\n" + f" --pp-size {self.bp_dict['pipeline_parallelism']} \\\n" + f" --nnodes {self.decode_nnodes} \\\n" + f" --node-rank {i} \\\n" + f" --dist-init-addr {dist_init_addr} \\\n" + f" --disable-radix-cache --disable-cuda-graph \\\n" + f" --mem-fraction-static {self.bp_dict['memory_fraction']} \\\n" + f"{flags_block}\n" + f" --log-level {self.inf_dict['log_level']}\n" + ) + write_cmd = "bash -c " + shlex.quote(f"cat > /tmp/decode_launch_script.sh <<'EOF'\n{launch_body}EOF") + self._container_exec(write_cmd, hosts=[node]) + + log.info('#================ * * * =========================#') + log.info('Launching Decode servers on Decode nodes') + log.info('#================ * * * =========================#') + for i in range(0, int(self.decode_nnodes)): + node = decode_node_list[i] + start_cmd = "bash -c " + shlex.quote( + f"chmod 755 /tmp/decode_launch_script.sh\n" + f"mkdir -p {self.log_dir}/decode_node{i}\n" + f"source /tmp/decode_env_script.sh\n" + f"nohup bash /tmp/decode_launch_script.sh > " + f"{self.log_dir}/decode_node{i}/decode_server.log 2>&1 &" + ) + self._container_exec(start_cmd, hosts=[node]) + + def poll_and_check_server_ready( + self, + ): + """ + Wait for Prefill and Decode servers to initialize and verify that they + are fully ready to accept inference requests. + + Purpose: + -------- + After launching Prefill and Decode server scripts, the servers require + time to: + - Initialize Python runtime + - Load model weights + - Allocate GPU memory + - Initialize RDMA / NCCL / Gloo communication + - Bind to network ports + + This method enforces a startup delay and then actively polls each server + to confirm readiness before inference traffic is sent. + """ + log.info('Waiting 120 secs after launching decode script') + time.sleep(120) + self.poll_for_server_ready(0, 'prefill') + self.poll_for_server_ready(0, 'decode') + + def launch_proxy_router( + self, + ): + """ + Generate and launch the SGLang Proxy Router for disaggregated + Prefill/Decode (PD) inference. + + Purpose: + -------- + The Proxy Router is the control-plane and data-plane entry point for + inference traffic in a disaggregated PD deployment. + + Responsibilities: + - Accept incoming inference requests + - Route prefill requests to Prefill servers + - Route decode requests to Decode servers + - Coordinate Prefill -> Decode handoff + + This method: + - Builds routing configuration dynamically based on cluster topology + - Creates a launch script on the Proxy Router node + - Launches the router as a background service + """ + prefill_str = ( + f"--prefill http://{self.inf_dict['prefill_coordinator_addr']}:{self.inf_dict['prefill_serv_port']} " + ) + decode_str = f"--decode http://{self.inf_dict['decode_coordinator_addr']}:{self.inf_dict['decode_serv_port']} " + log.info('#================ * * * =========================#') + log.info('Create Proxy Router launch script on Proxy Router nodes') + log.info('#================ * * * =========================#') + + launch_body = ( + "python3 -m sglang_router.launch_router \\\n" + f" --pd-disaggregation \\\n" + f" {prefill_str.strip()} \\\n" + f" {decode_str.strip()} \\\n" + f" --host 0.0.0.0 \\\n" + f" --port {self.router_serv_port} \\\n" + f" --log-dir {self.inf_dict['log_dir']}\n" + ) + write_cmd = "bash -c " + shlex.quote(f"cat > /tmp/proxy_router_launch_script.sh <<'EOF'\n{launch_body}EOF") + self._container_exec(write_cmd, hosts=self.proxy_node) + + log.info('#================ * * * =========================#') + log.info('Launch Proxy Router script on Proxy Router nodes') + log.info('#================ * * * =========================#') + start_cmd = "bash -c " + shlex.quote( + f"chmod 755 /tmp/proxy_router_launch_script.sh\n" + f"mkdir -p {self.log_dir}/proxy_router_node\n" + f"source /tmp/router_env_script.sh\n" + f"nohup bash /tmp/proxy_router_launch_script.sh > " + f"{self.log_dir}/proxy_router_node/proxy_router.log 2>&1 &" + ) + self._container_exec(start_cmd, hosts=self.proxy_node) + log.info('Waiting 120 secs after launching proxy router script') + time.sleep(120) + + def benchserv_test_random(self, d_type='auto'): + """ + Run SGLang serving benchmark using a synthetic random dataset and + validate inference performance and correctness. + + Purpose: + -------- + This benchmark exercises the inference serving stack using randomly + generated input/output sequences to: + - Stress-test request scheduling and batching + - Evaluate sustained throughput under synthetic load + - Validate end-to-end serving stability independent of real datasets + + The benchmark targets the Proxy Router endpoint, ensuring that + Prefill, Decode, and routing logic work together correctly. + + Args: + d_type (str): Data type identifier used to select expected + performance thresholds (e.g., fp16, bf16, auto). + """ + log.info('#================ * * * =========================#') + log.info('Benchmark Random Dataset') + log.info('#================ * * * =========================#') + i_dict = self.bp_dict['inference_tests']['bench_serv_random'] + self._bench_num_prompts = int(i_dict['num_prompts']) + inner = ( + f"mkdir -p {self.log_dir}/benchmark_node\n" + f"source /tmp/benchmark_env_script.sh\n" + f"export PYTHONPATH=/sgl-workspace/sglang/python:${{PYTHONPATH:-}}\n" + f"python3 -m sglang.bench_serving \\\n" + f" --backend {i_dict['backend']} \\\n" + f" --dataset-name random \\\n" + f" --num-prompts {i_dict['num_prompts']} \\\n" + f" --max-concurrency {self.bp_dict['max_concurrency']} \\\n" + f" --random-input {i_dict['input_length']} \\\n" + f" --random-output {i_dict['output_length']} \\\n" + f" --random-range-ratio {i_dict['random_range_ratio']} \\\n" + f" --host {self.client_host} --port {self.router_serv_port} \\\n" + f" > {self.log_dir}/benchmark_node/benchmark_results.log 2>&1" + ) + self._container_exec( + "bash -c " + shlex.quote(inner), + hosts=self.benchmark_serv_node, + timeout=1000, + ) + time.sleep(5) + self.poll_for_inference_completion(iterations=10, waittime_between_iters=60) + + peak_tflops = float(i_dict.get("peak_gpu_tflops", 1300)) + num_params = float(i_dict.get("model_num_params", 70e9)) + tp = int(self.bp_dict.get("tensor_parallelism", 1)) + pp = int(self.bp_dict.get("pipeline_parallelism", 1)) + num_gpus = (int(self.prefill_nnodes) + int(self.decode_nnodes)) * tp * pp + for node, m in (self.inference_results_dict or {}).items(): + duration = float(m.get("benchmark_duration") or 0) + in_tok = float(m.get("total_input_tokens") or 0) + out_tok = float(m.get("total_generated_tokens") or m.get("Total generated tokens:") or 0) + if duration > 0 and num_gpus > 0: + achieved = 6.0 * num_params * (in_tok + out_tok) + peak = peak_tflops * 1e12 * num_gpus * duration + m["mfu"] = f"{achieved / peak:.6f}" + + log_path = f"{self.log_dir}/benchmark_node/benchmark_results.log" + for node, m in (self.inference_results_dict or {}).items(): + gp = m.get("goodput", "n/a") + tpg = m.get("output_throughput_per_gpu_per_sec", "n/a") + tr = m.get("total_requests", "n/a") + sr = m.get("successful_requests", "n/a") + mfu = m.get("mfu", "n/a") + append_inner = ( + f"echo '' >> {log_path} && " + f"echo '============ Derived Benchmark Results ============' >> {log_path} && " + f"echo 'Goodput (successful / total): {sr} / {tr} => {gp}' >> {log_path} && " + f"echo 'Output token throughput per GPU (tok/s/GPU): {tpg}' >> {log_path} && " + f"echo 'MFU (estimated): {mfu}' >> {log_path} && " + f"echo '=====================================================================' >> {log_path}" + ) + self._container_exec( + "bash -c " + shlex.quote(append_inner), + hosts=self.benchmark_serv_node, + ) + + self.verify_inference_results('bench_serv', i_dict['expected_results'][d_type]) + + def poll_for_server_ready(self, node_no, sglang_function, no_of_iterations=16): + """Poll Prefill or Decode server logs inside the container for readiness.""" + if re.search('prefill', sglang_function): + self._poll_role_log_ready( + f"{self.log_dir}/prefill_node{node_no}/prefill_server.log", + [self.prefill_node_list[node_no]], + f'Prefill node {node_no}', + no_of_iterations, + ) + elif re.search('decode', sglang_function): + self._poll_role_log_ready( + f"{self.log_dir}/decode_node{node_no}/decode_server.log", + [self.decode_node_list[node_no]], + f'Decode node {node_no}', + no_of_iterations, + ) + + def _poll_role_log_ready( + self, + log_path: str, + hosts: list[str], + label: str, + no_of_iterations: int = 16, + ) -> None: + for iteration in range(1, no_of_iterations): + log.info('Starting %s readiness poll iteration %d', label, iteration) + grep_cmd = f"grep -B 20 -A 20 -E {_SERVER_READY_RE.pattern!r} {shlex.quote(log_path)} || true" + text = self._container_exec_text(grep_cmd, hosts=hosts) + if _SERVER_READY_RE.search(text): + log.info('Wait 60 secs before serving traffic') + time.sleep(60) + return + log.info('Wait 120 secs and continue polling') + time.sleep(120) + fail_test(f'{label} on {hosts[0]!r} did not reach ready state in {no_of_iterations} iterations') + + def get_inference_results_dict(self, out_dict): + """ + Parse inference benchmark output logs and extract key performance metrics + into a structured dictionary. + + Purpose: + -------- + This method processes raw text output generated by inference benchmarks + (e.g., sglang.bench_serving) and extracts important metrics such as: + - Request counts + - Token throughput + - Latency statistics (TTFT, TPOT) + - Benchmark duration + + The extracted metrics are stored per node in: + self.inference_results_dict + + Args: + out_dict (dict): + Dictionary keyed by node identifier, where each value is the + raw stdout/stderr text produced by the benchmark on that node. + """ + self.inference_results_dict = {} + log.info('Inside get_inference_results_dict') + log.info("%s", out_dict) + + for node in out_dict.keys(): + self.inference_results_dict[node] = {} + if re.search('Successful requests:', out_dict[node], re.I): + match = re.search('Successful requests:\s+([0-9]+)', out_dict[node], re.I) + self.inference_results_dict[node]['successful_requests'] = match.group(1) + if re.search('Benchmark duration\s+\(s\):\s+([0-9]+)', out_dict[node], re.I): + match = re.search('Benchmark duration\s+\(s\):\s+([0-9]+)', out_dict[node], re.I) + self.inference_results_dict[node]['benchmark_duration'] = match.group(1) + if re.search('Total input tokens:', out_dict[node], re.I): + match = re.search('Total input tokens:\s+([0-9\.]+)', out_dict[node], re.I) + self.inference_results_dict[node]['total_input_tokens'] = match.group(1) + if re.search('Total generated tokens:', out_dict[node], re.I): + match = re.search('Total generated tokens:\s+([0-9\.]+)', out_dict[node], re.I) + self.inference_results_dict[node]['total_generated_tokens'] = match.group(1) + if re.search('Request throughput \(req/s\):', out_dict[node], re.I): + match = re.search('Request throughput \(req/s\):\s+([0-9\.]+)', out_dict[node], re.I) + self.inference_results_dict[node]['request_throughput_per_sec'] = match.group(1) + if re.search('Output token throughput \(tok/s\):', out_dict[node], re.I): + match = re.search('Output token throughput \(tok/s\):\s+([0-9\.]+)', out_dict[node], re.I) + self.inference_results_dict[node]['output_throughput_per_sec'] = match.group(1) + if re.search('Mean TTFT \(ms\):', out_dict[node], re.I): + match = re.search('Mean TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) + self.inference_results_dict[node]['mean_ttft_ms'] = match.group(1) + if re.search('Median TTFT (ms):', out_dict[node], re.I): + match = re.search('Median TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) + self.inference_results_dict[node]['median_ttft_ms'] = match.group(1) + if re.search('P99 TTFT (ms):', out_dict[node], re.I): + match = re.search('P99 TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) + self.inference_results_dict[node]['p99_ttft_ms'] = match.group(1) + if re.search('Mean TPOT \(ms\)', out_dict[node], re.I): + match = re.search('Mean TPOT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) + self.inference_results_dict[node]['mean_tpot_ms'] = match.group(1) + if re.search('Median TPOT \(ms\):', out_dict[node], re.I): + match = re.search('Median TPOT \(ms\):\s+([0-9]+)', out_dict[node], re.I) + self.inference_results_dict[node]['median_tpot_ms'] = match.group(1) + if re.search('P99 TPOT (ms):', out_dict[node], re.I): + match = re.search('P99 TPOT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) + self.inference_results_dict[node]['p99_tpot_ms'] = match.group(1) + if re.search('Mean ITL \(ms\):', out_dict[node], re.I): + match = re.search('Mean ITL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) + self.inference_results_dict[node]['mean_itl_ms'] = match.group(1) + if re.search('Median ITL \(ms\):', out_dict[node], re.I): + match = re.search('Median ITL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) + self.inference_results_dict[node]['median_itl_ms'] = match.group(1) + if re.search('P99 ITL \(ms\):', out_dict[node], re.I): + match = re.search('P99 ITL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) + self.inference_results_dict[node]['p99_itl_ms'] = match.group(1) + m = first_float(r'Mean E2E Latency \(ms\):\s+([0-9\.]+)', out_dict[node]) + if m: + self.inference_results_dict[node]['mean_e2e_latency_ms'] = m + m = first_float(r'Median E2E Latency \(ms\):\s+([0-9\.]+)', out_dict[node]) + if m: + self.inference_results_dict[node]['median_e2e_latency_ms'] = m + for p in (90, 95, 99): + m = first_float(rf'P{p} E2E Latency \(ms\):\s+([0-9\.]+)', out_dict[node]) + if m: + self.inference_results_dict[node][f'p{p}_e2e_latency_ms'] = m + + total_req = first_float(r"Total requests:\s+([0-9]+)", out_dict[node]) + failed_req = first_float(r"Failed requests:\s+([0-9]+)", out_dict[node]) + succ = self.inference_results_dict[node].get("successful_requests") + if total_req: + self.inference_results_dict[node]["total_requests"] = total_req + elif succ is not None and failed_req is not None: + self.inference_results_dict[node]["total_requests"] = str(int(succ) + int(failed_req)) + elif succ is not None and getattr(self, "_bench_num_prompts", None) is not None: + self.inference_results_dict[node]["total_requests"] = str(int(self._bench_num_prompts)) + if succ and self.inference_results_dict[node].get("total_requests"): + s, t = int(succ), int(self.inference_results_dict[node]["total_requests"]) + self.inference_results_dict[node]["goodput"] = f"{(s / t):.6f}" if t else None + + out_tps = self.inference_results_dict[node].get("output_throughput_per_sec") + if out_tps: + tp = int(self.bp_dict.get("tensor_parallelism", "1")) + pp = int(self.bp_dict.get("pipeline_parallelism", "1")) + ng = (int(self.prefill_nnodes) + int(self.decode_nnodes)) * tp * pp + if ng > 0: + self.inference_results_dict[node]["output_throughput_per_gpu_per_sec"] = ( + f"{float(out_tps) / ng:.6f}" + ) + + log.info("%s", self.inference_results_dict) + return self.inference_results_dict + + def scan_for_inference_errors( + self, + ): + """ + Scan Prefill and Decode server logs for known inference error patterns + and fail the test if any are detected. + + Purpose: + -------- + This method performs a post-inference health check by scanning + server logs for known error signatures that indicate: + - Runtime failures + - Communication errors (RDMA/NCCL) + - Out-of-memory conditions + - Kernel or backend crashes + - Fatal exceptions during inference + + The method ensures that even if benchmarks complete, silent or + non-fatal errors do not go unnoticed. + """ + log.info('Scan for inference errors') + inference_pass = True + + for j in range(0, int(self.prefill_nnodes)): + node = self.prefill_node_list[j] + out_dict = self._container_exec( + f"tail -100 {shlex.quote(f'{self.log_dir}/prefill_node{j}/prefill_server.log')}", + hosts=[node], + ) + out = out_dict.get(node, '') + for err_key in inference_err_dict: + if re.search(f'{inference_err_dict[err_key]}', out): + fail_test(f'ERROR {inference_err_dict[err_key]} seen in inference logs ..') + log.error('Aborting inference log polling') + inference_pass = False + + for j in range(0, int(self.decode_nnodes)): + node = self.decode_node_list[j] + out_dict = self._container_exec( + f"tail -500 {shlex.quote(f'{self.log_dir}/decode_node{j}/decode_server.log')}", + hosts=[node], + ) + out = out_dict.get(node, '') + for err_key in inference_err_dict: + if re.search(f'{inference_err_dict[err_key]}', out): + fail_test(f'ERROR {inference_err_dict[err_key]} seen in inference logs ..') + log.error('Aborting inference log polling') + inference_pass = False + + return inference_pass + + def poll_for_inference_completion( + self, iterations=10, waittime_between_iters=60, total_timeout=3600, require_all_nodes=True + ): + """ + Poll benchmark logs to detect inference completion and extract results. + + Purpose: + -------- + This method monitors inference progress by periodically inspecting + benchmark output logs. It determines when inference has completed, + detects early failures, and enforces a global timeout. + + Completion criteria: + -------------------- + Inference is considered complete when the benchmark output contains + the pattern 'Serving Benchmark Result'. + + Failure criteria: + ----------------- + Any known inference error detected in Prefill or Decode logs + immediately aborts the process. + + Args: + iterations (int): + Maximum number of polling iterations. + waittime_between_iters (int): + Time (seconds) to wait between polling attempts. + total_timeout (int or None): + Maximum wall-clock time (seconds) allowed for inference. + require_all_nodes (bool): + If True, all nodes must report completion. + If False, completion by any node is sufficient. + """ + time.sleep(60) + + start_time = time.time() + + def timed_out() -> bool: + return total_timeout is not None and (time.time() - start_time) >= float(total_timeout) + + completed_pattern = re.compile('Serving Benchmark Result', re.I) + log_path = f"{self.log_dir}/benchmark_node/benchmark_results.log" + + for itr in range(1, iterations + 1): + log.info(f'Starting iteration {itr}') + + out_dict = self._container_exec( + f"tail -1000 {shlex.quote(log_path)}", + hosts=self.benchmark_serv_node, + ) + + node_completion = {} + for node, output in out_dict.items(): + node_completion[node] = bool(completed_pattern.search(output or '')) + + if require_all_nodes: + all_complete = all(node_completion.values()) if node_completion else False + else: + all_complete = any(node_completion.values()) if node_completion else False + + if not all_complete: + if timed_out(): + msg = f"Timeout while waiting for inference completion after ~{int(time.time() - start_time)}s" + log.warning("%s", msg) + return {"status": "timeout", "reason": msg} + log.info('Inference still in progress') + time.sleep(30) + time.sleep(int(waittime_between_iters)) + continue + + self.get_inference_results_dict(out_dict) + log.info('Completed Inference, returning !!!') + return {"status": "success", "results": self.inference_results_dict} + + if timed_out(): + msg = f"Timeout after maximum iterations ({self.inference_poll_iterations}) and ~{int(time.time() - start_time)}s" + log.warning("%s", msg) + return {"status": "timeout", "reason": msg} + msg = f"Reached iteration cap ({self.inference_poll_iterations}) without completion; still in progress" + log.warning("%s", msg) + return {"status": "stuck_in_progress", "reason": msg} + + def verify_inference_results(self, test_name, expected_result_dict): + """ + Validate inference benchmark results against expected performance + thresholds and check for system-level errors. + + Comparison rules (via ``evaluate_all`` + threshold ``kind``): + - Throughput, req/s, goodput, MFU: actual >= expected + - Latency (*_ms, *latency*): actual <= expected + + Threshold entries may be full specs ``{"kind": ..., "value": ...}`` from + threshold.json or legacy flat floats from ``flat_expected_from_specs``. + """ + thresholds = { + metric: normalize_sglang_threshold_spec(metric, spec) for metric, spec in expected_result_dict.items() + } + + for node in self.inference_results_dict: + actuals = { + metric: coerce_sglang_actual(value) + for metric, value in self.inference_results_dict[node].items() + if metric in thresholds + } + try: + evaluate_all(actuals, thresholds) + except ThresholdViolation as exc: + for msg in exc.violations: + fail_test(f"FAIL - {msg}") + + self.inference_end_time = self._host_exec('date +"%a %b %e %H:%M"') + time.sleep(2) + verify_dmesg_for_errors(self.orch.all, self.inference_start_time, self.inference_end_time) + + def sglang_disagg_gpu_counts(self, mem_threshold_mb=5000): + tp = int(self.bp_dict["tensor_parallelism"]) + pp = int(self.bp_dict.get("pipeline_parallelism", 1)) + + topo = collect_sglang_gpu_topology( + self._host_exec, + { + "prefill": self.prefill_node_list, + "decode": self.decode_node_list, + }, + mem_threshold_mb=mem_threshold_mb, + ) + prefill = topo["groups"]["prefill"] + decode = topo["groups"]["decode"] + + result = { + "configured_tp": tp, + "configured_pp": pp, + "prefill_per_node": prefill["per_node"], + "decode_per_node": decode["per_node"], + "prefill_occupied_gpus": prefill["total"], + "decode_occupied_gpus": decode["total"], + "total_occupied_gpus": topo["total_occupied_gpus"], + } + log.info( + "\n".join( + format_sglang_gpu_topology_lines( + configured_tp=tp, + configured_pp=pp, + groups={"Prefill": prefill, "Decode": decode}, + ) + ) + ) + return result + + def verify_openai_compatible_endpoints(self) -> list[str]: + """ + Smoke-test OpenAI-compatible HTTP API on the proxy router (inside the + benchmark container): GET /v1/models, + POST /v1/chat/completions, POST /v1/completions, and structured JSON + (book) via chat completions. + """ + port = int(self.router_serv_port) + model_name = self.bp_dict["model"] + + probe_src = OpenAIProbe.probe_script(port, model_name, host=self.client_host) + b64 = base64.b64encode(probe_src.encode("utf-8")).decode("ascii") + inner = ( + f"mkdir -p {self.log_dir}/benchmark_node && " + f"echo {shlex.quote(b64)} | base64 -d > /tmp/openai_mq_probe.py && " + f"python3 /tmp/openai_mq_probe.py && rm -f /tmp/openai_mq_probe.py" + ) + log.info( + "OpenAI endpoint probe inside benchmark container (%s:%r), same pattern as GSM8K/benchserv", + self.client_host, + port, + ) + out_dict = self._container_exec( + "bash -c " + shlex.quote(inner), + hosts=self.benchmark_serv_node, + timeout=min(900, 480 + 180), + ) + bench_host = self.benchmark_serv_node[0] + raw_out = out_dict.get(bench_host) or self._first_output(out_dict) + + probe_err: Optional[str] = None + results: dict[str, tuple[int, Any]] = {} + if not raw_out or not str(raw_out).strip(): + probe_err = f"OpenAI-compatible probe produced no output on {bench_host!r}: {out_dict!r}" + else: + lines_out = str(raw_out).strip().splitlines() + if not lines_out: + probe_err = f"OpenAI-compatible probe empty lines after strip on node {bench_host!r}: {raw_out!r}" + else: + last_line = lines_out[-1] + try: + parsed = json.loads(last_line) + except json.JSONDecodeError as e: + probe_err = f"OpenAI-compatible probe invalid JSON: {e!r} raw={raw_out!r}" + else: + if not isinstance(parsed, dict): + probe_err = f"OpenAI-compatible probe expected JSON object, got {type(parsed).__name__!r}" + else: + for step, val in parsed.items(): + if isinstance(val, (list, tuple)) and len(val) == 2: + results[step] = (int(val[0]), val[1]) + else: + probe_err = f"OpenAI-compatible probe bad shape at {step!r}: {val!r}" + break + + if probe_err is not None: + fail_test(probe_err) + return [] + + OpenAIProbe.log_results(results, log) + + ok, err = OpenAIProbe.check_results(results, port=port, logger=log) + if not ok: + summary = OpenAIProbe.summarize_results(results, ok, err) + fail_test(f"{err}") + return summary + + summary = OpenAIProbe.summarize_results(results, ok, err) + return summary + + def run_lm_eval_hellaswag_benchmark_test(self, _d_type="auto"): + return self.run_lm_eval_benchmark_test("lm_eval_hellaswag", _d_type=_d_type) + + def run_lm_eval_gsm8k_benchmark_test(self, _d_type="auto"): + return self.run_lm_eval_benchmark_test("lm_eval_gsm8k", _d_type=_d_type) + + def run_lm_eval_benchmark_test(self, bench_key: str, _d_type="auto"): + spec = LM_EVAL_SPECS[bench_key] + log.info("#================ * * * =========================#") + log.info("lm-eval %s benchmark", spec["display"]) + log.info("#================ * * * =========================#") + task_name = bench_key.removeprefix("lm_eval_") + i_dict = self.bp_dict["inference_tests"][bench_key] + inner_cmd, scoring = LmEvalBenchmark.prepare( + i_dict, + port=int(self.router_serv_port), + host=self.client_host, + model_id=self.bp_dict["model"], + task_name=task_name, + default_tasks=task_name, + default_metric=spec["default_metric"], + default_metric_key=spec["default_metric_key"], + log_dir=self.log_dir, + log_basename=f"{bench_key}.log", + default_num_concurrent=spec["default_num_concurrent"], + ) + + inner = f"mkdir -p {self.log_dir}/benchmark_node && source /tmp/benchmark_env_script.sh && {inner_cmd}" + out_dict = self._container_exec( + "bash -c " + shlex.quote(inner), + hosts=self.benchmark_serv_node, + timeout=scoring["exec_timeout_sec"], + ) + time.sleep(5) + + check_kwargs = LmEvalBenchmark.check_kwargs_from_scoring(scoring) + summary = None + errors: list[str] = [] + + for node, text in out_dict.items(): + ok, node_summary, err = LmEvalBenchmark.check_results(text, **check_kwargs) + if node_summary is not None: + summary = node_summary + if not ok: + errors.append(f"lm-eval {spec['display']} on node {node!r}: {err}") + + if summary is None: + summary = LmEvalBenchmark.fallback_summary( + scoring, + error=errors[-1] if errors else "no benchmark nodes produced output to score", + ) + errors.append(f"lm-eval {spec['display']}: no benchmark nodes produced output to score") + + for msg in errors: + fail_test(msg) + + return summary + + def run_long_context_niah_accuracy(self, *, isl: int, osl: int, d_type: str = "auto"): + """NIAH long-context accuracy at fixed ISL/OSL via /v1/chat/completions.""" + bench_key = "long_ctx_niah" + i_dict = self.bp_dict["inference_tests"][bench_key] + port = int(self.router_serv_port) + log_basename = f"long_ctx_niah_isl{isl}_osl{osl}.log" + + inner_cmd, scoring = LongContextNiahBenchmark.prepare( + i_dict, + port=port, + host=self.client_host, + model_id=self.bp_dict["model"], + isl=int(isl), + osl=int(osl), + log_dir=self.log_dir, + log_basename=log_basename, + ) + probe_src = LongContextNiahBenchmark.probe_script(**scoring["probe_kwargs"]) + b64 = base64.b64encode(probe_src.encode("utf-8")).decode("ascii") + + inner = ( + f"mkdir -p {self.log_dir}/benchmark_node && " + f"echo {shlex.quote(b64)} | base64 -d > /tmp/long_ctx_niah_probe.py && " + f"source /tmp/benchmark_env_script.sh && {inner_cmd}" + ) + out_dict = self._container_exec( + "bash -c " + shlex.quote(inner), + hosts=self.benchmark_serv_node, + timeout=int(scoring["exec_timeout_sec"]), + ) + time.sleep(5) + + check_kwargs = LongContextNiahBenchmark.check_kwargs_from_scoring(scoring) + summary = None + errors: list[str] = [] + + for node, text in out_dict.items(): + ok, node_summary, err = LongContextNiahBenchmark.check_results(text, **check_kwargs) + if node_summary is not None: + summary = node_summary + if not ok: + errors.append(f"long_ctx_niah on node {node!r}: {err}") + + if summary is None: + summary = { + "task": "long_ctx_niah", + "metric_key": scoring["metric_key"], + "actual": None, + "expected": float(scoring["expected"]), + "passed": False, + "error": errors[-1] if errors else "no benchmark output", + } + errors.append("long_ctx_niah: no benchmark nodes produced output to score") + + for msg in errors: + fail_test(msg) + + return summary diff --git a/cvs/lib/inference/sglang/sglang_distributed_lib.py b/cvs/lib/inference/sglang/sglang_distributed_lib.py new file mode 100644 index 000000000..c9d8a6637 --- /dev/null +++ b/cvs/lib/inference/sglang/sglang_distributed_lib.py @@ -0,0 +1,624 @@ +''' +Copyright 2026 Advanced Micro Devices, Inc. +All rights reserved. + +Multi-node unified SGLang inference controller (TP/PP across server nodes, no PD disagg). + +Each host in ``server_node_list`` (or the union of ``prefill_node_list`` + +``decode_node_list``) runs ``sglang.launch_server`` with ``--nnodes`` / +``--node-rank`` / ``--dist-init-addr``. Benchmark/smoke/lm-eval run on +``benchmark_serv_node`` and target rank-0 HTTP (``127.0.0.1`` when bench is rank 0). +''' + +from __future__ import annotations + +import base64 +import json +import os +import re +import shlex +import time +from typing import Any, Optional + +from cvs.lib import globals +from cvs.core.orchestrators.baremetal import BaremetalOrchestrator +from cvs.lib.inference.sglang.sglang_common import ( + LM_EVAL_SPECS, + add_cli_flags_block, + add_export_env_block, + as_node_list, + coerce_sglang_actual, + first_float, + normalize_sglang_threshold_spec, + resolve_distributed_client_host, + resolve_server_node_list, + collect_sglang_gpu_topology, + format_sglang_gpu_topology_lines, + _SERVER_READY_RE, +) +from cvs.lib.utils.model_query_lib import LmEvalBenchmark, OpenAIProbe +from cvs.lib.utils_lib import fail_test +from cvs.lib.utils.verdict import ThresholdViolation, evaluate_all +from cvs.lib.verify_lib import verify_dmesg_for_errors + +log = globals.log + + +class SglangDistributed: + """Unified multi-node SGLang serve + benchmark via ``ContainerOrchestrator``.""" + + def __init__( + self, + model_name, + inference_config_dict, + benchmark_params_dict, + hf_token, + orch=None, + gpu_type='mi325', + user_name=None, + priv_key_file=None, + ): + if orch is None: + raise ValueError("SglangDistributed requires orch= (ContainerOrchestrator)") + + self.orch = orch + self.user_name = user_name + self.priv_key_file = priv_key_file + self.model_name = model_name + self.hf_token = hf_token + self.gpu_type = gpu_type + + self.inf_dict = inference_config_dict + self.bp_dict = benchmark_params_dict + + self.mount_vol = self.inf_dict.get( + 'mount_vol', + '/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so', + ) + + self.inference_results_dict = {} + log.info("%s", self.gpu_type) + + self.home_dir = os.path.expanduser("~") + self._apply_inf_defaults() + self._apply_bp_defaults() + + self.server_node_list = resolve_server_node_list(self.inf_dict) + self.nnodes = int(self.inf_dict.get('nnodes') or len(self.server_node_list)) + if self.nnodes != len(self.server_node_list): + raise ValueError( + f"sglang_distributed nnodes={self.nnodes} must match " + f"server node count {len(self.server_node_list)} ({self.server_node_list!r})" + ) + self.rank0_node = self.server_node_list[0] + self.dist_init_addr = self._resolve_dist_init_addr() + self.benchmark_serv_node = self._resolve_benchmark_serv_node() + + self.container_name = self.inf_dict['container_name'] + self.nic_type = self.inf_dict['nic_type'] + self.hca_id_prefix = str(self.inf_dict['hca_id_prefix']).strip() + self.log_dir = self.inf_dict['log_dir'] + self.inference_poll_iterations = self.bp_dict['inference_poll_iterations'] + + self.inference_start_time = self._host_exec('date +"%a %b %e %H:%M"') + self.inference_end_time = None + + log.info('distributed inference_dict = %s', self.inf_dict) + log.info('distributed benchmark_params_dict = %s', self.bp_dict) + log.info( + 'distributed server_node_list=%s nnodes=%s rank0=%s client_host=%s ' + 'router_serv_port=%s benchmark_serv_node=%s dist_init=%s', + self.server_node_list, + self.nnodes, + self.rank0_node, + self.client_host, + self.router_serv_port, + self.benchmark_serv_node, + self.dist_init_addr, + ) + + def _resolve_dist_init_addr(self) -> str: + addr = self.inf_dict.get('dist_init_addr') or self.rank0_node + port = self.inf_dict.get('dist_init_port') or '40001' + return f"{addr}:{port}" + + def _resolve_benchmark_serv_node(self) -> str: + raw = self.inf_dict.get('benchmark_serv_node') + if not raw: + return self.rank0_node + hosts = as_node_list(raw) + if len(hosts) != 1: + raise ValueError(f"SglangDistributed requires exactly one benchmark_serv_node, got {hosts!r}") + return hosts[0] + + @property + def _head_host(self) -> str: + return self.rank0_node + + def server_log_path(self, rank: int = 0) -> str: + return f"{self.log_dir}/server_node{rank}/server.log" + + @property + def router_serv_port(self) -> str: + return str(self.inf_dict['proxy_router_serv_port']) + + @property + def client_host(self) -> str: + return resolve_distributed_client_host( + self.inf_dict, + rank0_node=self.rank0_node, + benchmark_serv_node=self.benchmark_serv_node, + ) + + @staticmethod + def _first_output(out_dict: dict) -> str: + if not out_dict: + return "" + return next(iter(out_dict.values())) or "" + + def _container_exec( + self, + cmd: str, + *, + hosts=None, + timeout: int | None = None, + ) -> dict: + normalized = as_node_list(hosts) if hosts is not None else self.server_node_list + return self.orch.exec(cmd, hosts=normalized, timeout=timeout) + + def _bench_exec(self, cmd: str, *, timeout: int | None = None) -> dict: + return self.orch.exec(cmd, hosts=[self.benchmark_serv_node], timeout=timeout) + + def _container_exec_text( + self, + cmd: str, + *, + hosts=None, + timeout: int | None = None, + ) -> str: + return self._first_output(self._container_exec(cmd, hosts=hosts, timeout=timeout)) + + def _host_exec( + self, + cmd: str, + *, + hosts=None, + timeout: int | None = None, + ) -> dict: + """Run ``cmd`` on baremetal (``orch.head`` / ``orch.all``), e.g. amd-smi / dmesg.""" + if hosts is None: + host = self.benchmark_serv_node + if host == self.orch.head_node and len(self.orch.hosts) == 1: + return self.orch.head.exec(cmd, timeout=timeout) + return BaremetalOrchestrator.exec(self.orch, cmd, hosts=[host], timeout=timeout) + normalized = as_node_list(hosts) + if not normalized: + return {} + if len(normalized) == 1 and normalized[0] == self._head_host: + return self.orch.head.exec(cmd, timeout=timeout) + if set(normalized) == set(self.orch.hosts): + return self.orch.all.exec(cmd, timeout=timeout) + return BaremetalOrchestrator.exec(self.orch, cmd, hosts=normalized, timeout=timeout) + + def _host_exec_text( + self, + cmd: str, + *, + hosts=None, + timeout: int | None = None, + ) -> str: + return self._first_output(self._host_exec(cmd, hosts=hosts, timeout=timeout)) + + def _apply_inf_defaults(self) -> None: + self.inf_dict.setdefault('container_image', 'lmsysorg/sglang:dev') + self.inf_dict.setdefault('container_name', 'sglang_container') + self.inf_dict.setdefault('nic_type', 'ainic') + self.inf_dict.setdefault('nccl_ib_hca', 'rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7') + self.inf_dict.setdefault('hca_id_prefix', 'bnxt_') + self.inf_dict.setdefault('nccl_socket_ifname', 'eno0') + self.inf_dict.setdefault('gloo_socket_ifname', 'eno0') + self.inf_dict.setdefault('nccl_ib_gid_index', '1') + self.inf_dict.setdefault('nccl_debug', 'ERROR') + self.inf_dict.setdefault('data_cache_dir', f'{self.home_dir}/cache') + self.inf_dict.setdefault('log_dir', f'{self.home_dir}/LOG_DIR') + self.inf_dict.setdefault('log_level', 'info') + self.inf_dict.setdefault('proxy_router_serv_port', '8000') + + def _apply_bp_defaults(self) -> None: + self.bp_dict.setdefault('backend', 'sglang') + self.bp_dict.setdefault('max_concurrency', '64') + self.bp_dict.setdefault('model', 'openai/gpt-oss-120b') + self.bp_dict.setdefault('tensor_parallelism', '8') + self.bp_dict.setdefault('pipeline_parallelism', '1') + self.bp_dict.setdefault('memory_fraction', '0.85') + self.bp_dict.setdefault('inference_poll_iterations', '16') + + def _server_env_body(self) -> str: + return ( + "export LD_LIBRARY_PATH=/usr/local/lib:/sgl-workspace/Mooncake/build/mooncake-common/etcd:/opt/rocm/lib:$LD_LIBRARY_PATH\n" + f"export NCCL_DEBUG={self.inf_dict['nccl_debug']}\n" + f"export NCCL_IB_HCA={self.inf_dict['nccl_ib_hca']}\n" + f"export NCCL_IB_GID_INDEX={self.inf_dict['nccl_ib_gid_index']}\n" + f"export NCCL_SOCKET_IFNAME={self.inf_dict['nccl_socket_ifname']}\n" + f"export GLOO_SOCKET_IFNAME={self.inf_dict['gloo_socket_ifname']}\n" + f"export GLOO_TCP_IFNAME={self.inf_dict['gloo_socket_ifname']}\n" + f"export HSA_FORCE_FINE_GRAIN_PCIE=1\n" + f"export MODEL={self.bp_dict['model']}\n" + f"export TP={self.bp_dict['tensor_parallelism']}\n" + f"export PP={self.bp_dict.get('pipeline_parallelism', '1')}\n" + f"export HF_TOKEN={self.hf_token}\n" + f"{add_export_env_block(self.bp_dict, indent='')}\n" + ) + + def _write_server_env_on_hosts(self, hosts: list[str]) -> None: + env_body = self._server_env_body() + write_cmd = "bash -c " + shlex.quote( + f"cat > /tmp/server_env_script.sh <<'EOF'\n{env_body}EOF\n" + "chmod 755 /tmp/server_env_script.sh && /tmp/server_env_script.sh" + ) + self._container_exec(write_cmd, hosts=hosts) + + def setup_server_container_env(self) -> None: + """Write and source ``/tmp/server_env_script.sh`` on all server nodes.""" + time.sleep(3) + self._write_server_env_on_hosts(self.server_node_list) + time.sleep(5) + + def setup_benchmark_serv_container_env(self) -> None: + self.setup_server_container_env() + if self.benchmark_serv_node not in self.server_node_list: + self._write_server_env_on_hosts([self.benchmark_serv_node]) + + def launch_server(self, dtype='auto', kv_cache_dtype='auto') -> None: + """Launch unified multi-node ``sglang.launch_server`` (no PD disagg).""" + log.info( + 'Launch unified multi-node SGLang on %d nodes (rank0=%s:%s)', + self.nnodes, + self.rank0_node, + self.router_serv_port, + ) + flags_block = add_cli_flags_block(self.bp_dict, indent=' ') + pp = self.bp_dict.get('pipeline_parallelism', '1') + + for i, node in enumerate(self.server_node_list): + host_flag = '0.0.0.0' if i == 0 else node + launch_body = ( + f"export NNODES={self.nnodes}\n" + f"export NODE_RANK={i}\n" + f"python3 -m sglang.launch_server --model {self.bp_dict['model']} \\\n" + f" --host {host_flag} \\\n" + f" --port {self.router_serv_port} \\\n" + f" --dtype {dtype} \\\n" + f" --kv-cache-dtype {kv_cache_dtype} \\\n" + f" --trust-remote-code \\\n" + f" --tp-size {self.bp_dict['tensor_parallelism']} \\\n" + f" --pp-size {pp} \\\n" + f" --nnodes {self.nnodes} \\\n" + f" --node-rank {i} \\\n" + f" --dist-init-addr {self.dist_init_addr} \\\n" + f" --disable-radix-cache --disable-cuda-graph \\\n" + f" --mem-fraction-static {self.bp_dict['memory_fraction']} \\\n" + f"{flags_block}\n" + f" --log-level {self.inf_dict['log_level']}\n" + ) + write_cmd = "bash -c " + shlex.quote(f"cat > /tmp/server_launch_script.sh <<'EOF'\n{launch_body}EOF") + self._container_exec(write_cmd, hosts=[node]) + + for i, node in enumerate(self.server_node_list): + start_cmd = "bash -c " + shlex.quote( + f"chmod 755 /tmp/server_launch_script.sh\n" + f"mkdir -p {self.log_dir}/server_node{i}\n" + f"source /tmp/server_env_script.sh\n" + f"nohup /tmp/server_launch_script.sh > {self.server_log_path(i)} 2>&1 &" + ) + self._container_exec(start_cmd, hosts=[node]) + time.sleep(5) + + def poll_for_server_ready(self, no_of_iterations=16) -> None: + log_path = self.server_log_path(0) + for iteration in range(1, no_of_iterations): + log.info('Starting rank-0 server readiness poll iteration %d', iteration) + grep_cmd = f"grep -B 20 -A 20 -E {_SERVER_READY_RE.pattern!r} {shlex.quote(log_path)} || true" + text = self._container_exec_text(grep_cmd, hosts=[self.rank0_node]) + if _SERVER_READY_RE.search(text): + log.info('Wait 60 secs before serving traffic') + time.sleep(60) + return + log.info('Wait 120 secs and continue polling') + time.sleep(120) + fail_test( + f'Distributed rank-0 server on {self.rank0_node} did not reach ready state in {no_of_iterations} iterations' + ) + + def poll_and_check_server_ready(self) -> None: + log.info('Waiting 120 secs after launching distributed server') + time.sleep(120) + self.poll_for_server_ready() + + def install_container_packages(self) -> None: + self._container_exec( + "bash -c " + shlex.quote("sudo apt -y update && sudo apt install -y iputils-ping iproute2 net-tools") + ) + + def exec_nic_setup_scripts(self) -> None: + if re.search('broadcom|thor', self.nic_type, re.I): + self.inf_dict['nccl_ib_gid_index'] = 3 + cmd = "bash -c " + shlex.quote(f"cp {self.mount_vol}.host {self.mount_vol}; sleep 2; ibv_devinfo; sleep 2;") + out_dict = self._container_exec(cmd) + hca_id_regex = rf'hca_id:\s+{re.escape(self.hca_id_prefix)}' + for node, out in out_dict.items(): + if not re.search(hca_id_regex, out or '', re.I): + fail_test(f'Broadcom libbnxt rdma driver is not properly copied on node {node}') + + def check_ibv_devices(self) -> None: + out_dict = self._container_exec("ibv_devinfo") + for node, out in out_dict.items(): + if re.search('No IB devices found', out or '', re.I): + fail_test(f'IB devices not seen inside the container for node {node}') + + def run_test_rmsnorm(self, max_jobs=192) -> None: + self._container_exec( + "bash -c " + + shlex.quote( + f"MAX_JOBS={max_jobs} python /sgl-workspace/aiter/op_tests/test_rmsnorm2d.py " + f"> /tmp/rsmnorm_test.log 2>&1 &" + ) + ) + time.sleep(180) + out_dict = self._container_exec("bash -c " + shlex.quote("cat /tmp/rsmnorm_test.log")) + for node, out in out_dict.items(): + if re.search('fail', out or '', re.I): + fail_test(f'Some failures observed in test rmsnorm on node {node}') + + def verify_openai_compatible_endpoints(self) -> list[str]: + port = int(self.router_serv_port) + probe_src = OpenAIProbe.probe_script(port, self.bp_dict['model'], host=self.client_host) + b64 = base64.b64encode(probe_src.encode('utf-8')).decode('ascii') + inner = ( + f"mkdir -p {self.log_dir}/benchmark_node && " + f"echo {shlex.quote(b64)} | base64 -d > /tmp/openai_mq_probe.py && " + f"python3 /tmp/openai_mq_probe.py && rm -f /tmp/openai_mq_probe.py" + ) + log.info( + 'OpenAI endpoint probe inside bench container (%s:%r)', + self.client_host, + port, + ) + out_dict = self._bench_exec("bash -c " + shlex.quote(inner), timeout=min(900, 480 + 180)) + raw_out = out_dict.get(self.benchmark_serv_node) or self._first_output(out_dict) + + probe_err: Optional[str] = None + results: dict[str, tuple[int, Any]] = {} + if not raw_out or not str(raw_out).strip(): + probe_err = f"OpenAI-compatible probe produced no output on {self.benchmark_serv_node!r}: {out_dict!r}" + else: + last_line = str(raw_out).strip().splitlines()[-1] + try: + parsed = json.loads(last_line) + except json.JSONDecodeError as e: + probe_err = f"OpenAI-compatible probe invalid JSON: {e!r} raw={raw_out!r}" + else: + if not isinstance(parsed, dict): + probe_err = f"OpenAI-compatible probe expected JSON object, got {type(parsed).__name__!r}" + else: + for step, val in parsed.items(): + if isinstance(val, (list, tuple)) and len(val) == 2: + results[step] = (int(val[0]), val[1]) + else: + probe_err = f"OpenAI-compatible probe bad shape at {step!r}: {val!r}" + break + + if probe_err is not None: + fail_test(probe_err) + return [] + + OpenAIProbe.log_results(results, log) + ok, err = OpenAIProbe.check_results(results, port=port, logger=log) + if not ok: + fail_test(f"{err}") + return OpenAIProbe.summarize_results(results, ok, err) + return OpenAIProbe.summarize_results(results, ok, err) + + def benchserv_test_random(self, d_type='auto') -> None: + i_dict = self.bp_dict['inference_tests']['bench_serv_random'] + self._bench_num_prompts = int(i_dict['num_prompts']) + inner = ( + f"mkdir -p {self.log_dir}/benchmark_node\n" + f"source /tmp/server_env_script.sh\n" + f"export PYTHONPATH=/sgl-workspace/sglang/python:${{PYTHONPATH:-}}\n" + f"python3 -m sglang.bench_serving \\\n" + f" --backend {i_dict['backend']} \\\n" + f" --dataset-name random \\\n" + f" --num-prompts {i_dict['num_prompts']} \\\n" + f" --max-concurrency {self.bp_dict['max_concurrency']} \\\n" + f" --random-input {i_dict['input_length']} \\\n" + f" --random-output {i_dict['output_length']} \\\n" + f" --random-range-ratio {i_dict['random_range_ratio']} \\\n" + f" --host {self.client_host} --port {self.router_serv_port} \\\n" + f" > {self.log_dir}/benchmark_node/benchmark_results.log 2>&1" + ) + self._bench_exec("bash -c " + shlex.quote(inner), timeout=1000) + time.sleep(5) + self.poll_for_inference_completion(iterations=10, waittime_between_iters=60) + + tp = int(self.bp_dict.get('tensor_parallelism', 1)) + int(self.bp_dict.get('pipeline_parallelism', 1)) + num_gpus = self.nnodes * tp + peak_tflops = float(i_dict.get('peak_gpu_tflops', 1300)) + num_params = float(i_dict.get('model_num_params', 70e9)) + for node, m in (self.inference_results_dict or {}).items(): + duration = float(m.get('benchmark_duration') or 0) + in_tok = float(m.get('total_input_tokens') or 0) + out_tok = float(m.get('total_generated_tokens') or 0) + if duration > 0 and num_gpus > 0: + achieved = 6.0 * num_params * (in_tok + out_tok) + peak = peak_tflops * 1e12 * num_gpus * duration + m['mfu'] = f'{achieved / peak:.6f}' + + self.verify_inference_results('bench_serv', i_dict['expected_results'][d_type]) + + def get_inference_results_dict(self, out_dict): + self.inference_results_dict = {} + for node, text in out_dict.items(): + self.inference_results_dict[node] = {} + patterns = [ + (r'Successful requests:\s+([0-9]+)', 'successful_requests'), + (r'Benchmark duration\s+\(s\):\s+([0-9]+)', 'benchmark_duration'), + (r'Total input tokens:\s+([0-9\.]+)', 'total_input_tokens'), + (r'Total generated tokens:\s+([0-9\.]+)', 'total_generated_tokens'), + (r'Request throughput \(req/s\):\s+([0-9\.]+)', 'request_throughput_per_sec'), + (r'Output token throughput \(tok/s\):\s+([0-9\.]+)', 'output_throughput_per_sec'), + (r'Mean TTFT \(ms\):\s+([0-9\.]+)', 'mean_ttft_ms'), + (r'Median TTFT \(ms\):\s+([0-9\.]+)', 'median_ttft_ms'), + (r'P99 TTFT \(ms\):\s+([0-9\.]+)', 'p99_ttft_ms'), + (r'Mean TPOT \(ms\):\s+([0-9\.]+)', 'mean_tpot_ms'), + (r'Median TPOT \(ms\):\s+([0-9]+)', 'median_tpot_ms'), + (r'P99 TPOT \(ms\):\s+([0-9\.]+)', 'p99_tpot_ms'), + ] + for pattern, key in patterns: + match = re.search(pattern, text, re.I) + if match: + self.inference_results_dict[node][key] = match.group(1) + for pattern, key in ( + (r'Mean E2E Latency \(ms\):\s+([0-9\.]+)', 'mean_e2e_latency_ms'), + (r'Median E2E Latency \(ms\):\s+([0-9\.]+)', 'median_e2e_latency_ms'), + (r'P99 E2E Latency \(ms\):\s+([0-9\.]+)', 'p99_e2e_latency_ms'), + ): + val = first_float(pattern, text) + if val: + self.inference_results_dict[node][key] = val + + total_req = first_float(r'Total requests:\s+([0-9]+)', text) + failed_req = first_float(r'Failed requests:\s+([0-9]+)', text) + succ = self.inference_results_dict[node].get('successful_requests') + if total_req: + self.inference_results_dict[node]['total_requests'] = total_req + elif succ is not None and failed_req is not None: + self.inference_results_dict[node]['total_requests'] = str(int(succ) + int(failed_req)) + elif succ is not None and getattr(self, '_bench_num_prompts', None) is not None: + self.inference_results_dict[node]['total_requests'] = str(int(self._bench_num_prompts)) + if succ and self.inference_results_dict[node].get('total_requests'): + s, t = int(succ), int(self.inference_results_dict[node]['total_requests']) + self.inference_results_dict[node]['goodput'] = f'{(s / t):.6f}' if t else None + + return self.inference_results_dict + + def poll_for_inference_completion( + self, iterations=10, waittime_between_iters=60, total_timeout=3600, require_all_nodes=True + ): + time.sleep(60) + start_time = time.time() + completed_pattern = re.compile('Serving Benchmark Result', re.I) + log_path = f"{self.log_dir}/benchmark_node/benchmark_results.log" + + for _itr in range(1, iterations + 1): + out_dict = self._bench_exec(f"tail -1000 {shlex.quote(log_path)}") + done = all(completed_pattern.search(o or '') for o in out_dict.values()) if out_dict else False + if done: + self.get_inference_results_dict(out_dict) + return {"status": "success", "results": self.inference_results_dict} + if total_timeout and (time.time() - start_time) >= total_timeout: + return {"status": "timeout", "reason": "benchmark timed out"} + time.sleep(30 + int(waittime_between_iters)) + return {"status": "stuck_in_progress", "reason": "benchmark did not complete"} + + def verify_inference_results(self, test_name, expected_result_dict): + thresholds = { + metric: normalize_sglang_threshold_spec(metric, spec) for metric, spec in expected_result_dict.items() + } + for node in self.inference_results_dict: + actuals = { + metric: coerce_sglang_actual(value) + for metric, value in self.inference_results_dict[node].items() + if metric in thresholds + } + try: + evaluate_all(actuals, thresholds) + except ThresholdViolation as exc: + for msg in exc.violations: + fail_test(f"FAIL - {msg}") + + self.inference_end_time = self._host_exec('date +"%a %b %e %H:%M"') + time.sleep(2) + verify_dmesg_for_errors(self.orch.all, self.inference_start_time, self.inference_end_time) + + def sglang_distributed_gpu_counts(self, mem_threshold_mb=5000): + tp = int(self.bp_dict["tensor_parallelism"]) + pp = int(self.bp_dict.get("pipeline_parallelism", 1)) + + topo = collect_sglang_gpu_topology( + self._host_exec, + {"server": self.server_node_list}, + mem_threshold_mb=mem_threshold_mb, + ) + server = topo["groups"]["server"] + + result = { + "configured_tp": tp, + "configured_pp": pp, + "configured_nnodes": self.nnodes, + "server_per_node": server["per_node"], + "total_occupied_gpus": topo["total_occupied_gpus"], + } + log.info( + "\n".join( + format_sglang_gpu_topology_lines( + configured_tp=tp, + configured_pp=pp, + configured_nnodes=self.nnodes, + groups={"Server nodes": server}, + ) + ) + ) + return result + + def run_lm_eval_hellaswag_benchmark_test(self, _d_type='auto'): + return self.run_lm_eval_benchmark_test('lm_eval_hellaswag', _d_type=_d_type) + + def run_lm_eval_gsm8k_benchmark_test(self, _d_type='auto'): + return self.run_lm_eval_benchmark_test('lm_eval_gsm8k', _d_type=_d_type) + + def run_lm_eval_benchmark_test(self, bench_key: str, _d_type='auto'): + spec = LM_EVAL_SPECS[bench_key] + task_name = bench_key.removeprefix('lm_eval_') + i_dict = self.bp_dict['inference_tests'][bench_key] + inner_cmd, scoring = LmEvalBenchmark.prepare( + i_dict, + port=int(self.router_serv_port), + host=self.client_host, + model_id=self.bp_dict['model'], + task_name=task_name, + default_tasks=task_name, + default_metric=spec['default_metric'], + default_metric_key=spec['default_metric_key'], + log_dir=self.log_dir, + log_basename=f'{bench_key}.log', + default_num_concurrent=spec['default_num_concurrent'], + ) + inner = f"mkdir -p {self.log_dir}/benchmark_node && source /tmp/server_env_script.sh && {inner_cmd}" + out_dict = self._bench_exec( + "bash -c " + shlex.quote(inner), + timeout=scoring['exec_timeout_sec'], + ) + time.sleep(5) + + check_kwargs = LmEvalBenchmark.check_kwargs_from_scoring(scoring) + summary = None + errors: list[str] = [] + for node, text in out_dict.items(): + ok, node_summary, err = LmEvalBenchmark.check_results(text, **check_kwargs) + if node_summary is not None: + summary = node_summary + if not ok: + errors.append(f"lm-eval {spec['display']} on node {node!r}: {err}") + + if summary is None: + summary = LmEvalBenchmark.fallback_summary( + scoring, + error=errors[-1] if errors else 'no benchmark nodes produced output to score', + ) + errors.append(f"lm-eval {spec['display']}: no benchmark nodes produced output to score") + + for msg in errors: + fail_test(msg) + return summary diff --git a/cvs/lib/inference/sglang/sglang_parsing.py b/cvs/lib/inference/sglang/sglang_parsing.py new file mode 100644 index 000000000..9208e2d43 --- /dev/null +++ b/cvs/lib/inference/sglang/sglang_parsing.py @@ -0,0 +1,95 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Pure parsers and metric vocabulary for SGLang benchmark reports. + +SGLang bench artifacts (log regex parsing in ``sglang_single_lib`` / ``sglang_disagg_lib``) +use bare metric keys (``mean_ttft_ms``, ``output_throughput_per_sec``) — not the +``client.*`` namespace vLLM uses. Report presets set ``metric_prefix=""`` accordingly. +''' + +from __future__ import annotations + +from cvs.lib.report.types import ReportChartSeries + +SGLANG_METRIC_UNITS: dict[str, str] = { + "request_throughput_per_sec": "req/s", + "output_throughput_per_sec": "tok/s", + "output_throughput_per_gpu_per_sec": "tok/s/GPU", + "mean_ttft_ms": "ms", + "median_ttft_ms": "ms", + "p99_ttft_ms": "ms", + "mean_tpot_ms": "ms", + "median_tpot_ms": "ms", + "p99_tpot_ms": "ms", + "p99_itl_ms": "ms", + "mean_e2e_latency_ms": "ms", + "median_e2e_latency_ms": "ms", + "p99_e2e_latency_ms": "ms", + "goodput": "ratio", + "mfu": "ratio", +} + +SGLANG_RESULTS_COLUMNS = ( + ("Model", None), + ("GPU", None), + ("ISL", None), + ("OSL", None), + ("Policy", None), + ("Conc", None), + ("Host", None), + ("Req/s", "request_throughput_per_sec"), + ("Output tok/s", "output_throughput_per_sec"), + ("Mean TTFT (ms)", "mean_ttft_ms"), + ("Mean TPOT (ms)", "mean_tpot_ms"), + ("P99 ITL (ms)", "p99_itl_ms"), + ("Mean E2E latency (ms)", "mean_e2e_latency_ms"), + ("Goodput", "goodput"), + ("MFU (estimated)", "mfu"), +) + +METRIC_TIERS: dict[str, tuple[str, ...]] = { + "throughput": ( + "output_throughput_per_sec", + "request_throughput_per_sec", + "output_throughput_per_gpu_per_sec", + ), + "latency": ( + "mean_ttft_ms", + "mean_tpot_ms", + "p99_ttft_ms", + "p99_tpot_ms", + "p99_itl_ms", + "mean_e2e_latency_ms", + ), + "health": ( + "goodput", + "mfu", + ), +} + +METRIC_TIER_ORDER: tuple[str, ...] = tuple(METRIC_TIERS.keys()) + ("record",) + +_tiered = {m for names in METRIC_TIERS.values() for m in names} +RECORD_METRICS: tuple[str, ...] = tuple(short for short in SGLANG_METRIC_UNITS if short not in _tiered) + +SGLANG_CHART_SERIES: tuple[ReportChartSeries, ...] = ( + ReportChartSeries("output_throughput_per_sec", "Output tok/s", "tok/s"), + ReportChartSeries("request_throughput_per_sec", "Req/s", "req/s"), + ReportChartSeries("mean_ttft_ms", "Mean TTFT", "ms", invert=True), + ReportChartSeries("mean_tpot_ms", "Mean TPOT", "ms", invert=True), + ReportChartSeries("p99_ttft_ms", "P99 TTFT", "ms", invert=True), + ReportChartSeries("p99_tpot_ms", "P99 TPOT", "ms", invert=True), +) + + +def tier_metric_specs(thresholds_cell: dict, tier: str) -> dict[str, dict]: + """Return threshold specs for one tier in a sweep cell (bare metric keys).""" + names = RECORD_METRICS if tier == "record" else METRIC_TIERS.get(tier, ()) + specs: dict[str, dict] = {} + for name in names: + spec = thresholds_cell.get(name) + if spec is not None: + specs[name] = spec + return specs diff --git a/cvs/lib/inference/sglang/sglang_single_lib.py b/cvs/lib/inference/sglang/sglang_single_lib.py new file mode 100644 index 000000000..aa4363b1a --- /dev/null +++ b/cvs/lib/inference/sglang/sglang_single_lib.py @@ -0,0 +1,468 @@ +''' +Copyright 2026 Advanced Micro Devices, Inc. +All rights reserved. + +Single-node SGLang inference controller (no PD disaggregation). + +One container on ``benchmark_serv_node`` (via ``ContainerOrchestrator``) runs a unified +``sglang.launch_server`` on ``proxy_router_serv_port``. Benchmark/smoke/lm-eval +traffic hits that port via ``client_host`` (default ``127.0.0.1`` inside the +container). +''' + +from __future__ import annotations + +import base64 +import json +import os +import re +import shlex +import time +from typing import Any, Optional + +from cvs.lib import globals +from cvs.core.orchestrators.baremetal import BaremetalOrchestrator +from cvs.lib.inference.sglang.sglang_common import ( + LM_EVAL_SPECS, + add_cli_flags_block, + add_export_env_block, + as_node_list, + coerce_sglang_actual, + first_float, + normalize_sglang_threshold_spec, + resolve_client_host, + _SERVER_READY_RE, +) +from cvs.lib.utils.model_query_lib import LmEvalBenchmark, OpenAIProbe +from cvs.lib.utils_lib import fail_test +from cvs.lib.utils.verdict import ThresholdViolation, evaluate_all +from cvs.lib.verify_lib import verify_dmesg_for_errors + +log = globals.log + + +class SglangSingle: + """Unified single-node SGLang serve + benchmark via ``ContainerOrchestrator``.""" + + def __init__( + self, + model_name, + inference_config_dict, + benchmark_params_dict, + hf_token, + orch=None, + gpu_type='mi300', + user_name=None, + priv_key_file=None, + ): + if orch is None: + raise ValueError("SglangSingle requires orch= (ContainerOrchestrator)") + + self.orch = orch + self.user_name = user_name + self.priv_key_file = priv_key_file + self.model_name = model_name + self.hf_token = hf_token + self.gpu_type = gpu_type + + self.inf_dict = inference_config_dict + self.bp_dict = benchmark_params_dict + + self.inference_results_dict = {} + log.info("%s", self.gpu_type) + + self.home_dir = os.path.expanduser("~") + self._apply_inf_defaults() + self._apply_bp_defaults() + + self.container_name = self.inf_dict['container_name'] + self.log_dir = self.inf_dict['log_dir'] + self.inference_poll_iterations = self.bp_dict['inference_poll_iterations'] + self.benchmark_serv_node = self._resolve_benchmark_serv_node() + + self.inference_start_time = self._host_exec('date +"%a %b %e %H:%M"') + self.inference_end_time = None + + log.info('single-node inference_dict = %s', self.inf_dict) + log.info('single-node benchmark_params_dict = %s', self.bp_dict) + log.info( + 'single-node client_host=%s router_serv_port=%s benchmark_serv_node=%s', + self.client_host, + self.router_serv_port, + self.benchmark_serv_node, + ) + + def _resolve_benchmark_serv_node(self) -> str: + raw = self.inf_dict.get('benchmark_serv_node') + if not raw: + raise ValueError("SglangSingle requires benchmark_serv_node in the inference config") + hosts = as_node_list(raw) + if len(hosts) != 1: + raise ValueError(f"SglangSingle requires exactly one benchmark_serv_node, got {hosts!r}") + return hosts[0] + + @property + def _head_host(self) -> str: + return self.benchmark_serv_node + + @property + def server_log_path(self) -> str: + return f"{self.log_dir}/server_node/server.log" + + @property + def router_serv_port(self) -> str: + """Unified server listen/client port (``proxy_router_serv_port``).""" + return str(self.inf_dict['proxy_router_serv_port']) + + @property + def client_host(self) -> str: + """HTTP client target when smoke/bench/lm-eval run inside the same container.""" + return resolve_client_host(self.inf_dict, unified_server=True) + + @staticmethod + def _first_output(out_dict: dict) -> str: + if not out_dict: + return "" + return next(iter(out_dict.values())) or "" + + def _container_exec(self, cmd: str, *, timeout: int | None = None) -> dict: + """Run ``cmd`` inside the container.""" + return self.orch.exec(cmd, timeout=timeout) + + def _container_exec_text(self, cmd: str, *, timeout: int | None = None) -> str: + return self._first_output(self._container_exec(cmd, timeout=timeout)) + + def _host_exec(self, cmd: str, *, timeout: int | None = None) -> dict: + """Run ``cmd`` on ``benchmark_serv_node`` (baremetal), e.g. amd-smi / dmesg.""" + host = self.benchmark_serv_node + if host == self.orch.head_node and len(self.orch.hosts) == 1: + return self.orch.head.exec(cmd, timeout=timeout) + return BaremetalOrchestrator.exec(self.orch, cmd, hosts=[host], timeout=timeout) + + def _host_exec_text(self, cmd: str, *, timeout: int | None = None) -> str: + return self._first_output(self._host_exec(cmd, timeout=timeout)) + + def _apply_inf_defaults(self) -> None: + self.inf_dict.setdefault('container_image', 'lmsysorg/sglang:dev') + self.inf_dict.setdefault('container_name', 'sglang_container') + self.inf_dict.setdefault('nccl_debug', 'ERROR') + self.inf_dict.setdefault('data_cache_dir', f'{self.home_dir}/cache') + self.inf_dict.setdefault('log_dir', f'{self.home_dir}/LOG_DIR') + self.inf_dict.setdefault('log_level', 'info') + self.inf_dict.setdefault('proxy_router_serv_port', '8000') + + def _apply_bp_defaults(self) -> None: + self.bp_dict.setdefault('backend', 'sglang') + self.bp_dict.setdefault('max_concurrency', '64') + self.bp_dict.setdefault('model', 'openai/gpt-oss-120b') + self.bp_dict.setdefault('tensor_parallelism', '8') + self.bp_dict.setdefault('memory_fraction', '0.85') + self.bp_dict.setdefault('inference_poll_iterations', '16') + + def setup_server_container_env(self) -> None: + """Write and source ``/tmp/server_env_script.sh`` inside the container.""" + env_body = ( + "export LD_LIBRARY_PATH=/usr/local/lib:/sgl-workspace/Mooncake/build/mooncake-common/etcd:/opt/rocm/lib:$LD_LIBRARY_PATH\n" + f"export NCCL_DEBUG={self.inf_dict['nccl_debug']}\n" + f"export HSA_FORCE_FINE_GRAIN_PCIE=1\n" + f"export MODEL={self.bp_dict['model']}\n" + f"export TP={self.bp_dict['tensor_parallelism']}\n" + f"export HF_TOKEN={self.hf_token}\n" + f"{add_export_env_block(self.bp_dict, indent='')}\n" + ) + write_cmd = "bash -c " + shlex.quote( + f"cat > /tmp/server_env_script.sh <<'EOF'\n{env_body}EOF\n" + "chmod 755 /tmp/server_env_script.sh && /tmp/server_env_script.sh" + ) + time.sleep(3) + self._container_exec(write_cmd) + time.sleep(5) + + def launch_server(self, dtype='auto', kv_cache_dtype='auto') -> None: + """Launch one unified SGLang server (no PD disaggregation).""" + log.info('Launch unified SGLang server on 0.0.0.0:%s', self.router_serv_port) + flags_block = add_cli_flags_block(self.bp_dict, indent=' ') + launch_body = ( + f"python3 -m sglang.launch_server --model {self.bp_dict['model']} \\\n" + f" --host 0.0.0.0 \\\n" + f" --port {self.router_serv_port} \\\n" + f" --dtype {dtype} \\\n" + f" --kv-cache-dtype {kv_cache_dtype} \\\n" + f" --trust-remote-code \\\n" + f" --tp-size {self.bp_dict['tensor_parallelism']} \\\n" + f" --disable-radix-cache --disable-cuda-graph \\\n" + f" --mem-fraction-static {self.bp_dict['memory_fraction']} \\\n" + f"{flags_block}\n" + f" --log-level {self.inf_dict['log_level']}\n" + ) + start_cmd = "bash -c " + shlex.quote( + f"cat > /tmp/server_launch_script.sh <<'EOF'\n{launch_body}EOF\n" + f"chmod 755 /tmp/server_launch_script.sh\n" + f"mkdir -p {self.log_dir}/server_node\n" + f"source /tmp/server_env_script.sh\n" + f"nohup /tmp/server_launch_script.sh > {self.server_log_path} 2>&1 &" + ) + self._container_exec(start_cmd) + time.sleep(5) + + def poll_for_server_ready(self, no_of_iterations=16) -> None: + for iteration in range(1, no_of_iterations): + log.info('Starting server readiness poll iteration %d', iteration) + grep_cmd = f"grep -B 20 -A 20 -E {_SERVER_READY_RE.pattern!r} {shlex.quote(self.server_log_path)} || true" + text = self._container_exec_text(grep_cmd) + if _SERVER_READY_RE.search(text): + log.info('Wait 60 secs before serving traffic') + time.sleep(60) + return + log.info('Wait 120 secs and continue polling') + time.sleep(120) + fail_test(f'Single-node server on {self._head_host} did not reach ready state in {no_of_iterations} iterations') + + def poll_and_check_server_ready(self) -> None: + log.info('Waiting 120 secs after launching server') + time.sleep(120) + self.poll_for_server_ready() + + def setup_benchmark_serv_container_env(self) -> None: + self.setup_server_container_env() + + def install_container_packages(self) -> None: + self._container_exec( + "bash -c " + shlex.quote("sudo apt -y update && sudo apt install -y iputils-ping iproute2 net-tools") + ) + + def run_test_rmsnorm(self, max_jobs=192) -> None: + self._container_exec( + "bash -c " + + shlex.quote( + f"MAX_JOBS={max_jobs} python /sgl-workspace/aiter/op_tests/test_rmsnorm2d.py " + f"> /tmp/rsmnorm_test.log 2>&1 &" + ) + ) + time.sleep(180) + out_dict = self._container_exec("bash -c " + shlex.quote("cat /tmp/rsmnorm_test.log")) + for node, out in out_dict.items(): + if re.search('fail', out or '', re.I): + fail_test(f'Some failures observed in test rmsnorm on node {node}') + + def verify_openai_compatible_endpoints(self) -> list[str]: + port = int(self.router_serv_port) + probe_src = OpenAIProbe.probe_script(port, self.bp_dict['model'], host=self.client_host) + b64 = base64.b64encode(probe_src.encode('utf-8')).decode('ascii') + inner = ( + f"mkdir -p {self.log_dir}/benchmark_node && " + f"echo {shlex.quote(b64)} | base64 -d > /tmp/openai_mq_probe.py && " + f"python3 /tmp/openai_mq_probe.py && rm -f /tmp/openai_mq_probe.py" + ) + log.info( + 'OpenAI endpoint probe inside container (%s:%r)', + self.client_host, + port, + ) + out_dict = self._container_exec("bash -c " + shlex.quote(inner), timeout=min(900, 480 + 180)) + raw_out = out_dict.get(self._head_host) or self._first_output(out_dict) + + probe_err: Optional[str] = None + results: dict[str, tuple[int, Any]] = {} + if not raw_out or not str(raw_out).strip(): + probe_err = f"OpenAI-compatible probe produced no output on {self._head_host!r}: {out_dict!r}" + else: + last_line = str(raw_out).strip().splitlines()[-1] + try: + parsed = json.loads(last_line) + except json.JSONDecodeError as e: + probe_err = f"OpenAI-compatible probe invalid JSON: {e!r} raw={raw_out!r}" + else: + if not isinstance(parsed, dict): + probe_err = f"OpenAI-compatible probe expected JSON object, got {type(parsed).__name__!r}" + else: + for step, val in parsed.items(): + if isinstance(val, (list, tuple)) and len(val) == 2: + results[step] = (int(val[0]), val[1]) + else: + probe_err = f"OpenAI-compatible probe bad shape at {step!r}: {val!r}" + break + + if probe_err is not None: + fail_test(probe_err) + return [] + + OpenAIProbe.log_results(results, log) + ok, err = OpenAIProbe.check_results(results, port=port, logger=log) + if not ok: + fail_test(f"{err}") + return OpenAIProbe.summarize_results(results, ok, err) + return OpenAIProbe.summarize_results(results, ok, err) + + def benchserv_test_random(self, d_type='auto') -> None: + i_dict = self.bp_dict['inference_tests']['bench_serv_random'] + self._bench_num_prompts = int(i_dict['num_prompts']) + inner = ( + f"mkdir -p {self.log_dir}/benchmark_node\n" + f"source /tmp/server_env_script.sh\n" + f"export PYTHONPATH=/sgl-workspace/sglang/python:${{PYTHONPATH:-}}\n" + f"python3 -m sglang.bench_serving \\\n" + f" --backend {i_dict['backend']} \\\n" + f" --dataset-name random \\\n" + f" --num-prompts {i_dict['num_prompts']} \\\n" + f" --max-concurrency {self.bp_dict['max_concurrency']} \\\n" + f" --random-input {i_dict['input_length']} \\\n" + f" --random-output {i_dict['output_length']} \\\n" + f" --random-range-ratio {i_dict['random_range_ratio']} \\\n" + f" --host {self.client_host} --port {self.router_serv_port} \\\n" + f" > {self.log_dir}/benchmark_node/benchmark_results.log 2>&1" + ) + self._container_exec("bash -c " + shlex.quote(inner), timeout=1000) + time.sleep(5) + self.poll_for_inference_completion(iterations=10, waittime_between_iters=60) + + tp = int(self.bp_dict.get('tensor_parallelism', 1)) + num_gpus = tp + peak_tflops = float(i_dict.get('peak_gpu_tflops', 1300)) + num_params = float(i_dict.get('model_num_params', 70e9)) + for node, m in (self.inference_results_dict or {}).items(): + duration = float(m.get('benchmark_duration') or 0) + in_tok = float(m.get('total_input_tokens') or 0) + out_tok = float(m.get('total_generated_tokens') or 0) + if duration > 0 and num_gpus > 0: + achieved = 6.0 * num_params * (in_tok + out_tok) + peak = peak_tflops * 1e12 * num_gpus * duration + m['mfu'] = f'{achieved / peak:.6f}' + + self.verify_inference_results('bench_serv', i_dict['expected_results'][d_type]) + + def get_inference_results_dict(self, out_dict): + self.inference_results_dict = {} + for node, text in out_dict.items(): + self.inference_results_dict[node] = {} + patterns = [ + (r'Successful requests:\s+([0-9]+)', 'successful_requests'), + (r'Benchmark duration\s+\(s\):\s+([0-9]+)', 'benchmark_duration'), + (r'Total input tokens:\s+([0-9\.]+)', 'total_input_tokens'), + (r'Total generated tokens:\s+([0-9\.]+)', 'total_generated_tokens'), + (r'Request throughput \(req/s\):\s+([0-9\.]+)', 'request_throughput_per_sec'), + (r'Output token throughput \(tok/s\):\s+([0-9\.]+)', 'output_throughput_per_sec'), + (r'Mean TTFT \(ms\):\s+([0-9\.]+)', 'mean_ttft_ms'), + (r'Median TTFT \(ms\):\s+([0-9\.]+)', 'median_ttft_ms'), + (r'P99 TTFT \(ms\):\s+([0-9\.]+)', 'p99_ttft_ms'), + (r'Mean TPOT \(ms\):\s+([0-9\.]+)', 'mean_tpot_ms'), + (r'Median TPOT \(ms\):\s+([0-9]+)', 'median_tpot_ms'), + (r'P99 TPOT \(ms\):\s+([0-9\.]+)', 'p99_tpot_ms'), + ] + for pattern, key in patterns: + match = re.search(pattern, text, re.I) + if match: + self.inference_results_dict[node][key] = match.group(1) + for pattern, key in ( + (r'Mean E2E Latency \(ms\):\s+([0-9\.]+)', 'mean_e2e_latency_ms'), + (r'Median E2E Latency \(ms\):\s+([0-9\.]+)', 'median_e2e_latency_ms'), + (r'P99 E2E Latency \(ms\):\s+([0-9\.]+)', 'p99_e2e_latency_ms'), + ): + val = first_float(pattern, text) + if val: + self.inference_results_dict[node][key] = val + + total_req = first_float(r'Total requests:\s+([0-9]+)', text) + failed_req = first_float(r'Failed requests:\s+([0-9]+)', text) + succ = self.inference_results_dict[node].get('successful_requests') + if total_req: + self.inference_results_dict[node]['total_requests'] = total_req + elif succ is not None and failed_req is not None: + self.inference_results_dict[node]['total_requests'] = str(int(succ) + int(failed_req)) + elif succ is not None and getattr(self, '_bench_num_prompts', None) is not None: + self.inference_results_dict[node]['total_requests'] = str(int(self._bench_num_prompts)) + if succ and self.inference_results_dict[node].get('total_requests'): + s, t = int(succ), int(self.inference_results_dict[node]['total_requests']) + self.inference_results_dict[node]['goodput'] = f'{(s / t):.6f}' if t else None + + return self.inference_results_dict + + def poll_for_inference_completion( + self, iterations=10, waittime_between_iters=60, total_timeout=3600, require_all_nodes=True + ): + time.sleep(60) + start_time = time.time() + completed_pattern = re.compile('Serving Benchmark Result', re.I) + log_path = f"{self.log_dir}/benchmark_node/benchmark_results.log" + + for _itr in range(1, iterations + 1): + out_dict = self._container_exec(f"tail -1000 {shlex.quote(log_path)}") + done = all(completed_pattern.search(o or '') for o in out_dict.values()) if out_dict else False + if done: + self.get_inference_results_dict(out_dict) + return {"status": "success", "results": self.inference_results_dict} + if total_timeout and (time.time() - start_time) >= total_timeout: + return {"status": "timeout", "reason": "benchmark timed out"} + time.sleep(30 + int(waittime_between_iters)) + return {"status": "stuck_in_progress", "reason": "benchmark did not complete"} + + def verify_inference_results(self, test_name, expected_result_dict): + thresholds = { + metric: normalize_sglang_threshold_spec(metric, spec) for metric, spec in expected_result_dict.items() + } + for node in self.inference_results_dict: + actuals = { + metric: coerce_sglang_actual(value) + for metric, value in self.inference_results_dict[node].items() + if metric in thresholds + } + try: + evaluate_all(actuals, thresholds) + except ThresholdViolation as exc: + for msg in exc.violations: + fail_test(f"FAIL - {msg}") + + self.inference_end_time = self._host_exec('date +"%a %b %e %H:%M"') + time.sleep(2) + verify_dmesg_for_errors(self.orch.all, self.inference_start_time, self.inference_end_time) + + def run_lm_eval_hellaswag_benchmark_test(self, _d_type='auto'): + return self.run_lm_eval_benchmark_test('lm_eval_hellaswag', _d_type=_d_type) + + def run_lm_eval_gsm8k_benchmark_test(self, _d_type='auto'): + return self.run_lm_eval_benchmark_test('lm_eval_gsm8k', _d_type=_d_type) + + def run_lm_eval_benchmark_test(self, bench_key: str, _d_type='auto'): + spec = LM_EVAL_SPECS[bench_key] + task_name = bench_key.removeprefix('lm_eval_') + i_dict = self.bp_dict['inference_tests'][bench_key] + inner_cmd, scoring = LmEvalBenchmark.prepare( + i_dict, + port=int(self.router_serv_port), + host=self.client_host, + model_id=self.bp_dict['model'], + task_name=task_name, + default_tasks=task_name, + default_metric=spec['default_metric'], + default_metric_key=spec['default_metric_key'], + log_dir=self.log_dir, + log_basename=f'{bench_key}.log', + default_num_concurrent=spec['default_num_concurrent'], + ) + inner = f"mkdir -p {self.log_dir}/benchmark_node && source /tmp/server_env_script.sh && {inner_cmd}" + out_dict = self._container_exec( + "bash -c " + shlex.quote(inner), + timeout=scoring['exec_timeout_sec'], + ) + time.sleep(5) + + check_kwargs = LmEvalBenchmark.check_kwargs_from_scoring(scoring) + summary = None + errors: list[str] = [] + for node, text in out_dict.items(): + ok, node_summary, err = LmEvalBenchmark.check_results(text, **check_kwargs) + if node_summary is not None: + summary = node_summary + if not ok: + errors.append(f"lm-eval {spec['display']} on node {node!r}: {err}") + + if summary is None: + summary = LmEvalBenchmark.fallback_summary( + scoring, + error=errors[-1] if errors else 'no benchmark nodes produced output to score', + ) + errors.append(f"lm-eval {spec['display']}: no benchmark nodes produced output to score") + + for msg in errors: + fail_test(msg) + return summary diff --git a/cvs/lib/inference/unittests/__init__.py b/cvs/lib/inference/unittests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cvs/lib/inference/unittests/fake_orch.py b/cvs/lib/inference/unittests/fake_orch.py new file mode 100644 index 000000000..22e60b401 --- /dev/null +++ b/cvs/lib/inference/unittests/fake_orch.py @@ -0,0 +1,26 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Minimal stand-in for :class:`ContainerOrchestrator` in inference Job unit tests. + +Reusable by any suite's ``test_*_orch_parse.py`` — import ``FakeOrch`` instead of +copying the class into each test module. +''' + + +class FakeOrch: + def __init__(self, exec_return=None, hosts=None, exec_on_head_return=None): + self.hosts = list(hosts or ["node0"]) + self.exec_return = exec_return if exec_return is not None else {} + self.exec_on_head_return = exec_on_head_return if exec_on_head_return is not None else self.exec_return + self.commands = [] + self.exec_on_head_commands = [] + + def exec(self, cmd, hosts=None, **kwargs): + self.commands.append((cmd, hosts)) + return self.exec_return + + def exec_on_head(self, cmd, **kwargs): + self.exec_on_head_commands.append(cmd) + return self.exec_on_head_return diff --git a/cvs/lib/inference/unittests/fixtures/vllm_results_sample.json b/cvs/lib/inference/unittests/fixtures/vllm_results_sample.json new file mode 100644 index 000000000..842304822 --- /dev/null +++ b/cvs/lib/inference/unittests/fixtures/vllm_results_sample.json @@ -0,0 +1 @@ +{"date": "20260616-195845", "endpoint_type": "vllm", "backend": "vllm", "label": null, "model_id": "amd/Llama-3.1-70B-Instruct-FP8-KV", "tokenizer_id": "amd/Llama-3.1-70B-Instruct-FP8-KV", "num_prompts": 12800, "request_rate": "inf", "burstiness": 1.0, "max_concurrency": 256, "duration": 2689.1403915379196, "completed": 12800, "failed": 0, "total_input_tokens": 1638400, "total_output_tokens": 26214400, "request_throughput": 4.7598853671896535, "request_goodput": null, "output_throughput": 9748.24523200441, "total_token_throughput": 10357.510559004686, "max_output_tokens_per_s": 10765.0, "max_concurrent_requests": 512, "rtfx": 0.0, "mean_ttft_ms": 668.48989902217, "median_ttft_ms": 639.2352399416268, "std_ttft_ms": 450.33186696146356, "p99_ttft_ms": 2471.584795164014, "mean_tpot_ms": 25.931252678514355, "median_tpot_ms": 25.9539894319472, "std_tpot_ms": 0.21713041382746362, "p99_tpot_ms": 26.324707068246756, "mean_itl_ms": 25.9318070919762, "median_itl_ms": 25.50674183294177, "std_itl_ms": 8.843604314389477, "p99_itl_ms": 32.19962654635305, "mean_e2el_ms": 53749.76413194105, "median_e2el_ms": 53752.437153831124, "std_e2el_ms": 298.81966449031785, "p99_e2el_ms": 55124.09303063527} \ No newline at end of file diff --git a/cvs/lib/inference/unittests/fixtures/vllm_results_widened.json b/cvs/lib/inference/unittests/fixtures/vllm_results_widened.json new file mode 100644 index 000000000..3e32870f9 --- /dev/null +++ b/cvs/lib/inference/unittests/fixtures/vllm_results_widened.json @@ -0,0 +1,52 @@ +{ + "date": "20260617-163232", + "endpoint_type": "vllm", + "backend": "vllm", + "label": null, + "model_id": "amd/Llama-3.1-70B-Instruct-FP8-KV", + "tokenizer_id": "amd/Llama-3.1-70B-Instruct-FP8-KV", + "num_prompts": 3200, + "request_rate": "inf", + "burstiness": 1.0, + "max_concurrency": 64, + "duration": 564.1147056743503, + "completed": 1791, + "failed": 1409, + "total_input_tokens": 229974, + "total_output_tokens": 2312408, + "request_throughput": 3.174886210170704, + "request_goodput": 3.174886210170704, + "output_throughput": 4099.180497760143, + "total_token_throughput": 4506.852904961594, + "max_output_tokens_per_s": 4590.0, + "max_concurrent_requests": 76, + "rtfx": 0.0, + "mean_ttft_ms": 291.88843878398956, + "median_ttft_ms": 73.2587007805705, + "std_ttft_ms": 1137.8571870825917, + "p50_ttft_ms": 73.2587007805705, + "p90_ttft_ms": 85.64653992652893, + "p95_ttft_ms": 91.81479038670659, + "p99_ttft_ms": 6259.152442589402, + "mean_tpot_ms": 15.032167084785439, + "median_tpot_ms": 15.03020008988302, + "std_tpot_ms": 0.3247818510561047, + "p50_tpot_ms": 15.03020008988302, + "p90_tpot_ms": 15.209271169796184, + "p95_tpot_ms": 15.24944565870449, + "p99_tpot_ms": 15.943741001209947, + "mean_itl_ms": 15.019441688915942, + "median_itl_ms": 14.628257602453232, + "std_itl_ms": 6.1097319902977505, + "p50_itl_ms": 14.628257602453232, + "p90_itl_ms": 15.559395030140879, + "p95_itl_ms": 17.392832040786708, + "p99_itl_ms": 27.511265948414778, + "mean_e2el_ms": 19668.871285726116, + "median_e2el_ms": 19835.17629932612, + "std_e2el_ms": 7829.601121737766, + "p50_e2el_ms": 19835.17629932612, + "p90_e2el_ms": 30145.559770055115, + "p95_e2el_ms": 31765.095902141184, + "p99_e2el_ms": 34034.53387795014 +} diff --git a/cvs/lib/inference/unittests/test_accuracy_config.py b/cvs/lib/inference/unittests/test_accuracy_config.py new file mode 100644 index 000000000..26b0cbffd --- /dev/null +++ b/cvs/lib/inference/unittests/test_accuracy_config.py @@ -0,0 +1,511 @@ +''' +Copyright 2026 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.inference.utils.accuracy_config (AccuracyTask, +AccuracyConfig). These pin the construction-time validation contract of the two +pydantic models: field defaults/typing/coercion, extra="forbid" (inherited from +_Forbid), and the model_validator(mode="after") that rejects duplicate task ids. +No hardware. + +Authored black-box from the behavioral spec; the implementation was not read. +Classification: both models are validation *subsystems* -- the only operation is +construct-time schema validation (taxonomy #4 Schema Boundary Strictness / #5 +Cross-Field Relational Invariants). They expose no state-transition methods, so +no *Lifecycle transition table applies (see StructuredOutput justification). +''' + +import unittest + +from pydantic import ValidationError + +from cvs.lib.inference.utils.accuracy_config import AccuracyConfig, AccuracyTask + + +def _task(**over): + """A valid AccuracyTask with fields overridable per case.""" + base = {"id": "a", "task": "gsm8k"} + base.update(over) + return AccuracyTask(**base) + + +def _dupes_message(exc): + """Isolate the duplicate-id validator's own message from pydantic's wrapper. + + str(ValidationError) appends '[type=..., input_value=..., input_type=...]', + and input_value repeats each task (and thus each id). To assert on the + validator's rendered sorted-dupes list (count/order/quoting) without the + wrapper's echoes, slice from the documented prefix up to pydantic's + '[type=' metadata marker. + """ + msg = str(exc) + prefix = "duplicate task id(s):" + i = msg.find(prefix) + if i == -1: + return None + tail = msg[i:] + j = tail.find("[type=") + if j != -1: + tail = tail[:j] + return tail + + +class TestAccuracyTaskDefaults(unittest.TestCase): + """AC12, AC23, AC24: defaults, empty-string id, include_path (no I/O).""" + + def test_defaults_on_minimal_task(self): + t = AccuracyTask(id="a", task="gsm8k") + self.assertEqual(t.id, "a") + self.assertEqual(t.task, "gsm8k") + self.assertEqual(t.num_fewshot, 0) + self.assertEqual(t.metadata, {}) + self.assertEqual(t.include_path, "") + self.assertEqual(t.num_concurrent, 8) + self.assertIs(t.apply_chat_template, False) + self.assertEqual(t.gen_kwargs, {}) + + def test_empty_string_id_is_valid(self): + # AC23: "" is a valid id, not treated as missing. + t = AccuracyTask(id="", task="gsm8k") + self.assertEqual(t.id, "") + + def test_include_path_no_filesystem_check(self): + # AC24: nonexistent path accepted as plain string, no I/O. + t = AccuracyTask(id="a", task="gsm8k", include_path="/some/nonexistent/path") + self.assertEqual(t.include_path, "/some/nonexistent/path") + + def test_default_dicts_are_isolated_per_instance(self): + # Mutable-default isolation for metadata/gen_kwargs on AccuracyTask. + t1 = AccuracyTask(id="a", task="gsm8k") + t2 = AccuracyTask(id="b", task="gsm8k") + self.assertIsNot(t1.metadata, t2.metadata) + self.assertIsNot(t1.gen_kwargs, t2.gen_kwargs) + + +class TestAccuracyTaskRequiredFields(unittest.TestCase): + """AC13: id and task are required.""" + + def test_missing_required_field_raises(self): + for missing in ("id", "task"): + with self.subTest(missing=missing): + kwargs = {"id": "a", "task": "gsm8k"} + del kwargs[missing] + with self.assertRaises(ValidationError): + AccuracyTask(**kwargs) + + +class TestAccuracyTaskExplicitNone(unittest.TestCase): + """Explicit None is a distinct equivalence class from omission: no field is + Optional, so None is rejected for every field (required str fields AND the + non-None-defaulted int/dict/bool/str fields). Mirror of AC28 for the config's + tasks field. Guards against a mutated schema (e.g. id: Optional[str], or + metadata: Optional[Dict] = {}) silently accepting None while still passing + the omission-only required-field test.""" + + def test_explicit_none_per_field_raises(self): + # (field, None value passed via a valid base task) + for field in ( + "id", + "task", + "num_fewshot", + "metadata", + "include_path", + "num_concurrent", + "apply_chat_template", + "gen_kwargs", + ): + with self.subTest(field=field): + with self.assertRaises(ValidationError): + _task(**{field: None}) + + +class TestAccuracyTaskIntCoercion(unittest.TestCase): + """AC14, AC15, AC16: int fields coerce numeric strings; no range constraint.""" + + def test_int_coercion_success(self): + # (field, input, expected int) + cases = [ + ("num_fewshot", "5", 5), + ("num_fewshot", 5, 5), + ("num_fewshot", 5.0, 5), # whole-number float coerces (vs 1.9 which rejects) + ("num_fewshot", -1, -1), # AC16: negative allowed, no ge + ("num_fewshot", 0, 0), + ("num_fewshot", True, 1), # pydantic lax int: bool coerces to 1/0 + ("num_concurrent", "3", 3), + ("num_concurrent", 3, 3), + ("num_concurrent", 4.0, 4), # whole-number float coerces (vs 2.5 which rejects) + ("num_concurrent", 0, 0), # AC16: zero allowed, no gt + ("num_concurrent", -1, -1), + ("num_concurrent", False, 0), # pydantic lax int: bool coerces to 1/0 + ] + for field, value, expected in cases: + with self.subTest(field=field, value=value): + t = _task(**{field: value}) + got = getattr(t, field) + self.assertEqual(got, expected) + self.assertIsInstance(got, int) + + def test_int_coercion_failure_raises(self): + cases = [ + ("num_fewshot", "not-an-int"), + ("num_fewshot", 1.9), # float with fractional part + ("num_concurrent", "bad"), + ("num_concurrent", 2.5), + ] + for field, value in cases: + with self.subTest(field=field, value=value): + with self.assertRaises(ValidationError): + _task(**{field: value}) + + +class TestAccuracyTaskDictCoercion(unittest.TestCase): + """AC17, AC18, AC19: dict fields accept mappings only.""" + + def test_dict_success(self): + cases = [ + ("metadata", {"k": "v"}), + ("metadata", {}), + ("gen_kwargs", {"a": 1}), + ("gen_kwargs", {}), + ] + for field, value in cases: + with self.subTest(field=field, value=value): + t = _task(**{field: value}) + self.assertEqual(getattr(t, field), value) + + def test_non_mapping_raises(self): + cases = [ + ("metadata", "not-a-dict"), + ("metadata", [1, 2]), + ("metadata", 123), + ("gen_kwargs", 123), + ("gen_kwargs", "not-a-dict"), + ("gen_kwargs", [1, 2]), + ] + for field, value in cases: + with self.subTest(field=field, value=value): + with self.assertRaises(ValidationError): + _task(**{field: value}) + + +class TestAccuracyTaskBoolCoercion(unittest.TestCase): + """AC20, AC21: bool field; 'maybe' is the guaranteed-fail non-bool word.""" + + def test_bool_true_accepted(self): + t = _task(apply_chat_template=True) + self.assertIs(t.apply_chat_template, True) + + def test_bool_false_accepted(self): + t = _task(apply_chat_template=False) + self.assertIs(t.apply_chat_template, False) + + def test_non_bool_word_raises(self): + with self.assertRaises(ValidationError): + _task(apply_chat_template="maybe") + + +class TestAccuracyTaskStringTyping(unittest.TestCase): + """AC22: id/task accept only str; non-str scalars are not auto-coerced.""" + + def test_non_str_id_or_task_raises(self): + cases = [ + {"id": 123, "task": "gsm8k"}, + {"id": "a", "task": 123}, + {"id": 1.5, "task": "gsm8k"}, + {"id": True, "task": "gsm8k"}, # bool is a non-str scalar; not coerced to str + {"id": "a", "task": False}, # bool is a non-str scalar; not coerced to str + ] + for kwargs in cases: + with self.subTest(kwargs=kwargs): + with self.assertRaises(ValidationError): + AccuracyTask(**kwargs) + + +class TestAccuracyTaskExtraForbid(unittest.TestCase): + """AC10: unknown fields rejected (extra='forbid' from _Forbid).""" + + def test_unknown_field_raises(self): + with self.assertRaises(ValidationError): + AccuracyTask(id="a", task="gsm8k", extra_field=1) + + +class TestAccuracyConfigConstruction(unittest.TestCase): + """AC1, AC2, AC3, AC25, AC26, AC29: happy-path construction + element typing.""" + + def test_empty_config_has_empty_tasks(self): + # AC1 + edge case: zero tasks constructs, tasks == []. + cfg = AccuracyConfig() + self.assertEqual(cfg.tasks, []) + + def test_single_task_constructs(self): + # AC2 + edge case: exactly one task is trivially unique. + cfg = AccuracyConfig(tasks=[AccuracyTask(id="a", task="gsm8k")]) + self.assertEqual(len(cfg.tasks), 1) + self.assertEqual(cfg.tasks[0].id, "a") + + def test_three_distinct_ids_construct(self): + # AC3. + cfg = AccuracyConfig( + tasks=[ + AccuracyTask(id="a", task="gsm8k"), + AccuracyTask(id="b", task="gsm8k"), + AccuracyTask(id="c", task="gsm8k"), + ] + ) + self.assertEqual([t.id for t in cfg.tasks], ["a", "b", "c"]) + + def test_list_of_dicts_becomes_tasks(self): + # AC25. + cfg = AccuracyConfig(tasks=[{"id": "a", "task": "gsm8k"}]) + self.assertIsInstance(cfg.tasks[0], AccuracyTask) + self.assertEqual(cfg.tasks[0].id, "a") + + def test_mixed_dicts_and_instances(self): + # AC26: every element ends up an AccuracyTask. + cfg = AccuracyConfig(tasks=[AccuracyTask(id="a", task="gsm8k"), {"id": "b", "task": "gsm8k"}]) + self.assertTrue(all(isinstance(t, AccuracyTask) for t in cfg.tasks)) + self.assertEqual([t.id for t in cfg.tasks], ["a", "b"]) + + def test_order_and_length_preserved(self): + # AC29. + cfg = AccuracyConfig( + tasks=[ + AccuracyTask(id="a", task="m"), + AccuracyTask(id="b", task="m"), + AccuracyTask(id="c", task="m"), + ] + ) + self.assertEqual([t.id for t in cfg.tasks], ["a", "b", "c"]) + + +class TestAccuracyConfigTasksField(unittest.TestCase): + """AC11, AC27, AC28: extra forbid + tasks element/None handling.""" + + def test_unknown_field_raises(self): + # AC11. + with self.assertRaises(ValidationError): + AccuracyConfig(tasks=[], extra_field=1) + + def test_non_dict_non_instance_element_raises(self): + # AC27. + with self.assertRaises(ValidationError): + AccuracyConfig(tasks=["not-a-task"]) + + def test_tasks_none_raises(self): + # AC28: field is not Optional; only omission yields the [] default. + with self.assertRaises(ValidationError): + AccuracyConfig(tasks=None) + + +class TestAccuracyConfigDuplicateIds(unittest.TestCase): + """AC4-AC9: the model_validator(mode='after') duplicate-id contract.""" + + def test_single_duplicate_group(self): + # AC4: prefix + the offending id present. + with self.assertRaises(ValidationError) as ctx: + AccuracyConfig( + tasks=[ + AccuracyTask(id="dup-mmlu", task="mmlu"), + AccuracyTask(id="dup-mmlu", task="mmlu"), + ] + ) + dupes = _dupes_message(ctx.exception) + self.assertIsNotNone(dupes, "expected 'duplicate task id(s):' prefix") + self.assertIn("dup-mmlu", dupes) + + def test_two_groups_sorted_ascending(self): + # AC5: both ids present; 'dup-gsm8k' before 'dup-mmlu' (sorted, not + # encounter order -- input intentionally lists mmlu first). + with self.assertRaises(ValidationError) as ctx: + AccuracyConfig( + tasks=[ + AccuracyTask(id="dup-mmlu", task="mmlu"), + AccuracyTask(id="dup-gsm8k", task="gsm8k"), + AccuracyTask(id="dup-mmlu", task="mmlu"), + AccuracyTask(id="dup-gsm8k", task="gsm8k"), + ] + ) + dupes = _dupes_message(ctx.exception) + self.assertIsNotNone(dupes) + i_gsm = dupes.find("dup-gsm8k") + i_mmlu = dupes.find("dup-mmlu") + self.assertNotEqual(i_gsm, -1) + self.assertNotEqual(i_mmlu, -1) + self.assertLess(i_gsm, i_mmlu, "ids must be sorted ascending in the message") + + def test_mixed_case_dupes_sorted_case_sensitively(self): + # AC5 (sort discriminator): the sort must be case-SENSITIVE lexicographic, + # distinct from AC8's case-sensitive equality. All-lowercase fixtures + # (dup-gsm8k/dup-mmlu) cannot tell a correct sorted(dupes) from a + # spec-violating sorted(dupes, key=str.lower). Use ids that differ in + # leading case: case-sensitive sort orders uppercase before lowercase + # ('Dup-Zebra' < 'dup-apple'), whereas a case-insensitive key flips them + # ('dup-apple' < 'Dup-Zebra' since 'a' < 'z'). + with self.assertRaises(ValidationError) as ctx: + AccuracyConfig( + tasks=[ + AccuracyTask(id="dup-apple", task="mmlu"), + AccuracyTask(id="Dup-Zebra", task="gsm8k"), + AccuracyTask(id="dup-apple", task="mmlu"), + AccuracyTask(id="Dup-Zebra", task="gsm8k"), + ] + ) + dupes = _dupes_message(ctx.exception) + self.assertIsNotNone(dupes) + i_zebra = dupes.find("Dup-Zebra") + i_apple = dupes.find("dup-apple") + self.assertNotEqual(i_zebra, -1) + self.assertNotEqual(i_apple, -1) + self.assertLess( + i_zebra, + i_apple, + "dupes must be sorted case-sensitively: uppercase-leading 'Dup-Zebra' " + "precedes 'dup-apple' (a case-insensitive sort key would reverse this)", + ) + + def test_triple_duplicate_id_listed_once(self): + # AC6: id repeated 3x appears exactly once in the sorted-dupes list. + with self.assertRaises(ValidationError) as ctx: + AccuracyConfig( + tasks=[ + AccuracyTask(id="dup-mmlu", task="mmlu"), + AccuracyTask(id="dup-mmlu", task="mmlu"), + AccuracyTask(id="dup-mmlu", task="mmlu"), + ] + ) + dupes = _dupes_message(ctx.exception) + self.assertIsNotNone(dupes) + self.assertEqual(dupes.count("dup-mmlu"), 1) + + def test_duplicate_by_id_only_ignores_other_fields(self): + # AC7: same id, different task -> still duplicates. The raise itself is + # the discriminator: full-object comparison would construct successfully. + # Use a distinctive id ("dup-x") that cannot be a substring of the fixed + # "duplicate task id(s):" prefix, so assertIn actually probes the + # validator's rendered dupes list rather than the constant prefix text. + with self.assertRaises(ValidationError) as ctx: + AccuracyConfig( + tasks=[ + AccuracyTask(id="dup-x", task="mmlu"), + AccuracyTask(id="dup-x", task="gsm8k"), + ] + ) + dupes = _dupes_message(ctx.exception) + self.assertIsNotNone(dupes, "must be the duplicate-id validator, not another error") + self.assertIn("dup-x", dupes) + + def test_only_repeated_ids_listed_not_unique_ones(self): + # AC4 (message contents): the dupes list must contain ONLY ids that + # actually repeat, not every distinct id in the config. Mix a duplicated + # id with an id that appears exactly once and assert the unique one is + # absent -- this fails a validator that reports sorted(set(all_ids)). + with self.assertRaises(ValidationError) as ctx: + AccuracyConfig( + tasks=[ + AccuracyTask(id="dup-repeated", task="mmlu"), + AccuracyTask(id="dup-repeated", task="gsm8k"), + AccuracyTask(id="only-once", task="mmlu"), + ] + ) + dupes = _dupes_message(ctx.exception) + self.assertIsNotNone(dupes) + self.assertIn("dup-repeated", dupes) + self.assertNotIn("only-once", dupes) + + def test_case_sensitive_ids_not_duplicates(self): + # AC8: 'MMLU' vs 'mmlu' differ only by case -> NOT duplicates. + cfg = AccuracyConfig( + tasks=[ + AccuracyTask(id="MMLU", task="mmlu"), + AccuracyTask(id="mmlu", task="mmlu"), + ] + ) + self.assertEqual([t.id for t in cfg.tasks], ["MMLU", "mmlu"]) + + def test_empty_string_duplicates_render_as_quotes(self): + # AC9: two id="" -> duplicate; renders as '' in the sorted list. + with self.assertRaises(ValidationError) as ctx: + AccuracyConfig( + tasks=[ + AccuracyTask(id="", task="a"), + AccuracyTask(id="", task="b"), + ] + ) + dupes = _dupes_message(ctx.exception) + self.assertIsNotNone(dupes) + self.assertIn("''", dupes) + + +class TestAccuracyConfigNonMutation(unittest.TestCase): + """AC30, AC31: input list not mutated; default tasks list not shared.""" + + def test_caller_dict_list_not_mutated(self): + # AC30. + lst = [{"id": "a", "task": "m"}] + AccuracyConfig(tasks=lst) + self.assertEqual(len(lst), 1) + self.assertIsInstance(lst[0], dict) + self.assertEqual(lst[0], {"id": "a", "task": "m"}) + + def test_default_tasks_not_shared_between_instances(self): + # AC31. + a = AccuracyConfig() + b = AccuracyConfig() + self.assertIsNot(a.tasks, b.tasks) + a.tasks.append(AccuracyTask(id="x", task="m")) + self.assertEqual(b.tasks, []) + + +class TestValidationPrecedence(unittest.TestCase): + """AC32: per-element field error surfaces before the duplicate-id validator.""" + + def test_field_error_preempts_duplicate_validator(self): + with self.assertRaises(ValidationError) as ctx: + AccuracyConfig(tasks=[{"id": "a", "task": "gsm8k"}, {"id": "a"}]) + msg = str(ctx.exception) + # The missing required 'task' field on element index 1 is what surfaces. + # Assert the fully-qualified error location "tasks.1.task" rather than a + # bare "task": the parent field name "tasks" means a plain "task" + # substring would also match "tasks.1.id" (i.e. the *other* field being + # the one missing), so it cannot tell which required field failed. + self.assertIn("tasks.1.task", msg) + self.assertTrue( + ("Field required" in msg) or ("missing" in msg), + f"expected a missing-required-field marker, got: {msg}", + ) + # ...and the duplicate-id validator must NOT have run. + self.assertNotIn("duplicate task id(s):", msg) + + +class TestModelFieldMembership(unittest.TestCase): + """AC33, AC34 + regression constraints: closed-set field membership.""" + + def test_accuracy_task_fields_exact(self): + # AC33. + self.assertEqual( + set(AccuracyTask.model_fields), + { + "id", + "task", + "num_fewshot", + "metadata", + "include_path", + "num_concurrent", + "apply_chat_template", + "gen_kwargs", + }, + ) + + def test_accuracy_config_fields_exact(self): + # AC34. + self.assertEqual(set(AccuracyConfig.model_fields), {"tasks"}) + + def test_no_threshold_or_gate_fields(self): + # Regression constraint: no gating/threshold wiring exists on these models. + forbidden = {"threshold", "gate", "min_score", "accuracy_gate", "accuracy"} + self.assertEqual(set(AccuracyTask.model_fields) & forbidden, set()) + self.assertEqual(set(AccuracyConfig.model_fields) & forbidden, set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_accuracy_eval_stage.py b/cvs/lib/inference/unittests/test_accuracy_eval_stage.py new file mode 100644 index 000000000..fdacdacb0 --- /dev/null +++ b/cvs/lib/inference/unittests/test_accuracy_eval_stage.py @@ -0,0 +1,236 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.inference.utils.inference_suite_lifecycle.test_accuracy_eval. + +Isolated via unittest.mock.patch on run_accuracy_tasks (imported into +inference_suite_lifecycle at module load time) so these tests never touch a +real orch or the network -- only the stage's own selection/gating/skip logic +is under test. + +test_accuracy_eval is parametrized by `accuracy_task` (one pytest node per +task id, mirroring test_metric/test_gpu_metric), so it is invoked here once +per task under test rather than once per variant_config as before. +''' + +import unittest +from types import SimpleNamespace +from unittest import mock + +import pytest + +from cvs.lib.inference.utils import inference_suite_lifecycle as lifecycle_mod +from cvs.lib.utils.verdict import ThresholdViolation + + +def _variant_config(tasks=(), thresholds=None, enforce_thresholds=True): + return SimpleNamespace( + accuracy=SimpleNamespace(tasks=list(tasks)) if tasks is not None else None, + params=SimpleNamespace(base_url="http://0.0.0.0", port_no="8000"), + paths=SimpleNamespace(log_dir="/logs"), + model=SimpleNamespace(id="meta-llama/Llama-3-8b"), + thresholds=thresholds or {}, + enforce_thresholds=enforce_thresholds, + ) + + +def _task(id_): + return SimpleNamespace(id=id_) + + +class _Lifecycle: + def __init__(self, failed=False): + self.failed = failed + self.report = {} + + def record(self, nodeid, label, value, unit="s"): + self.report.setdefault(nodeid, []).append((label, value, unit)) + + +def _request(nodeid="test_accuracy_eval"): + node = SimpleNamespace(nodeid=nodeid) + return SimpleNamespace(node=node) + + +class TestAccuracyEvalSkip(unittest.TestCase): + def test_skips_when_prior_stage_failed(self): + with self.assertRaises(pytest.skip.Exception): + lifecycle_mod.test_accuracy_eval( + orch=object(), + variant_config=_variant_config(tasks=[_task("mmlu")]), + accuracy_task="mmlu", + lifecycle=_Lifecycle(failed=True), + request=_request(), + ) + + def test_skips_when_accuracy_block_absent(self): + vc = _variant_config(tasks=None) + with self.assertRaises(pytest.skip.Exception): + lifecycle_mod.test_accuracy_eval( + orch=object(), variant_config=vc, accuracy_task="mmlu", lifecycle=_Lifecycle(), request=_request() + ) + + def test_skips_when_tasks_empty(self): + vc = _variant_config(tasks=[]) + with self.assertRaises(pytest.skip.Exception): + lifecycle_mod.test_accuracy_eval( + orch=object(), variant_config=vc, accuracy_task="mmlu", lifecycle=_Lifecycle(), request=_request() + ) + + def test_skips_when_task_id_not_in_configured_tasks(self): + # config.json's accuracy.tasks no longer includes this id (e.g. removed + # after collection-time parametrization but before this node ran). + vc = _variant_config(tasks=[_task("mmlu")]) + with self.assertRaises(pytest.skip.Exception): + lifecycle_mod.test_accuracy_eval( + orch=object(), variant_config=vc, accuracy_task="gsm8k", lifecycle=_Lifecycle(), request=_request() + ) + + +class TestAccuracyEvalRun(unittest.TestCase): + def test_calls_run_accuracy_tasks_with_only_this_task(self): + vc = _variant_config(tasks=[_task("mmlu"), _task("gsm8k")]) + lc = _Lifecycle() + with mock.patch.object( + lifecycle_mod, "run_accuracy_tasks", return_value={"mmlu": {"mmlu.acc__none": 0.7}} + ) as m: + lifecycle_mod.test_accuracy_eval( + orch="ORCH", variant_config=vc, accuracy_task="mmlu", lifecycle=lc, request=_request() + ) + m.assert_called_once() + kwargs = m.call_args.kwargs + self.assertEqual(kwargs["orch"], "ORCH") + self.assertEqual(kwargs["base_url"], "http://0.0.0.0:8000") + self.assertEqual(kwargs["model_id"], "meta-llama/Llama-3-8b") + self.assertEqual(kwargs["model_path"], "meta-llama/Llama-3-8b") + self.assertEqual(kwargs["output_dir"], "/logs/accuracy") + self.assertEqual([t.id for t in kwargs["tasks"]], ["mmlu"]) + + def test_record_only_when_no_threshold_entry(self): + vc = _variant_config(tasks=[_task("mmlu")], thresholds={}) + lc = _Lifecycle() + with mock.patch.object(lifecycle_mod, "run_accuracy_tasks", return_value={"mmlu": {"mmlu.acc__none": 0.7}}): + lifecycle_mod.test_accuracy_eval( + orch="ORCH", variant_config=vc, accuracy_task="mmlu", lifecycle=lc, request=_request() + ) + self.assertFalse(lc.failed) + recorded = dict((label, (value, unit)) for label, value, unit in lc.report["test_accuracy_eval"]) + self.assertEqual(recorded["mmlu.mmlu.acc__none"], (0.7, "")) + + def test_threshold_pass_does_not_raise(self): + vc = _variant_config( + tasks=[_task("mmlu")], + thresholds={"accuracy": {"mmlu": {"mmlu.acc__none": {"kind": "min", "value": 0.5}}}}, + ) + lc = _Lifecycle() + with mock.patch.object(lifecycle_mod, "run_accuracy_tasks", return_value={"mmlu": {"mmlu.acc__none": 0.7}}): + lifecycle_mod.test_accuracy_eval( + orch="ORCH", variant_config=vc, accuracy_task="mmlu", lifecycle=lc, request=_request() + ) + self.assertFalse(lc.failed) + + def test_threshold_miss_raises_threshold_violation(self): + vc = _variant_config( + tasks=[_task("mmlu")], + thresholds={"accuracy": {"mmlu": {"mmlu.acc__none": {"kind": "min", "value": 0.9}}}}, + ) + lc = _Lifecycle() + with mock.patch.object(lifecycle_mod, "run_accuracy_tasks", return_value={"mmlu": {"mmlu.acc__none": 0.7}}): + with self.assertRaises(ThresholdViolation): + lifecycle_mod.test_accuracy_eval( + orch="ORCH", variant_config=vc, accuracy_task="mmlu", lifecycle=lc, request=_request() + ) + # A threshold violation on this task's own node must NOT flip the + # shared lifecycle flag -- sibling task nodes must still run. + self.assertFalse(lc.failed) + + def test_removed_task_stale_threshold_entry_ignored(self): + # config.json only selects "mmlu"; threshold.json still has a stale + # "gsm8k" entry from a since-removed task -- must not be looked up. + vc = _variant_config( + tasks=[_task("mmlu")], + thresholds={ + "accuracy": { + "mmlu": {"mmlu.acc__none": {"kind": "min", "value": 0.5}}, + "gsm8k": {"exact_match__strict-match": {"kind": "min", "value": 0.99}}, + } + }, + ) + lc = _Lifecycle() + with mock.patch.object(lifecycle_mod, "run_accuracy_tasks", return_value={"mmlu": {"mmlu.acc__none": 0.7}}): + lifecycle_mod.test_accuracy_eval( + orch="ORCH", variant_config=vc, accuracy_task="mmlu", lifecycle=lc, request=_request() + ) + self.assertFalse(lc.failed) + + def test_run_failure_pytest_fails_without_setting_shared_lifecycle_failed(self): + vc = _variant_config(tasks=[_task("mmlu")]) + lc = _Lifecycle() + with mock.patch.object(lifecycle_mod, "run_accuracy_tasks", side_effect=RuntimeError("boom")): + with self.assertRaises(pytest.fail.Exception): + lifecycle_mod.test_accuracy_eval( + orch="ORCH", variant_config=vc, accuracy_task="mmlu", lifecycle=lc, request=_request() + ) + # Independent failure isolation: a run failure in one task's node must + # not set the shared lifecycle.failed flag, or every sibling task node + # (parametrized on the same fixture) would skip instead of running. + self.assertFalse(lc.failed) + + def test_threshold_miss_recorded_but_not_raised_when_enforce_thresholds_false(self): + vc = _variant_config( + tasks=[_task("mmlu")], + thresholds={"accuracy": {"mmlu": {"mmlu.acc__none": {"kind": "min", "value": 0.9}}}}, + enforce_thresholds=False, + ) + lc = _Lifecycle() + with mock.patch.object(lifecycle_mod, "run_accuracy_tasks", return_value={"mmlu": {"mmlu.acc__none": 0.7}}): + lifecycle_mod.test_accuracy_eval( + orch="ORCH", variant_config=vc, accuracy_task="mmlu", lifecycle=lc, request=_request() + ) + self.assertFalse(lc.failed) + recorded = dict((label, (value, unit)) for label, value, unit in lc.report["test_accuracy_eval"]) + self.assertEqual(recorded["mmlu.mmlu.acc__none"], (0.7, "")) + + def test_two_tasks_gated_independently_one_fails_other_unaffected(self): + # Simulates two parametrized nodes sharing one lifecycle object, in + # collection order: mmlu's node raises ThresholdViolation, but gsm8k's + # node (called after, as pytest would for the next parametrized item) + # must still run and pass on its own merits. + vc = _variant_config( + tasks=[_task("mmlu"), _task("gsm8k")], + thresholds={ + "accuracy": { + "mmlu": {"mmlu.acc__none": {"kind": "min", "value": 0.9}}, + "gsm8k": {"gsm8k.exact_match__strict-match": {"kind": "min", "value": 0.5}}, + } + }, + ) + lc = _Lifecycle() + + with mock.patch.object(lifecycle_mod, "run_accuracy_tasks", return_value={"mmlu": {"mmlu.acc__none": 0.7}}): + with self.assertRaises(ThresholdViolation): + lifecycle_mod.test_accuracy_eval( + orch="ORCH", + variant_config=vc, + accuracy_task="mmlu", + lifecycle=lc, + request=_request("test_accuracy_eval[mmlu]"), + ) + self.assertFalse(lc.failed) + + with mock.patch.object( + lifecycle_mod, "run_accuracy_tasks", return_value={"gsm8k": {"gsm8k.exact_match__strict-match": 0.6}} + ): + lifecycle_mod.test_accuracy_eval( + orch="ORCH", + variant_config=vc, + accuracy_task="gsm8k", + lifecycle=lc, + request=_request("test_accuracy_eval[gsm8k]"), + ) + self.assertFalse(lc.failed) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_atom_config_loader.py b/cvs/lib/inference/unittests/test_atom_config_loader.py new file mode 100644 index 000000000..616a5f51f --- /dev/null +++ b/cvs/lib/inference/unittests/test_atom_config_loader.py @@ -0,0 +1,382 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.inference.atom.atom_config_loader. +''' + +import unittest +from pathlib import Path + +from cvs.lib.inference.atom.atom_config_loader import ( + AtomVariantConfig, + expand_sweep, + expand_sweep_parametrize, + load_variant, + orchestrator_container_from_variant, + placeholder_gated_threshold_cell, + reuse_server_flag, + server_session_key, +) +from cvs.lib.inference.utils.inferencing_config_loader import Run, SeqCombo, Sweep + + +def _cluster_dict(): + return {"username": "testuser"} + + +class TestATOMAtomConfigLoader(unittest.TestCase): + def test_load_mi300x_sample_config(self): + root = Path(__file__).resolve().parents[3] + config = root / ("input/config_file/inference/atom/mi300x_atom_gpt-oss-120b_bf16.json") + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.framework, "atom") + self.assertEqual(variant.params.driver, "vllm") + self.assertEqual(variant.expected_cells(), ["ISL=7168,OSL=1024,TP=8,CONC=64"]) + self.assertIn("enforce-eager", variant.roles.server.serve_args) + + def test_load_w1_mi300x_atom_variant(self): + root = Path(__file__).resolve().parents[3] + config = root / ("input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_single.json") + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.threshold_json, "mi300x_atom_deepseek-r1_fp8_single_threshold.json") + self.assertEqual(variant.gpu_arch, "mi300x") + self.assertEqual(variant.params.driver, "atom") + self.assertEqual(variant.params.metric_percentiles, "95,99") + self.assertEqual( + variant.roles.server.atom_args[:4], + ["-tp", "8", "--kv_cache_dtype", "fp8"], + ) + self.assertEqual( + variant.expected_cells(), + ["ISL=1024,OSL=1024,TP=8,CONC=128", "ISL=1024,OSL=1024,TP=8,CONC=256"], + ) + cell = "ISL=1024,OSL=1024,TP=8,CONC=128" + for key in ( + "client.per_gpu_throughput", + "client.output_tput_per_gpu", + "client.p99_ttft_ms", + "client.p99_tpot_ms", + "client.p95_tpot_ms", + ): + self.assertIn(key, variant.thresholds[cell]) + + def test_load_w1_mi300x_multinode_variant(self): + root = Path(__file__).resolve().parents[3] + config = root / ("input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_distributed.json") + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.params.nnodes, "2") + self.assertEqual(variant.params.driver, "vllm_atom") + self.assertEqual(variant.params.pipeline_parallel_size, "2") + self.assertEqual(variant.roles.server.ib_netdev, "auto") + self.assertEqual(variant.roles.server.ib_hca_devices, "auto") + self.assertEqual(variant.params.scaling_baseline_output_throughput, "1500") + self.assertTrue(variant.enforce_thresholds) + self.assertEqual(len(variant.expected_cells()), 15) + cell = "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=16" + self.assertIn(cell, variant.expected_cells()) + self.assertEqual( + variant.thresholds[cell]["scaling.efficiency_pct"], + {"kind": "min", "value": 11}, + ) + + def test_load_w1_mi300x_multinode_sglang_variant(self): + root = Path(__file__).resolve().parents[3] + config = root / ("input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_sglang_distributed.json") + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.params.driver, "sglang") + self.assertEqual(variant.params.pipeline_parallel_size, "2") + self.assertFalse(variant.enforce_thresholds) + cell = "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=16" + self.assertIn(cell, variant.expected_cells()) + + def test_load_w1_mi355x_multinode_variant(self): + root = Path(__file__).resolve().parents[3] + config = root / ("input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_distributed.json") + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.gpu_arch, "mi355x") + self.assertEqual(variant.params.nnodes, "2") + self.assertEqual(variant.params.driver, "vllm_atom") + self.assertEqual(variant.params.pipeline_parallel_size, "2") + self.assertEqual(variant.params.scaling_baseline_output_throughput, "4000") + self.assertFalse(variant.enforce_thresholds) + self.assertEqual(len(variant.expected_cells()), 15) + cell = "ISL=512,OSL=512,TP=8,PP=2,NNODES=2,CONC=16" + self.assertIn(cell, variant.expected_cells()) + self.assertEqual( + variant.thresholds[cell]["scaling.efficiency_pct"], + {"kind": "min", "value": 50}, + ) + + def test_load_baseline_sweep_mi300x_variant(self): + root = Path(__file__).resolve().parents[3] + config = root / ("input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_baseline_sweep.json") + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.params.max_model_length, "10240") + self.assertTrue(variant.enforce_thresholds) + self.assertEqual(len(variant.expected_cells()), 14) + self.assertIn("ISL=1024,OSL=1024,TP=8,CONC=4", variant.expected_cells()) + self.assertIn("ISL=8192,OSL=1024,TP=8,CONC=256", variant.expected_cells()) + cell = "ISL=8192,OSL=1024,TP=8,CONC=128" + self.assertIn("client.output_throughput", variant.thresholds[cell]) + self.assertEqual(variant.thresholds[cell]["client.success_rate"]["value"], 1) + + def test_load_baseline_sweep_multinode_mi300x_variant(self): + root = Path(__file__).resolve().parents[3] + config = root / ("input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_baseline_sweep_distributed.json") + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.params.nnodes, "2") + self.assertEqual(variant.params.driver, "vllm_atom") + self.assertEqual(variant.params.pipeline_parallel_size, "2") + self.assertEqual(variant.params.max_model_length, "10240") + self.assertEqual(variant.params.scaling_baseline_output_throughput, "1500") + self.assertTrue(variant.enforce_thresholds) + self.assertEqual(len(variant.expected_cells()), 14) + cell = "ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=4" + self.assertIn(cell, variant.expected_cells()) + self.assertEqual( + variant.thresholds[cell]["scaling.efficiency_pct"], + {"kind": "min", "value": 9.0}, + ) + root = Path(__file__).resolve().parents[3] + config = root / ("input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_baseline_sweep.json") + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.gpu_arch, "mi355x") + self.assertFalse(variant.enforce_thresholds) + self.assertEqual(len(variant.expected_cells()), 14) + + def test_load_w1_mi355x_atom_single_variant_and_thresholds(self): + root = Path(__file__).resolve().parents[3] + config = root / ("input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_single.json") + variant = load_variant(config, _cluster_dict()) + self.assertEqual(variant.gpu_arch, "mi355x") + self.assertIn("--trust-remote-code", variant.roles.server.atom_args) + self.assertEqual( + variant.expected_cells(), + ["ISL=1024,OSL=1024,TP=8,CONC=128", "ISL=1024,OSL=1024,TP=8,CONC=256"], + ) + cell = "ISL=1024,OSL=1024,TP=8,CONC=128" + self.assertEqual( + variant.thresholds[cell]["client.output_throughput"]["value"], + 4004.66, + ) + self.assertEqual( + variant.thresholds[cell]["client.mean_ttft_ms"]["value"], + 362.18, + ) + + def test_load_w1_mi355x_atom_mtp3_inline_bench_args(self): + root = Path(__file__).resolve().parents[3] + config = root / ("input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_mtp3.json") + variant = load_variant(config, _cluster_dict()) + self.assertIn("--method", variant.roles.server.atom_args) + self.assertEqual(variant.params.bench_extra_args, "--use-chat-template") + + def test_load_w1_mi355x_atom_mtp3_thresholds(self): + root = Path(__file__).resolve().parents[3] + config = root / ("input/config_file/inference/atom/mi355x_atom_deepseek-r1_fp8_mtp3.json") + variant = load_variant(config, _cluster_dict()) + cell = "ISL=1024,OSL=1024,TP=8,CONC=256" + self.assertEqual( + variant.thresholds[cell]["client.output_throughput"]["value"], + 6451.59, + ) + + def test_orchestrator_container_includes_server_env(self): + sweep = Sweep( + sequence_combinations=[SeqCombo(name="legacy_profile", isl="7168", osl="1024")], + runs=[Run(combo="legacy_profile", concurrency=64)], + ) + thresholds = { + "ISL=7168,OSL=1024,TP=8,CONC=64": placeholder_gated_threshold_cell(), + } + variant = AtomVariantConfig( + schema_version=1, + framework="atom", + gpu_arch="mi300x", + enforce_thresholds=False, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "openai/gpt-oss-120b", "remote": 0, "precision": "bf16"}, + container={ + "name": "c", + "image": "img", + "runtime": {"name": "docker", "args": {"volumes": ["/home/x:/home/x"]}}, + }, + roles={"server": {"env": {"VLLM_ROCM_USE_AITER": "1"}}}, + params={"tensor_parallelism": "8"}, + sweep=sweep, + thresholds=thresholds, + ) + block = orchestrator_container_from_variant(variant) + self.assertEqual(block["env"]["VLLM_ROCM_USE_AITER"], "1") + + def test_expand_sweep_matches_w1_single(self): + root = Path(__file__).resolve().parents[3] + config = root / ("input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_single.json") + import json + + raw = json.loads(config.read_text()) + cases, ids = expand_sweep(raw["sweep"]) + self.assertEqual(len(cases), 2) + self.assertEqual(ids[0], "w1_1k_1k-conc128") + self.assertEqual(ids[1], "w1_1k_1k-conc256") + self.assertEqual(cases[0][1], 128) + + def test_w1_single_threshold_health_gates_tight_when_enforcing(self): + root = Path(__file__).resolve().parents[3] + config = root / ("input/config_file/inference/atom/mi300x_atom_deepseek-r1_fp8_single.json") + variant = load_variant(config, _cluster_dict()) + self.assertTrue(variant.enforce_thresholds) + cell = "ISL=1024,OSL=1024,TP=8,CONC=128" + self.assertEqual(variant.thresholds[cell]["client.success_rate"]["value"], 1) + self.assertEqual(variant.thresholds[cell]["client.failed"]["value"], 0) + + def test_placeholder_threshold_cell_covers_gated_metrics(self): + cell = placeholder_gated_threshold_cell() + from cvs.lib.inference.atom.atom_parsing import GATED_METRICS + + for short in GATED_METRICS: + self.assertIn(f"client.{short}", cell, short) + + def test_atom_driver_requires_inline_atom_args(self): + sweep = Sweep( + sequence_combinations=[SeqCombo(name="w1", isl="1024", osl="1024")], + runs=[Run(combo="w1", concurrency=128)], + ) + thresholds = {"ISL=1024,OSL=1024,TP=8,CONC=128": placeholder_gated_threshold_cell()} + with self.assertRaises(ValueError): + AtomVariantConfig( + schema_version=1, + framework="atom", + gpu_arch="mi300x", + enforce_thresholds=False, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "deepseek-ai/DeepSeek-R1-0528", "remote": 0, "precision": "fp8"}, + container={ + "name": "c", + "image": "img", + "runtime": {"name": "docker", "args": {"volumes": ["/home/x:/home/x"]}}, + }, + roles={"server": {"env": {}}}, + params={"driver": "atom", "tensor_parallelism": "8"}, + sweep=sweep, + thresholds=thresholds, + ) + + def test_reuse_server_flag_and_session_key_helpers(self): + from types import SimpleNamespace + + self.assertFalse(reuse_server_flag(SimpleNamespace())) + variant = SimpleNamespace( + model=SimpleNamespace(id="m"), + params=SimpleNamespace( + driver="atom", + tensor_parallelism="8", + nnodes="1", + pipeline_parallel_size="1", + master_addr="", + master_port="29501", + ), + roles=SimpleNamespace(server=SimpleNamespace(atom_args=("-tp", "8"))), + ) + self.assertNotEqual(server_session_key(variant, "1", "2"), server_session_key(variant, "3", "4")) + + def test_expand_sweep_parametrize_tier_ids(self): + sweep = { + "sequence_combinations": [{"name": "w1", "isl": "1024", "osl": "1024"}], + "runs": [{"combo": "w1", "concurrency": 128}], + } + _, _, ids = expand_sweep_parametrize(sweep, ("metric_tier",)) + self.assertIn("w1-conc128-throughput", ids) + + def test_ib_netdev_coerces_mlx5_hca_name_to_auto(self): + sweep = Sweep( + sequence_combinations=[SeqCombo(name="w1", isl="512", osl="512")], + runs=[Run(combo="w1", concurrency=16)], + ) + variant = AtomVariantConfig( + schema_version=1, + framework="atom", + gpu_arch="mi300x", + enforce_thresholds=False, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "deepseek-ai/DeepSeek-R1-0528", "remote": 0, "precision": "fp8"}, + container={ + "name": "c", + "image": "img", + "runtime": {"name": "docker", "args": {"volumes": ["/home/x:/home/x"]}}, + }, + roles={"server": {"ib_netdev": "mlx5_0"}}, + params={ + "driver": "vllm_atom", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "10.0.0.1", + }, + sweep=sweep, + thresholds={}, + ) + self.assertEqual(variant.roles.server.ib_netdev, "auto") + + def test_server_env_strips_orchestrator_network_keys(self): + sweep = Sweep( + sequence_combinations=[SeqCombo(name="w1", isl="512", osl="512")], + runs=[Run(combo="w1", concurrency=16)], + ) + variant = AtomVariantConfig( + schema_version=1, + framework="atom", + gpu_arch="mi300x", + enforce_thresholds=False, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "deepseek-ai/DeepSeek-R1-0528", "remote": 0, "precision": "fp8"}, + container={ + "name": "c", + "image": "img", + "runtime": {"name": "docker", "args": {"volumes": ["/home/x:/home/x"]}}, + }, + roles={ + "server": { + "env": { + "GLOO_SOCKET_IFNAME": "mlx5_0", + "NCCL_IB_HCA": "mlx5_0", + "NCCL_IB_GID_INDEX": "1", + } + } + }, + params={ + "driver": "vllm_atom", + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "10.0.0.1", + }, + sweep=sweep, + thresholds={}, + ) + self.assertEqual(variant.roles.server.env, {"NCCL_IB_GID_INDEX": "1"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_atom_orch_parse.py b/cvs/lib/inference/unittests/test_atom_orch_parse.py new file mode 100644 index 000000000..da8569979 --- /dev/null +++ b/cvs/lib/inference/unittests/test_atom_orch_parse.py @@ -0,0 +1,632 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for AtomJob.parse_results (stock ``results`` artifact -> client.*). +No hardware: a fake orch returns committed fixture text. +''' + +import json +import unittest +from pathlib import Path +from types import SimpleNamespace +from typing import ClassVar +from unittest.mock import patch + +from cvs.lib.inference.atom.atom_orch import AtomJob +from cvs.lib.inference.unittests.fake_orch import FakeOrch + +_HERE = Path(__file__).parent +_FIXTURES = _HERE / "fixtures" +_ISL = 7168 +_OSL = 1024 +_TP = 8 +# Prefill fabric so build_server_cmd skips lazy IB discovery in FakeOrch tests. +_PREFILLED_FABRIC = {"ib_hcas": ["mlx5_0"], "ib_netdev": "eth0"} + + +def _fake_variant( + *, + driver="vllm", + nnodes="1", + pipeline_parallel_size="1", + master_addr="", + scaling_baseline_output_throughput="", + ib_netdev="eth0", + ib_hca_devices=None, +): + params = SimpleNamespace( + driver=driver, + tensor_parallelism=str(_TP), + pipeline_parallel_size=pipeline_parallel_size, + nnodes=nnodes, + master_addr=master_addr, + master_port="29501", + scaling_baseline_output_throughput=scaling_baseline_output_throughput, + port_no="8000", + random_range_ratio="0.8", + random_prefix_len="0", + burstiness="1.0", + seed="0", + request_rate="inf", + tokenizer_mode="auto", + percentile_metrics="ttft,tpot,itl,e2el", + metric_percentiles="99", + base_url="http://0.0.0.0", + dataset_name="random", + backend="vllm", + max_model_length="8192", + bench_extra_args="", + result_filename="results", + ) + roles = SimpleNamespace( + server=SimpleNamespace( + serve_args={}, atom_args=[], sglang_args=[], env={}, ib_netdev=ib_netdev, ib_hca_devices=ib_hca_devices + ) + ) + paths = SimpleNamespace(log_dir="/LOGS", models_dir="/models") + model = SimpleNamespace(id="openai/gpt-oss-120b") + return SimpleNamespace(params=params, roles=roles, paths=paths, model=model) + + +class TestATOMAtomOrchParse(unittest.TestCase): + def test_parse_results_maps_client_metrics(self): + raw = json.loads((_FIXTURES / "vllm_results_sample.json").read_text()) + job = AtomJob( + orch=FakeOrch(exec_return={"node0": json.dumps(raw)}), + variant=_fake_variant(driver="vllm"), + hf_token="tok", + isl=_ISL, + osl=_OSL, + concurrency=64, + num_prompts=100, + ) + out = job.parse_results() + metrics = out["node0"] + w = raw + self.assertIn("client.output_throughput", metrics) + self.assertIn("client.mean_ttft_ms", metrics) + self.assertAlmostEqual(metrics["client.per_gpu_throughput"], w["total_token_throughput"] / _TP) + self.assertAlmostEqual(metrics["client.output_tput_per_gpu"], w["output_throughput"] / _TP) + self.assertEqual(metrics["client.p99_ttft_ms"], w["p99_ttft_ms"]) + + def test_parse_results_w1_tail_metrics_from_widened_fixture(self): + raw = json.loads((_FIXTURES / "vllm_results_widened.json").read_text()) + job = AtomJob( + orch=FakeOrch(exec_return={"node0": json.dumps(raw)}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + metrics = job.parse_results()["node0"] + self.assertEqual(metrics["client.p95_tpot_ms"], raw["p95_tpot_ms"]) + self.assertEqual(metrics["client.p99_ttft_ms"], raw["p99_ttft_ms"]) + + def test_parse_results_atom_json_suffix(self): + raw = json.loads((_FIXTURES / "vllm_results_sample.json").read_text()) + job = AtomJob( + orch=FakeOrch(exec_return={"node0": json.dumps(raw)}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + self.assertTrue(job._result_artifact.endswith("/results.json")) + out = job.parse_results() + self.assertIn("client.output_throughput", out["node0"]) + + def test_run_client_clears_stale_result_artifact(self): + orch = FakeOrch() + job = AtomJob( + orch=orch, + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=1000, + ) + job.run_client() + rm_cmds = [c for c, _ in orch.commands if c.startswith("rm -f ")] + self.assertEqual(len(rm_cmds), 1) + self.assertIn(job._result_artifact, rm_cmds[0]) + self.assertTrue(any("benchmark_serving" in c for c, _ in orch.commands)) + + def test_parse_results_empty_artifact_raises(self): + job = AtomJob( + orch=FakeOrch(exec_return={"node0": ""}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + with self.assertRaisesRegex(RuntimeError, "empty/missing results artifact"): + job.parse_results() + + def test_parse_results_invalid_json_raises(self): + job = AtomJob( + orch=FakeOrch(exec_return={"node0": "not-json"}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + with self.assertRaisesRegex(RuntimeError, "unparseable results artifact"): + job.parse_results() + + def test_merged_serve_args_promotes_gpu_memory_util(self): + variant = _fake_variant(driver="vllm") + variant.roles.server.env = {"CVS_GPU_MEMORY_UTIL": "0.92"} + merged = AtomJob._merged_serve_args(variant) + self.assertEqual(merged["gpu-memory-utilization"], "0.92") + + def test_merged_serve_args_skips_promotion_when_flag_present(self): + variant = _fake_variant(driver="vllm") + variant.roles.server.serve_args = {"gpu-memory-utilization": "0.75"} + variant.roles.server.env = {"CVS_GPU_MEMORY_UTIL": "0.92"} + merged = AtomJob._merged_serve_args(variant) + self.assertEqual(merged["gpu-memory-utilization"], "0.75") + + def test_build_server_cmd_suppresses_gpu_memory_env_vars(self): + orch = FakeOrch() + variant = _fake_variant(driver="vllm") + variant.roles.server.env = { + "CVS_GPU_MEMORY_UTIL": "0.92", + "VLLM_GPU_MEMORY_UTIL": "0.91", + "VLLM_ENFORCE_EAGER": "1", + "CUSTOM_FLAG": "on", + } + job = AtomJob( + orch=orch, + variant=variant, + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job.build_server_cmd() + env_cmd = orch.commands[0][0] + self.assertNotIn("CVS_GPU_MEMORY_UTIL", env_cmd) + self.assertNotIn("VLLM_GPU_MEMORY_UTIL", env_cmd) + self.assertNotIn("VLLM_ENFORCE_EAGER", env_cmd) + self.assertIn("CUSTOM_FLAG", env_cmd) + + def test_client_log_failures_traceback(self): + job = AtomJob( + orch=FakeOrch(exec_return={"node0": "Traceback (most recent call last):\n boom"}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + failed = job._client_log_failures() + self.assertEqual(len(failed), 1) + self.assertIn("node0", failed[0][0]) + + def test_client_log_failures_launch_error(self): + job = AtomJob( + orch=FakeOrch(exec_return={"node0": "error: argument --foo: invalid choice"}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + self.assertEqual(len(job._client_log_failures()), 1) + + def test_client_log_failures_failed_requests_over_cap(self): + job = AtomJob( + orch=FakeOrch(exec_return={"node0": "Failed requests: 3\n"}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job._bench_max_failed_requests = 0 + failed = job._client_log_failures() + self.assertEqual(len(failed), 1) + self.assertIn("Failed requests: 3", failed[0][1]) + + def test_client_log_failures_failed_requests_within_cap_warns(self): + job = AtomJob( + orch=FakeOrch(exec_return={"node0": "Failed requests: 1\n"}), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job._bench_max_failed_requests = 2 + failed = job._client_log_failures() + self.assertEqual(failed, []) + + def test_early_failure_regexes(self): + job = AtomJob( + orch=FakeOrch(), + variant=_fake_variant(driver="atom"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + self.assertTrue(job.FAILED_REQUESTS_RE.search("Failed requests: 2")) + self.assertTrue(job.CLIENT_CRASH_RE.search("Traceback (most recent call last)")) + self.assertTrue(job.CLIENT_LAUNCH_FAIL_RE.search("unrecognized arguments: --bad")) + self.assertTrue(job.EARLY_FAILURE_RE.search("No such file or directory")) + + def test_distributed_start_server_targets_each_host(self): + orch = FakeOrch(hosts=["10.0.0.1", "10.0.0.2"]) + job = AtomJob( + orch=orch, + variant=_fake_variant(driver="atom", nnodes="2", pipeline_parallel_size="2", master_addr="10.0.0.1"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + **_PREFILLED_FABRIC, + ) + job.build_server_cmd(clear_atom_cache=False) + job.start_server() + launch_cmds = [c for c, hosts in orch.commands if hosts] + self.assertEqual(len(launch_cmds), 2) + self.assertNotIn("--node-rank", launch_cmds[0]) + self.assertNotIn("--distributed-executor-backend", launch_cmds[0]) + self.assertIn("openai_server", launch_cmds[0]) + + def test_distributed_atom_spmd_env_and_dp_when_tp_allows(self): + orch = FakeOrch(hosts=["10.0.0.1", "10.0.0.2"]) + variant = _fake_variant(driver="atom", nnodes="2", pipeline_parallel_size="2", master_addr="10.0.0.1") + variant.params.tensor_parallelism = "4" + variant.roles.server.atom_args = ["-tp", "4"] + job = AtomJob( + orch=orch, + variant=variant, + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job.start_server() + launch_cmds = [c for c, hosts in orch.commands if hosts] + self.assertIn("-dp 2", launch_cmds[0]) + self.assertIn("ATOM_DP_RANK=0", launch_cmds[0]) + self.assertIn("ATOM_DP_RANK=1", launch_cmds[1]) + self.assertIn("ATOM_DP_MASTER_IP=10.0.0.1", launch_cmds[0]) + + def test_distributed_atom_tp8_multinode_couples_spmd_dp(self): + orch = FakeOrch(hosts=["10.0.0.1", "10.0.0.2"]) + variant = _fake_variant(driver="atom", nnodes="2", pipeline_parallel_size="2", master_addr="10.0.0.1") + variant.params.tensor_parallelism = "8" + variant.roles.server.atom_args = ["-tp", "8"] + job = AtomJob( + orch=orch, + variant=variant, + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job.start_server() + launch_cmds = [c for c, hosts in orch.commands if hosts] + self.assertEqual(len(launch_cmds), 2) + self.assertIn("-dp 2", launch_cmds[0]) + self.assertIn("-dp 2", launch_cmds[1]) + self.assertIn("ATOM_DP_RANK=0", launch_cmds[0]) + self.assertIn("ATOM_DP_RANK=1", launch_cmds[1]) + self.assertIn("ATOM_DP_SIZE=2", launch_cmds[0]) + + def test_distributed_atom_tp8_multinode_never_passes_vllm_flags(self): + orch = FakeOrch(hosts=["10.0.0.1", "10.0.0.2"]) + variant = _fake_variant(driver="atom", nnodes="2", pipeline_parallel_size="2", master_addr="10.0.0.1") + variant.roles.server.atom_args = [ + "-tp", + "8", + "--node-rank", + "1", + "--pipeline-parallel-size", + "2", + ] + job = AtomJob( + orch=orch, + variant=variant, + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + argv = job._atom_server_argv(rank=1) + joined = " ".join(argv) + self.assertNotIn("--node-rank", joined) + self.assertNotIn("--pipeline-parallel-size", joined) + self.assertNotIn("--master-addr", joined) + self.assertIn("-tp 8", joined) + job.start_server() + launch_cmds = [c for c, hosts in orch.commands if hosts] + self.assertNotIn("--node-rank", launch_cmds[1]) + self.assertNotIn("--pipeline-parallel-size", launch_cmds[1]) + + def test_distributed_client_uses_exec_on_head(self): + orch = FakeOrch(hosts=["10.0.0.1", "10.0.0.2"]) + job = AtomJob( + orch=orch, + variant=_fake_variant(driver="atom", nnodes="2", pipeline_parallel_size="2"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job.run_client() + self.assertEqual(len(orch.exec_on_head_commands), 2) + self.assertTrue(any("benchmark_serving" in c for c in orch.exec_on_head_commands)) + + def test_distributed_vllm_atom_pp2_passes_vllm_executor_flags(self): + orch = FakeOrch(hosts=["10.0.0.1", "10.0.0.2"]) + variant = _fake_variant(driver="vllm_atom", nnodes="2", pipeline_parallel_size="2", master_addr="10.0.0.1") + job = AtomJob( + orch=orch, + variant=variant, + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + argv0 = job._server_argv(rank=0) + argv1 = job._server_argv(rank=1) + joined0 = " ".join(argv0) + joined1 = " ".join(argv1) + self.assertIn("--pipeline-parallel-size 2", joined0) + self.assertIn("--node-rank 0", joined0) + self.assertIn("--node-rank 1", joined1) + self.assertIn("--headless", joined1) + self.assertNotIn("--headless", joined0) + job.start_server() + launch_cmds = [c for c, hosts in orch.commands if hosts] + self.assertIn("vllm serve", launch_cmds[0]) + self.assertIn("--pipeline-parallel-size 2", launch_cmds[1]) + + def test_distributed_sglang_pp2_passes_sglang_dist_flags(self): + orch = FakeOrch(hosts=["10.0.0.1", "10.0.0.2"]) + variant = _fake_variant(driver="sglang", nnodes="2", pipeline_parallel_size="2", master_addr="10.0.0.1") + variant.roles.server.sglang_args = ["--trust-remote-code"] + job = AtomJob( + orch=orch, + variant=variant, + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + argv = job._sglang_server_argv(rank=1) + joined = " ".join(argv) + self.assertIn("sglang.launch_server", joined) + self.assertIn("--pp-size 2", joined) + self.assertIn("--node-rank 1", joined) + self.assertIn("--dist-init-addr 10.0.0.1:29501", joined) + client = " ".join(job._sglang_client_argv()) + self.assertIn("sglang.bench_serving", client) + + def test_parse_results_scaling_efficiency(self): + raw = json.loads((_FIXTURES / "vllm_results_sample.json").read_text()) + job = AtomJob( + orch=FakeOrch(exec_on_head_return={"head": json.dumps(raw)}), + variant=_fake_variant( + driver="atom", + nnodes="2", + scaling_baseline_output_throughput="100", + ), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + metrics = job.parse_results()["head"] + self.assertIn("scaling.efficiency_pct", metrics) + expected = (raw["output_throughput"] / (100.0 * 2)) * 100.0 + self.assertAlmostEqual(metrics["scaling.efficiency_pct"], expected) + + +class TestATOMAtomBuildServerCmd(unittest.TestCase): + @staticmethod + def _env_script(orch): + return orch.commands[0][0] + + @patch("cvs.lib.utils.ib_discovery.resolve_multinode_fabric", return_value=([], "eth0")) + def test_nccl_ib_hca_line_present_only_when_ib_hcas_supplied(self, _mock_resolve): + cases = [ + (["mlx5_0", "mlx5_1"], True), + ([], False), + (None, False), + ] + for ib_hcas, present in cases: + with self.subTest(ib_hcas=ib_hcas): + orch = FakeOrch() + job = AtomJob( + orch=orch, + variant=_fake_variant( + driver="vllm_atom", + nnodes="2", + pipeline_parallel_size="2", + master_addr="10.0.0.1", + ), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ib_hcas=ib_hcas, + ib_netdev="eth0", + ) + job.build_server_cmd() + script = self._env_script(orch) + if present: + self.assertIn("NCCL_IB_HCA", script) + self.assertIn("mlx5_0", script) + else: + self.assertNotIn("NCCL_IB_HCA", script) + + def test_socket_ifname_exports_present_only_when_distributed_ib_netdev_set(self): + orch = FakeOrch() + job = AtomJob( + orch=orch, + variant=_fake_variant( + driver="vllm_atom", + nnodes="2", + pipeline_parallel_size="2", + master_addr="10.0.0.1", + ), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + **_PREFILLED_FABRIC, + ) + job.build_server_cmd() + script = self._env_script(orch) + self.assertEqual(script.count("SOCKET_IFNAME"), 3) + self.assertIn("eth0", script) + + orch_single = FakeOrch() + job_single = AtomJob( + orch=orch_single, + variant=_fake_variant(driver="vllm"), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job_single.build_server_cmd() + script_single = self._env_script(orch_single) + self.assertNotIn("SOCKET_IFNAME", script_single) + + @patch("cvs.lib.utils.ib_discovery.resolve_multinode_fabric") + def test_build_server_cmd_resolves_topology_when_lifecycle_skipped(self, mock_resolve): + mock_resolve.return_value = (["mlx5_0", "mlx5_1"], "ens51f1np1") + orch = FakeOrch(hosts=["10.32.80.112", "10.32.80.113"]) + job = AtomJob( + orch=orch, + variant=_fake_variant( + driver="vllm_atom", + nnodes="2", + pipeline_parallel_size="2", + master_addr="10.32.80.112", + ib_netdev="auto", + ib_hca_devices="auto", + ), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=128, + num_prompts=100, + ) + job.build_server_cmd() + mock_resolve.assert_called_once() + script = self._env_script(orch) + self.assertIn("NCCL_IB_HCA", script) + self.assertIn("mlx5_0", script) + self.assertEqual(script.count("SOCKET_IFNAME"), 3) + self.assertIn("ens51f1np1", script) + + +class _RecordingOrch: + hosts: ClassVar[list[str]] = ["10.0.0.1", "10.0.0.2"] + + def __init__(self, responder=None, hosts=None): + self.calls = [] + self._responder = responder + if hosts is not None: + self.hosts = list(hosts) + + def exec(self, cmd, hosts=None, detailed=False, **kwargs): + self.calls.append((cmd, hosts)) + if self._responder is not None: + return self._responder(cmd, hosts, detailed) + return {} + + def exec_on_head(self, cmd, **kwargs): + return {} + + +def _readiness_responder(exit_code=0, empty=False): + def responder(cmd, hosts, detailed): + if empty: + return {} + host = hosts[0] if hosts else _RecordingOrch.hosts[0] + if detailed: + return {host: {"exit_code": exit_code, "output": "", "stdout": ""}} + return {host: ""} + + return responder + + +class TestATOMAtomIsReady(unittest.TestCase): + def test_multinode_vllm_atom_skips_worker_readiness_grep(self): + head, worker = _RecordingOrch.hosts + orch = _RecordingOrch(responder=_readiness_responder(exit_code=0)) + job = AtomJob( + orch=orch, + variant=_fake_variant( + driver="vllm_atom", + nnodes="2", + pipeline_parallel_size="2", + master_addr=head, + ), + hf_token="tok", + isl="512", + osl="512", + concurrency=16, + num_prompts=128, + ) + self.assertTrue(job.is_ready()) + worker_calls = [hosts for _cmd, hosts in orch.calls if hosts == [worker]] + self.assertEqual(worker_calls, [], "headless worker must not be grepped for Uvicorn startup") + self.assertTrue(any(hosts == [head] for _cmd, hosts in orch.calls)) + + def test_multinode_vllm_atom_false_when_head_not_ready(self): + head = _RecordingOrch.hosts[0] + orch = _RecordingOrch(responder=_readiness_responder(exit_code=1)) + job = AtomJob( + orch=orch, + variant=_fake_variant( + driver="vllm_atom", + nnodes="2", + pipeline_parallel_size="2", + master_addr=head, + ), + hf_token="tok", + isl="512", + osl="512", + concurrency=16, + num_prompts=128, + ) + self.assertFalse(job.is_ready()) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_atom_parsing.py b/cvs/lib/inference/unittests/test_atom_parsing.py new file mode 100644 index 000000000..990bfd157 --- /dev/null +++ b/cvs/lib/inference/unittests/test_atom_parsing.py @@ -0,0 +1,74 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +import unittest + +from cvs.lib.inference.atom.atom_parsing import ( + CLIENT_METRICS, + ENFORCED_METRICS, + GATED_METRICS, + METRIC_TIERS, + tier_metric_specs, +) + + +class TestATOMAtomParsing(unittest.TestCase): + def test_gated_metrics_include_w1_extras(self): + for name in ("per_gpu_throughput", "output_tput_per_gpu", "p99_tpot_ms", "p99_ttft_ms"): + self.assertIn(name, GATED_METRICS) + + def test_enforced_metrics_cover_all_tiers(self): + tiered = {m for names in METRIC_TIERS.values() for m in names} + self.assertEqual(ENFORCED_METRICS, frozenset(tiered)) + + def test_tier_metric_specs_throughput(self): + cell = { + "client.output_throughput": {"kind": "min_tok_s", "value": 1}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 2}, + } + specs = tier_metric_specs(cell, "throughput") + self.assertIn("client.output_throughput", specs) + self.assertNotIn("client.mean_ttft_ms", specs) + + def test_tier_metric_specs_tpot_uses_p99_tail(self): + cell = { + "client.mean_tpot_ms": {"kind": "max_ms", "value": 46.8}, + "client.p99_tpot_ms": {"kind": "max_ms", "value": 51.36}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 53.76}, + } + specs = tier_metric_specs(cell, "tpot") + self.assertIn("client.p99_tpot_ms", specs) + self.assertNotIn("client.p95_tpot_ms", specs) + + def test_tier_metric_specs_record_includes_non_tiered(self): + cell = { + "client.median_ttft_ms": {"kind": "max_ms", "value": 9}, + "client.output_throughput": {"kind": "min_tok_s", "value": 1}, + } + specs = tier_metric_specs(cell, "record") + self.assertIn("client.median_ttft_ms", specs) + self.assertNotIn("client.output_throughput", specs) + + def test_tier_metric_specs_scaling(self): + cell = { + "scaling.efficiency_pct": {"kind": "min", "value": 50}, + "client.output_throughput": {"kind": "min_tok_s", "value": 1}, + } + specs = tier_metric_specs(cell, "scaling") + self.assertEqual(specs, {"scaling.efficiency_pct": {"kind": "min", "value": 50}}) + + def test_gated_metrics_subset_of_client_metrics(self): + client_short = {short for short, _unit in CLIENT_METRICS} + missing = GATED_METRICS - client_short + self.assertEqual(missing, set(), f"GATED_METRICS not in CLIENT_METRICS: {missing}") + + def test_health_tier_metrics_in_enforced_set(self): + for name in ("success_rate", "failed"): + self.assertIn(name, ENFORCED_METRICS) + self.assertIn(name, METRIC_TIERS["health"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_atom_server_reuse.py b/cvs/lib/inference/unittests/test_atom_server_reuse.py new file mode 100644 index 000000000..227d5b6fb --- /dev/null +++ b/cvs/lib/inference/unittests/test_atom_server_reuse.py @@ -0,0 +1,92 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for ATOM server-reuse helpers and sweep parametrization. +''' + +import unittest +from types import SimpleNamespace + +from cvs.lib.inference.atom.atom_config_loader import ( + expand_sweep_parametrize, + reuse_server_flag, + server_session_key, +) +from cvs.lib.inference.atom.atom_parsing import METRIC_TIER_ORDER + + +def _session_key_variant(*, model_id="model-a", **params_kw): + params = { + "driver": "atom", + "tensor_parallelism": "8", + "nnodes": "1", + "pipeline_parallel_size": "1", + "master_addr": "", + "master_port": "29501", + } + params.update(params_kw) + return SimpleNamespace( + model=SimpleNamespace(id=model_id), + params=SimpleNamespace(**params), + roles=SimpleNamespace(server=SimpleNamespace(atom_args=("-tp", "8"))), + ) + + +class TestServerReuseHelpers(unittest.TestCase): + def test_reuse_server_flag_truthy_values(self): + for raw in ("true", "1", "yes", "TRUE", " Yes "): + params = SimpleNamespace(reuse_server_across_sweep=raw) + self.assertTrue(reuse_server_flag(params), raw) + + def test_reuse_server_flag_falsey_values(self): + for raw in ("false", "0", "no", "", "maybe"): + params = SimpleNamespace(reuse_server_across_sweep=raw) + self.assertFalse(reuse_server_flag(params), raw) + + def test_reuse_server_flag_defaults_false_when_missing(self): + self.assertFalse(reuse_server_flag(SimpleNamespace())) + + def test_server_session_key_differs_for_model(self): + base = _session_key_variant(model_id="model-a") + other = _session_key_variant(model_id="model-b") + k1 = server_session_key(base, "1024", "1024") + k2 = server_session_key(other, "1024", "1024") + self.assertNotEqual(k1, k2) + + def test_server_session_key_differs_for_shape(self): + variant = _session_key_variant() + self.assertNotEqual( + server_session_key(variant, "1024", "1024"), + server_session_key(variant, "2048", "2048"), + ) + + +class TestExpandSweepParametrize(unittest.TestCase): + def test_metric_tier_expansion_multiplies_cases(self): + sweep = { + "sequence_combinations": [{"name": "w1_1k_1k", "isl": "1024", "osl": "1024"}], + "runs": [{"combo": "w1_1k_1k", "concurrency": 128}], + } + spec = expand_sweep_parametrize(sweep, ("metric_tier",)) + argnames, argvalues, ids = spec + self.assertEqual(argnames, "seq_combo,concurrency,metric_tier") + self.assertEqual(len(argvalues), len(METRIC_TIER_ORDER)) + self.assertEqual(len(ids), len(METRIC_TIER_ORDER)) + self.assertEqual(ids[0], "w1_1k_1k-conc128-throughput") + + def test_inference_only_parametrize_without_metric_tier(self): + sweep = { + "sequence_combinations": [{"name": "w1_1k_1k", "isl": "1024", "osl": "1024"}], + "runs": [ + {"combo": "w1_1k_1k", "concurrency": 128}, + {"combo": "w1_1k_1k", "concurrency": 256}, + ], + } + _, argvalues, ids = expand_sweep_parametrize(sweep, ("seq_combo", "concurrency")) + self.assertEqual(len(argvalues), 2) + self.assertEqual(ids, ["w1_1k_1k-conc128", "w1_1k_1k-conc256"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_inference_suite_lifecycle.py b/cvs/lib/inference/unittests/test_inference_suite_lifecycle.py new file mode 100644 index 000000000..c9c914a83 --- /dev/null +++ b/cvs/lib/inference/unittests/test_inference_suite_lifecycle.py @@ -0,0 +1,29 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for inference_suite_lifecycle helpers. +''' + +import unittest + +from cvs.lib.inference.utils.cache_probe import du_bytes +from cvs.lib.inference.unittests.fake_orch import FakeOrch + + +class TestDuBytes(unittest.TestCase): + def test_sums_bytes_across_hosts(self): + orch = FakeOrch(exec_return={"node0": "1000", "node1": "2000"}) + self.assertEqual(du_bytes(orch, "/models"), 3000) + + def test_missing_path_returns_zero(self): + orch = FakeOrch(exec_return={"node0": "__MISSING__"}) + self.assertEqual(du_bytes(orch, "/models"), 0) + + def test_du_error_returns_none(self): + orch = FakeOrch(exec_return={"node0": "__DU_ERROR__"}) + self.assertIsNone(du_bytes(orch, "/models")) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_inferencing_config_loader.py b/cvs/lib/inference/unittests/test_inferencing_config_loader.py new file mode 100644 index 000000000..0e3dc67f2 --- /dev/null +++ b/cvs/lib/inference/unittests/test_inferencing_config_loader.py @@ -0,0 +1,407 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.utils.config_loader (ModelSpec, BaseVariantConfig, +substitute_config) and cvs.lib.inference.utils.inferencing_config_loader +(Sweep, SeqCombo, GoodputSlo, Run, VariantConfig.expected_cells, +_check_thresholds_cover_sweep). No hardware. +''' + +import unittest +import warnings + +from pydantic import ValidationError + +from cvs.lib.inference.utils.inferencing_config_loader import ( + GoodputSlo, + Run, + SeqCombo, + Sweep, + VariantConfig, +) +from cvs.lib.utils.config_loader import ModelSpec +from cvs.lib.inference.utils.vllm_parsing import GATED_METRICS + + +def _combo(name, isl="128", osl="2048"): + return SeqCombo(name=name, isl=isl, osl=osl) + + +def _full_gated_specs(): + """A spec for every gated metric -- the minimum that satisfies coverage. + + Values are inert (a 0 floor / huge ceiling) so the set passes without + asserting anything; these tests pin the coverage gate, not the numbers. + """ + out = {} + for m in GATED_METRICS: + kind = "max_ms" if m.endswith("_ms") else "max" if m == "failed" else "min" + out[f"client.{m}"] = {"kind": kind, "value": 0 if kind == "min" else 1e12} + return out + + +def _variant(sweep, tp="8", thresholds=None, enforce_thresholds=False): + """A minimal VariantConfig carrying just enough to exercise expected_cells. + + remote=0 (the remote guard would otherwise reject it) and + enforce_thresholds=False so the empty threshold dict does not trip the + coverage check -- this test pins the selector expansion, not the gate. + """ + return VariantConfig( + schema_version=1, + framework="vllm_single", + gpu_arch="mi300x", + enforce_thresholds=enforce_thresholds, + threshold_json="", + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "amd/Llama-3.1-70B-Instruct-FP8-KV", "remote": 0}, + container={ + "name": "c", + "image": "rocm/vllm-dev:nightly-sshd", + "runtime": {"name": "docker"}, + }, + params={"tensor_parallelism": tp}, + sweep=sweep, + thresholds=thresholds or {}, + ) + + +class TestSweepValidator(unittest.TestCase): + def test_valid_runs_selector_constructs(self): + sw = Sweep( + sequence_combinations=[_combo("a"), _combo("b", osl="4096")], + runs=[Run(combo="a", concurrency=16), Run(combo="b", concurrency=32)], + ) + self.assertEqual([r.combo for r in sw.runs], ["a", "b"]) + + def test_unknown_run_combo_raises(self): + with self.assertRaises(ValidationError) as ctx: + Sweep( + sequence_combinations=[_combo("a")], + runs=[Run(combo="typo", concurrency=16)], + ) + self.assertIn("names no sequence_combination", str(ctx.exception)) + + def test_duplicate_combo_names_raise(self): + with self.assertRaises(ValidationError) as ctx: + Sweep( + sequence_combinations=[_combo("a"), _combo("a", osl="4096")], + runs=[Run(combo="a", concurrency=16)], + ) + self.assertIn("duplicate sequence_combination names", str(ctx.exception)) + + def test_concurrency_levels_is_rejected(self): + # The old cartesian key must be gone (extra=forbid): a config still + # carrying concurrency_levels should fail loudly, not silently ignore it. + with self.assertRaises(ValidationError): + Sweep( + sequence_combinations=[_combo("a")], + runs=[Run(combo="a", concurrency=16)], + concurrency_levels=[16], + ) + + +class TestExpectedCells(unittest.TestCase): + def test_runs_expand_to_exactly_their_cells(self): + sw = Sweep( + sequence_combinations=[_combo("a", isl="128", osl="2048"), _combo("b", isl="256", osl="4096")], + runs=[ + Run(combo="a", concurrency=16), + Run(combo="b", concurrency=32), + Run(combo="a", concurrency=64), + ], + ) + vc = _variant(sw) + self.assertEqual( + vc.expected_cells(), + [ + "ISL=128,OSL=2048,TP=8,CONC=16", + "ISL=256,OSL=4096,TP=8,CONC=32", + "ISL=128,OSL=2048,TP=8,CONC=64", + ], + ) + + def test_no_cartesian_blowup(self): + # Two combos + two runs must yield TWO cells, not 2x2=4 (the old bug). + sw = Sweep( + sequence_combinations=[_combo("a"), _combo("b", osl="4096")], + runs=[Run(combo="a", concurrency=16), Run(combo="b", concurrency=16)], + ) + self.assertEqual(len(_variant(sw).expected_cells()), 2) + + +class TestGatedMetricCoverage(unittest.TestCase): + """The gated-metric axis of _check_thresholds_cover_sweep.""" + + _CELL = "ISL=128,OSL=2048,TP=8,CONC=16" + + def _variant_with(self, thresholds, enforce): + sw = Sweep( + sequence_combinations=[_combo("a")], + runs=[Run(combo="a", concurrency=16)], + ) + return VariantConfig( + schema_version=1, + framework="vllm_single", + gpu_arch="mi300x", + enforce_thresholds=enforce, + threshold_json="", + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "amd/Llama-3.1-70B-Instruct-FP8-KV", "remote": 0}, + container={"name": "c", "image": "rocm/vllm-dev:nightly-sshd", "runtime": {"name": "docker"}}, + params={"tensor_parallelism": "8"}, + sweep=sw, + thresholds=thresholds, + ) + + def test_full_gated_set_constructs(self): + vc = self._variant_with({self._CELL: _full_gated_specs()}, enforce=True) + self.assertEqual(vc.enforce_thresholds, True) + + def test_missing_gated_metric_raises_when_enforced(self): + specs = _full_gated_specs() + del specs["client.p99_ttft_ms"] # drop one gated metric + with self.assertRaises(ValidationError) as ctx: + self._variant_with({self._CELL: specs}, enforce=True) + self.assertIn("missing gated-metric specs", str(ctx.exception)) + self.assertIn("client.p99_ttft_ms", str(ctx.exception)) + + def test_missing_gated_metric_warns_when_record_only(self): + specs = _full_gated_specs() + del specs["client.failed"] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self._variant_with({self._CELL: specs}, enforce=False) + self.assertTrue(any("missing gated-metric specs" in str(x.message) for x in caught)) + + def test_extra_non_gated_spec_is_allowed(self): + # A spec for a non-gated metric (record-only display extra) must not + # trip coverage -- gating is a floor, not an allow-list. + specs = _full_gated_specs() + specs["client.num_prompts"] = {"kind": "min", "value": 0} + vc = self._variant_with({self._CELL: specs}, enforce=True) + self.assertIn("client.num_prompts", vc.thresholds[self._CELL]) + + +class TestModelSpecPrecision(unittest.TestCase): + """precision is an accepted optional field on ModelSpec (default '').""" + + def test_precision_field_is_accepted(self): + ms = ModelSpec(id="amd/Llama-3.1-70B", remote=0, precision="fp8") + self.assertEqual(ms.precision, "fp8") + + def test_valid_model_spec_without_precision(self): + ms = ModelSpec(id="amd/Llama-3.1-70B", remote=0) + self.assertEqual(ms.id, "amd/Llama-3.1-70B") + self.assertEqual(ms.remote, 0) + # precision is optional and defaults to empty. + self.assertEqual(ms.precision, "") + + def test_unknown_field_is_rejected(self): + # ModelSpec is _Forbid: a truly unknown field still fails validation. + with self.assertRaises(ValidationError): + ModelSpec(id="amd/Llama-3.1-70B", remote=0, bogus="x") + + +class TestThresholdJsonField(unittest.TestCase): + """threshold_json is an optional field on BaseVariantConfig / VariantConfig + (default ''); when absent, threshold discovery falls back to the sibling + *threshold.json next to the config.""" + + def _base_kwargs(self): + sw = Sweep( + sequence_combinations=[_combo("a")], + runs=[Run(combo="a", concurrency=16)], + ) + return dict( + schema_version=1, + framework="vllm_single", + gpu_arch="mi300x", + enforce_thresholds=False, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "amd/Llama-3.1-70B", "remote": 0}, + container={"name": "c", "image": "img", "runtime": {"name": "docker"}}, + params={"tensor_parallelism": "8"}, + sweep=sw, + thresholds={}, + ) + + def test_missing_threshold_json_defaults_to_empty(self): + kwargs = self._base_kwargs() + # threshold_json deliberately absent -> optional, defaults to "". + vc = VariantConfig(**kwargs) + self.assertEqual(vc.threshold_json, "") + + def test_threshold_json_present_constructs(self): + kwargs = self._base_kwargs() + kwargs["threshold_json"] = "/some/absolute/path/threshold.json" + vc = VariantConfig(**kwargs) + self.assertEqual(vc.threshold_json, "/some/absolute/path/threshold.json") + + +class TestCellCoverageAxis(unittest.TestCase): + """Axis-1 of _check_thresholds_cover_sweep: cell vs threshold key mismatch.""" + + _CELL = "ISL=128,OSL=2048,TP=8,CONC=16" + + def _variant_with(self, thresholds, enforce=True): + sw = Sweep( + sequence_combinations=[_combo("a")], + runs=[Run(combo="a", concurrency=16)], + ) + return VariantConfig( + schema_version=1, + framework="vllm_single", + gpu_arch="mi300x", + enforce_thresholds=enforce, + threshold_json="", + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "amd/Llama-3.1-70B", "remote": 0}, + container={"name": "c", "image": "img", "runtime": {"name": "docker"}}, + params={"tensor_parallelism": "8"}, + sweep=sw, + thresholds=thresholds, + ) + + def test_sweep_cell_with_no_threshold_entry_raises(self): + # thresholds is empty -> sweep cell has no entry -> axis-1 fires + with self.assertRaises(ValidationError) as ctx: + self._variant_with(thresholds={}, enforce=True) + self.assertIn("sweep cells with no threshold entry", str(ctx.exception)) + self.assertIn(self._CELL, str(ctx.exception)) + + def test_threshold_key_matching_no_sweep_cell_raises(self): + # thresholds has the real cell PLUS a bogus key -> extra set is non-empty + specs = _full_gated_specs() + with self.assertRaises(ValidationError) as ctx: + self._variant_with( + thresholds={self._CELL: specs, "ISL=999,OSL=999,TP=8,CONC=99": specs}, + enforce=True, + ) + self.assertIn("threshold keys matching no sweep cell", str(ctx.exception)) + self.assertIn("ISL=999,OSL=999,TP=8,CONC=99", str(ctx.exception)) + + def test_cell_mismatch_warns_when_record_only(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self._variant_with(thresholds={}, enforce=False) + self.assertTrue(any("sweep cells with no threshold entry" in str(w.message) for w in caught)) + + def test_accuracy_key_does_not_trip_extra_key_check(self): + # "accuracy" is a top-level threshold key for lm-eval gating, not a + # sweep cell -- it must not be flagged as an unrecognized extra key. + specs = _full_gated_specs() + vc = self._variant_with( + thresholds={self._CELL: specs, "accuracy": {"mmlu": {"mmlu.acc__none": {"kind": "min", "value": 0.5}}}}, + enforce=True, + ) + self.assertIn("accuracy", vc.thresholds) + + def test_unrecognized_key_still_raises_alongside_accuracy(self): + # The "accuracy" exclusion must be narrow: a genuinely unrecognized + # key (typo'd or bogus) alongside a valid "accuracy" block still trips + # the extra-key check. + specs = _full_gated_specs() + with self.assertRaises(ValidationError) as ctx: + self._variant_with( + thresholds={ + self._CELL: specs, + "accuracy": {"mmlu": {"mmlu.acc__none": {"kind": "min", "value": 0.5}}}, + "acuracy": {}, + }, + enforce=True, + ) + self.assertIn("threshold keys matching no sweep cell", str(ctx.exception)) + self.assertIn("acuracy", str(ctx.exception)) + + +class TestExpectedCellsBoundaries(unittest.TestCase): + """Boundary cases for VariantConfig.expected_cells.""" + + def test_empty_runs_yields_empty_cells(self): + sw = Sweep(sequence_combinations=[_combo("a")], runs=[]) + self.assertEqual(_variant(sw).expected_cells(), []) + + def test_unreferenced_combo_not_in_expected_cells(self): + # combo 'unused' is declared but never referenced by any run + sw = Sweep( + sequence_combinations=[_combo("a"), _combo("unused", isl="999", osl="999")], + runs=[Run(combo="a", concurrency=16)], + ) + cells = _variant(sw).expected_cells() + self.assertEqual(len(cells), 1) + self.assertNotIn("ISL=999", cells[0]) + + +class TestGoodputSlo(unittest.TestCase): + """GoodputSlo is a _Forbid model with three required float fields.""" + + def test_valid_goodput_slo_constructs(self): + slo = GoodputSlo(ttft_ms=100.0, tpot_ms=50.0, e2el_ms=5000.0) + self.assertEqual(slo.ttft_ms, 100.0) + self.assertEqual(slo.tpot_ms, 50.0) + self.assertEqual(slo.e2el_ms, 5000.0) + + def test_missing_required_field_raises(self): + for missing in ("ttft_ms", "tpot_ms", "e2el_ms"): + with self.subTest(missing=missing): + kwargs = {"ttft_ms": 1.0, "tpot_ms": 1.0, "e2el_ms": 1.0} + del kwargs[missing] + with self.assertRaises(ValidationError): + GoodputSlo(**kwargs) + + def test_extra_key_raises(self): + with self.assertRaises(ValidationError): + GoodputSlo(ttft_ms=1.0, tpot_ms=1.0, e2el_ms=1.0, ttft_msec=1.0) + + def test_seq_combo_with_goodput_slo(self): + slo = GoodputSlo(ttft_ms=1000.0, tpot_ms=50.0, e2el_ms=10000.0) + combo = SeqCombo(name="a", isl="128", osl="2048", goodput_slo=slo) + self.assertIsNotNone(combo.goodput_slo) + self.assertEqual(combo.goodput_slo.e2el_ms, 10000.0) + + def test_seq_combo_without_goodput_slo(self): + combo = SeqCombo(name="a", isl="128", osl="2048") + self.assertIsNone(combo.goodput_slo) + + +class TestSeqComboForbid(unittest.TestCase): + """SeqCombo is _Forbid: missing required fields and extra keys must raise.""" + + def test_missing_required_field_raises(self): + for missing in ("name", "isl", "osl"): + with self.subTest(missing=missing): + kwargs = {"name": "a", "isl": "128", "osl": "2048"} + del kwargs[missing] + with self.assertRaises(ValidationError): + SeqCombo(**kwargs) + + def test_extra_key_raises(self): + with self.assertRaises(ValidationError): + SeqCombo(name="a", isl="128", osl="2048", unknown_field="x") + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_lm_eval_job.py b/cvs/lib/inference/unittests/test_lm_eval_job.py new file mode 100644 index 000000000..8e575c50c --- /dev/null +++ b/cvs/lib/inference/unittests/test_lm_eval_job.py @@ -0,0 +1,337 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.inference.utils.lm_eval_job (build_lm_eval_cmd, run_accuracy_tasks). + +build_lm_eval_cmd is a PURE function (no I/O); run_accuracy_tasks is +orchestration-dependent and is tested via a FakeOrch test double, following the +FakeOrch pattern established in +cvs/lib/inference/unittests/test_vllm_job_server_reuse.py. +''' + +import copy +import json +import unittest + +from cvs.lib.inference.utils.accuracy_config import AccuracyTask +from cvs.lib.inference.utils.lm_eval_job import ( + LM_EVAL_INSTALL_CHECK_CMD, + LmEvalCtx, + build_lm_eval_cmd, + run_accuracy_tasks, +) + + +def _task(**overrides): + defaults = dict(id="mmlu", task="mmlu") + defaults.update(overrides) + return AccuracyTask(**defaults) + + +def _ctx(**overrides): + defaults = dict( + base_url="http://127.0.0.1:8000", + model_id="meta-llama/Llama-3-8b", + model_path="/data/models/Llama-3-8b", + output_dir="/tmp/accuracy-out", + ) + defaults.update(overrides) + return LmEvalCtx(**defaults) + + +class TestBuildLmEvalCmd(unittest.TestCase): + def test_env_script_sourced_before_install_guard(self): + # HF_HUB_CACHE/HF_TOKEN are written to /tmp/server_env_script.sh by + # server setup (see vllm_job.build_server_cmd) and are NOT inherited + # across separate exec_on_head invocations -- lm_eval must source + # that script itself, matching VllmJob's client-launch convention. + cmd = build_lm_eval_cmd(_task(), _ctx()) + self.assertTrue(cmd.startswith("source /tmp/server_env_script.sh && " + LM_EVAL_INSTALL_CHECK_CMD + " && ")) + + def test_default_single_task_shape(self): + cmd = build_lm_eval_cmd(_task(), _ctx()) + self.assertIn("lm_eval", cmd) + self.assertIn("--model local-completions", cmd) + self.assertIn( + "--model_args base_url=http://127.0.0.1:8000/v1/completions," + "model=meta-llama/Llama-3-8b,tokenizer=/data/models/Llama-3-8b," + "tokenizer_backend=huggingface,num_concurrent=8,max_retries=3," + "trust_remote_code=True", + cmd, + ) + self.assertIn("--tasks mmlu", cmd) + self.assertIn("--num_fewshot 0", cmd) + self.assertIn("--output_path /tmp/accuracy-out/mmlu", cmd) + self.assertIn("--log_samples", cmd) + + def test_trust_remote_code_always_present(self): + # Models with custom tokenizer code (Qwen, ChatGLM, Phi, MPT, ...) + # fail to load without this -- unconditional since it's a no-op for + # models that don't need it. + cmd = build_lm_eval_cmd(_task(), _ctx()) + self.assertIn("trust_remote_code=True", cmd) + + def test_apply_chat_template_switches_model_flag(self): + cmd = build_lm_eval_cmd(_task(apply_chat_template=True), _ctx()) + self.assertIn("--model local-chat-completions", cmd) + self.assertNotIn("--model local-completions", cmd) + # chat-template tasks must hit the chat endpoint, not /v1/completions. + self.assertIn("base_url=http://127.0.0.1:8000/v1/chat/completions", cmd) + self.assertNotIn("base_url=http://127.0.0.1:8000/v1/completions,", cmd) + + def test_apply_chat_template_passes_lm_eval_flag(self): + # local-chat-completions requires lm-eval's own --apply_chat_template + # flag to format prompts as messages=list[dict]; without it lm-eval + # sends a plain string and the chat-completions client asserts. + cmd = build_lm_eval_cmd(_task(apply_chat_template=True), _ctx()) + self.assertIn("--apply_chat_template", cmd) + + def test_default_task_uses_completions_endpoint(self): + cmd = build_lm_eval_cmd(_task(apply_chat_template=False), _ctx()) + self.assertIn("base_url=http://127.0.0.1:8000/v1/completions", cmd) + self.assertNotIn("--apply_chat_template", cmd) + + def test_num_fewshot_and_num_concurrent_reflected(self): + cmd = build_lm_eval_cmd(_task(num_fewshot=5, num_concurrent=32), _ctx()) + self.assertIn("--num_fewshot 5", cmd) + self.assertIn("num_concurrent=32", cmd) + + def test_metadata_json_encoded_when_present(self): + cmd = build_lm_eval_cmd(_task(metadata={"seq_len": 4096}), _ctx()) + self.assertIn("--metadata", cmd) + self.assertIn(json.dumps({"seq_len": 4096}), cmd) + + def test_metadata_omitted_when_empty(self): + cmd = build_lm_eval_cmd(_task(metadata={}), _ctx()) + self.assertNotIn("--metadata", cmd) + + def test_include_path_included_when_nonempty(self): + cmd = build_lm_eval_cmd(_task(include_path="/opt/custom_tasks"), _ctx()) + self.assertIn("--include_path /opt/custom_tasks", cmd) + + def test_include_path_omitted_when_empty_string(self): + cmd = build_lm_eval_cmd(_task(include_path=""), _ctx()) + self.assertNotIn("--include_path", cmd) + + def test_gen_kwargs_comma_joined_and_insertion_order_preserved(self): + cmd = build_lm_eval_cmd( + _task(gen_kwargs={"temperature": 0, "max_gen_toks": 128, "top_p": 0.9}), + _ctx(), + ) + self.assertIn("--gen_kwargs temperature=0,max_gen_toks=128,top_p=0.9", cmd) + + def test_gen_kwargs_omitted_when_empty(self): + cmd = build_lm_eval_cmd(_task(gen_kwargs={}), _ctx()) + self.assertNotIn("--gen_kwargs", cmd) + + def test_shell_quoting_safety_for_special_characters(self): + task = _task( + id="weird id", + task="weird task", + include_path="/path with spaces/tasks", + gen_kwargs={"stop": "a b\"c"}, + ) + ctx = _ctx(output_dir="/tmp/out dir") + cmd = build_lm_eval_cmd(task, ctx) + # Command must be shell-parseable without raising, and round-trip the + # exact values through shlex (proves quoting, not just substring presence). + import shlex as _shlex + + parts = _shlex.split(cmd) + self.assertIn("weird task", parts) + self.assertIn("/tmp/out dir/weird id", parts) + self.assertIn("/path with spaces/tasks", parts) + self.assertIn('stop=a b"c', parts) + + def test_does_not_mutate_task_or_ctx(self): + task = _task(metadata={"a": 1}, gen_kwargs={"b": 2}) + ctx = _ctx() + task_before = copy.deepcopy(task) + ctx_before = copy.deepcopy(ctx) + build_lm_eval_cmd(task, ctx) + self.assertEqual(task, task_before) + self.assertEqual(ctx, ctx_before) + + def test_returns_single_string_not_list(self): + cmd = build_lm_eval_cmd(_task(), _ctx()) + self.assertIsInstance(cmd, str) + + +class TestInstallGuard(unittest.TestCase): + '''The install guard must cover the `math` extra and detect it by capability. + + leaderboard_math_hard imports math_verify at task-build time; the `api` + extra alone omits it, so the task dies with ModuleNotFoundError after the + server is already up (observed on GLM-5.2, 2026-07-31). + ''' + + def test_installs_math_extra(self): + self.assertIn("lm-eval[api,math]", LM_EVAL_INSTALL_CHECK_CMD) + + def test_guard_probes_math_verify_import_not_just_lm_eval_presence(self): + # A `pip list | grep lm_eval` guard short-circuits on an image that + # preinstalls bare lm-eval, silently skipping the math extra. Probing + # the import instead fails closed. + self.assertIn("import lm_eval, math_verify", LM_EVAL_INSTALL_CHECK_CMD) + self.assertNotIn("pip list", LM_EVAL_INSTALL_CHECK_CMD) + + def test_install_still_runs_only_when_probe_fails(self): + self.assertIn("||", LM_EVAL_INSTALL_CHECK_CMD) + self.assertIn("pip install", LM_EVAL_INSTALL_CHECK_CMD) + + +class FakeOrch: + """Head-only orch test double: records commands, returns queued responses. + + The first exec_on_head call (the lm_eval run itself) is made with + detailed=True and expects a {'output': ..., 'exit_code': ...} response; + responses for that call may be given as a bare string (wrapped here with + exit_code=0) or as an explicit dict to simulate a non-zero exit. + """ + + def __init__(self, responses=None): + self.head_cmds = [] + self.head_kwargs = [] + self._responses = list(responses or []) + + def exec_on_head(self, cmd, *a, **k): + self.head_cmds.append(cmd) + self.head_kwargs.append(k) + if self._responses: + response = self._responses.pop(0) + else: + response = "" if not k.get("detailed") else {"output": "", "exit_code": 0} + if k.get("detailed") and not isinstance(response, dict): + response = {"output": response, "exit_code": 0} + return {"10.0.0.1": response} + + +class TestRunAccuracyTasks(unittest.TestCase): + def _run_kwargs(self, orch, tasks): + return dict( + orch=orch, + tasks=tasks, + base_url="http://127.0.0.1:8000", + model_id="meta-llama/Llama-3-8b", + model_path="/data/models/Llama-3-8b", + output_dir="/tmp/accuracy-out", + ) + + def test_single_task_success_id_keyed_and_projected(self): + payload = {"results": {"mmlu": {"acc,none": 0.5, "alias": "mmlu"}}} + orch = FakeOrch( + responses=[ + "", # lm_eval run output + "1700000000.123456 /tmp/accuracy-out/mmlu/model/results_2025.json", # find + json.dumps(payload), # cat + ] + ) + out = run_accuracy_tasks(**self._run_kwargs(orch, [_task()])) + self.assertEqual(out, {"mmlu": {"mmlu.acc__none": 0.5}}) + + def test_multiple_tasks_each_contribute_own_dict(self): + orch = FakeOrch( + responses=[ + "", + "1700000000.0 /out/mmlu/model/results.json", + json.dumps({"results": {"mmlu": {"acc,none": 0.5}}}), + "", + "1700000001.0 /out/gsm8k/model/results.json", + json.dumps({"results": {"gsm8k": {"acc,none": 0.7}}}), + ] + ) + tasks = [_task(id="mmlu", task="mmlu"), _task(id="gsm8k", task="gsm8k")] + out = run_accuracy_tasks(**self._run_kwargs(orch, tasks)) + self.assertEqual( + out, + {"mmlu": {"mmlu.acc__none": 0.5}, "gsm8k": {"gsm8k.acc__none": 0.7}}, + ) + + def test_missing_results_file_raises_runtime_error(self): + orch = FakeOrch(responses=["", ""]) # run output, then empty find output + with self.assertRaises(RuntimeError): + run_accuracy_tasks(**self._run_kwargs(orch, [_task()])) + + def test_exec_on_head_called_head_only_not_broadcast(self): + payload = {"results": {"mmlu": {"acc,none": 0.5}}} + orch = FakeOrch(responses=["", "1700000000.0 /out/mmlu/model/results.json", json.dumps(payload)]) + run_accuracy_tasks(**self._run_kwargs(orch, [_task()])) + # exactly 3 exec_on_head calls for a single task: run, find, cat. + self.assertEqual(len(orch.head_cmds), 3) + self.assertFalse(hasattr(orch, "exec")) + + def test_install_guard_present_in_executed_command(self): + payload = {"results": {"mmlu": {"acc,none": 0.5}}} + orch = FakeOrch(responses=["", "1700000000.0 /out/mmlu/model/results.json", json.dumps(payload)]) + run_accuracy_tasks(**self._run_kwargs(orch, [_task()])) + self.assertIn(LM_EVAL_INSTALL_CHECK_CMD, orch.head_cmds[0]) + + def test_picks_newest_result_when_multiple_present(self): + payload = {"results": {"mmlu": {"acc,none": 0.5}}} + orch = FakeOrch( + responses=[ + "", + # `find ... -printf '%T@ %p\n' | sort -rn` sorts newest-first on + # the real host -- FakeOrch can't execute the shell pipeline + # itself, so this fixture simulates the already-sorted output a + # real run would produce. + "1700000999.0 /out/mmlu/model/results_new.json\n1700000000.0 /out/mmlu/model/results_old.json", + json.dumps(payload), + ] + ) + run_accuracy_tasks(**self._run_kwargs(orch, [_task()])) + # find step's own command text must actually sort by mtime descending -- + # FakeOrch ignores command text when producing its response, so this + # assertion is the only thing that would catch a shell-logic regression + # (e.g. `sort -n` instead of `sort -rn`); the fixture above only proves + # the Python-side parsing of already-sorted output picks line 0. + find_cmd = orch.head_cmds[1] + self.assertIn("-printf", find_cmd) + self.assertIn("sort -rn", find_cmd) + # cat must be issued against the first (newest) line's path. + self.assertIn("results_new.json", orch.head_cmds[2]) + self.assertNotIn("results_old.json", orch.head_cmds[2]) + + def test_malformed_json_result_raises_runtime_error(self): + orch = FakeOrch(responses=["", "1700000000.0 /out/mmlu/model/results.json", "{not valid json"]) + with self.assertRaises(RuntimeError): + run_accuracy_tasks(**self._run_kwargs(orch, [_task()])) + + def test_none_run_output_does_not_crash_on_missing_result(self): + orch = FakeOrch(responses=[None, ""]) # run output is None, find is empty + with self.assertRaises(RuntimeError): + run_accuracy_tasks(**self._run_kwargs(orch, [_task()])) + + def test_nonzero_exit_code_raises_before_checking_for_results(self): + # A results*.json can exist on disk from a prior run even though this + # invocation of lm_eval itself failed -- exit_code must be checked + # before treating the run as successful, independent of file presence. + orch = FakeOrch(responses=[{"output": "traceback...", "exit_code": 1}]) + with self.assertRaises(RuntimeError) as ctx: + run_accuracy_tasks(**self._run_kwargs(orch, [_task()])) + self.assertIn("exited with code 1", str(ctx.exception)) + # must fail fast: no find/cat calls issued after a nonzero exit. + self.assertEqual(len(orch.head_cmds), 1) + + def test_run_invocation_requests_detailed_exit_code(self): + payload = {"results": {"mmlu": {"acc,none": 0.5}}} + orch = FakeOrch(responses=["", "1700000000.0 /out/mmlu/model/results.json", json.dumps(payload)]) + run_accuracy_tasks(**self._run_kwargs(orch, [_task()])) + self.assertTrue(orch.head_kwargs[0].get("detailed")) + + def test_zero_exit_code_with_dict_response_succeeds(self): + payload = {"results": {"mmlu": {"acc,none": 0.5}}} + orch = FakeOrch( + responses=[ + {"output": "", "exit_code": 0}, + "1700000000.0 /out/mmlu/model/results.json", + json.dumps(payload), + ] + ) + out = run_accuracy_tasks(**self._run_kwargs(orch, [_task()])) + self.assertEqual(out, {"mmlu": {"mmlu.acc__none": 0.5}}) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_lm_eval_parsing.py b/cvs/lib/inference/unittests/test_lm_eval_parsing.py new file mode 100644 index 000000000..9b896b7b5 --- /dev/null +++ b/cvs/lib/inference/unittests/test_lm_eval_parsing.py @@ -0,0 +1,355 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.inference.utils.lm_eval_parsing (_is_real_number, project). + +Both public units are PURE functions (no state, output depends only on args, no +I/O, no side effects). Per the authoring discipline this means: + - range/equivalence testing via a (input -> expected) subTest table, not N + copy-pasted methods; + - boundary cases as their own equivalence classes; + - an invariant test where one plausibly exists (return-type invariant for the + predicate; value-type / comma-free-key / determinism / non-mutation + invariants for the flattener). +There is no lifecycle class because neither unit carries mutable state. + +Tests are authored black-box from the behavioral spec only; the implementation +was not read. Written greenfield (RED before implementation). +''' + +import copy +import inspect +import typing +import unittest +from unittest.mock import patch + +from cvs.lib.inference.utils.lm_eval_parsing import _is_real_number, project + + +class TestIsRealNumber(unittest.TestCase): + """Pure predicate: any object -> bool (never raises). Spec AC 1-6, 22.""" + + def test_is_real_number_ranges(self): + # (value, expected) equivalence-class + boundary table. Spec AC 1-5, 22. + # Sentinel objects for nan/inf are constructed inline so identity is + # unambiguous. + cases = [ + # real numbers -> True + (1, True), # AC1 int + (1.5, True), # AC1 float + (0, True), # boundary: zero int is a real number + (0.0, True), # boundary: zero float is a real number + (-3, True), # AC22 negative int + (-1.25, True), # AC22 negative float + (float("inf"), True), # AC4 +inf is a real number + (float("-inf"), True), # AC4 -inf is a real number + # not real numbers -> False + (True, False), # AC2 bool excluded despite subclass of int + (False, False), # AC2 bool excluded + (float("nan"), False), # AC3 NaN excluded + ("0.71", False), # AC5 numeric-looking string excluded + (None, False), # AC5 None excluded + ({}, False), # AC5 dict excluded + ([], False), # AC5 list excluded + (complex(1, 2), False), # complex number is numeric but NOT real + (1 + 2j, False), # same boundary, literal form + (complex(3, 0), False), # zero-imaginary complex is still not real + ] + for value, expected in cases: + with self.subTest(value=repr(value)): + self.assertEqual(_is_real_number(value), expected) + + def test_is_real_number_always_returns_plain_bool(self): + # Invariant (AC6): result is a genuine bool, never a truthy/falsy + # non-bool. `type(...) is bool` is stricter than isinstance and would + # reject e.g. returning the int 0/1 or the object itself. + samples = [ + 1, + 1.5, + 0, + -3, + -1.25, + float("inf"), + float("-inf"), + float("nan"), + True, + False, + "0.71", + None, + {}, + [], + object(), + ] + for value in samples: + with self.subTest(value=repr(value)): + result = _is_real_number(value) + self.assertIs(type(result), bool) + + def test_is_real_number_bool_is_not_a_real_number(self): + # Regression constraint: bool is a subclass of int but must be excluded. + # Pinned separately from the table so a bool-passthrough mutant is killed + # explicitly. + self.assertIs(_is_real_number(True), False) + self.assertIs(_is_real_number(False), False) + + def test_is_real_number_nan_excluded_but_inf_included(self): + # Boundary between the two float special values (AC3 vs AC4). + self.assertIs(_is_real_number(float("nan")), False) + self.assertIs(_is_real_number(float("inf")), True) + self.assertIs(_is_real_number(float("-inf")), True) + + def test_is_real_number_complex_is_not_a_real_number(self): + # The function's whole purpose (its name) is real-vs-not-real: complex is + # the one unambiguously-numeric Python type that is NOT real. Pinned + # separately so a widened type check (e.g. numbers.Number/Complex instead + # of (int, float)) that admits complex values is killed explicitly. + self.assertIs(_is_real_number(complex(1, 2)), False) + self.assertIs(_is_real_number(1 + 2j), False) + # even a complex whose imaginary part is exactly zero is still complex, + # not real, and must be rejected on type, not value. + self.assertIs(_is_real_number(complex(3, 0)), False) + + +class TestProject(unittest.TestCase): + """Pure flattener: payload -> {'task.metric': float}. Spec AC 7-22.""" + + def test_project_flatten_cases(self): + # (payload, expected) table covering the enumerated spec behaviors. + # Expected dicts are asserted whole (structured-output assertion, not + # substring-in-blob). + cases = [ + # AC7: no 'results' key + ({}, {}), + # AC8: empty results dict + ({"results": {}}, {}), + # AC21: task whose value is an empty dict + ({"results": {"empty_task": {}}}, {}), + # AC9: only alias, no numeric metrics + ({"results": {"mmlu": {"alias": "mmlu"}}}, {}), + # AC10: single numeric metric alongside alias + ( + {"results": {"mmlu": {"acc,none": 0.71, "alias": "mmlu"}}}, + {"mmlu.acc__none": 0.71}, + ), + # Output Contract: only metric_key EXACTLY == 'alias' is excluded. + # A real-numeric metric whose key merely CONTAINS 'alias' as a + # substring (prefix/infix/suffix) must survive, while the literal + # 'alias' key alongside it is dropped. Distinguishes exact-key + # exclusion from a substring/startswith exclusion. + ( + { + "results": { + "t": { + "alias_score,none": 0.9, # prefix substring, survives + "task_alias": 0.8, # suffix substring, survives + "has_alias,none": 0.7, # infix substring, survives + "alias": "mmlu", # exact key, excluded + } + } + }, + { + "t.alias_score__none": 0.9, + "t.task_alias": 0.8, + "t.has_alias__none": 0.7, + }, + ), + # AC11: bool value excluded + ({"results": {"mmlu": {"acc,none": True}}}, {}), + # AC12: non-numeric string value excluded + ({"results": {"mmlu": {"acc,none": "not a number"}}}, {}), + # AC13: every comma in the metric key replaced (not just the first) + ( + {"results": {"t": {"a,b,c": 1.0}}}, + {"t.a__b__c": 1.0}, + ), + # metric key with zero commas is used as-is after the '.' join + ( + {"results": {"t": {"nocomma": 0.5}}}, + {"t.nocomma": 0.5}, + ), + # AC14: RULER-style numeric-string metric-key prefixes, two metrics + ( + {"results": {"niah_single_1": {"4096,none": 0.5, "32768,none": 0.9}}}, + {"niah_single_1.4096__none": 0.5, "niah_single_1.32768__none": 0.9}, + ), + # AC15: two tasks, multiple metrics each, no loss/merge across tasks + ( + { + "results": { + "mmlu": {"acc,none": 0.71, "alias": "mmlu"}, + "gsm8k": { + "exact_match,strict-match": 0.5, + "exact_match,flexible-extract": 0.6, + "alias": "gsm8k", + }, + } + }, + { + "mmlu.acc__none": 0.71, + "gsm8k.exact_match__strict-match": 0.5, + "gsm8k.exact_match__flexible-extract": 0.6, + }, + ), + # Boundary: metric value of exactly zero is a real, meaningful + # score (a fully-failing task) and must survive -- a truthiness- + # based short-circuit (`not value`) would silently drop it. The + # int 0 is coerced to the float 0.0. + ({"results": {"t": {"m,none": 0}}}, {"t.m__none": 0.0}), + # Boundary: float zero likewise survives and stays 0.0. + ({"results": {"t": {"m,none": 0.0}}}, {"t.m__none": 0.0}), + # AC17: int metric value coerced to float + ({"results": {"t": {"m": 3}}}, {"t.m": 3.0}), + # AC22: negative value preserved and coerced + ({"results": {"t": {"m,none": -1.25}}}, {"t.m__none": -1.25}), + # AC20: comma in the TASK name sanitized the same way + ( + {"results": {"group,4096": {"acc,none": 0.5}}}, + {"group__4096.acc__none": 0.5}, + ), + # AC19: sibling top-level keys ignored, incl. 'versions' number + ( + { + "results": {"mmlu": {"acc,none": 0.71, "alias": "mmlu"}}, + "versions": {"mmlu": 2}, + "configs": {"mmlu": {"task": "mmlu"}}, + "n-shot": {"mmlu": 5}, + "model_name": "meta-llama/Llama-3.1-70B", + }, + {"mmlu.acc__none": 0.71}, + ), + ] + for payload, expected in cases: + with self.subTest(payload=repr(payload)): + self.assertEqual(project(payload), expected) + + def test_project_infinity_included_nan_excluded(self): + # inf/-inf are real numbers and survive coercion; NaN is dropped. + result = project( + {"results": {"t": {"good,none": float("inf"), "bad,none": float("nan"), "neg,none": float("-inf")}}} + ) + self.assertEqual(set(result.keys()), {"t.good__none", "t.neg__none"}) + self.assertEqual(result["t.good__none"], float("inf")) + self.assertEqual(result["t.neg__none"], float("-inf")) + self.assertNotIn("t.bad__none", result) + + def test_project_alias_never_contributes(self): + # 'alias' key is excluded regardless of value type (str, number, etc.). + for alias_val in ("mmlu", 0.5, 7, None, {"x": 1}): + with self.subTest(alias=repr(alias_val)): + out = project({"results": {"t": {"alias": alias_val}}}) + self.assertEqual(out, {}) + + # --- invariants ------------------------------------------------------- + + def test_project_values_are_native_float(self): + # Invariant (AC17): every output value is a native float, even when the + # source was an int. isinstance(3, float) is False, so this distinguishes + # real coercion from a same-type passthrough. + out = project({"results": {"t": {"i,none": 3, "f,none": 0.71, "neg": -2}}}) + self.assertEqual(set(out.keys()), {"t.i__none", "t.f__none", "t.neg"}) + for key, val in out.items(): + with self.subTest(key=key): + self.assertIs(type(val), float) + # explicit int->float coercion pin (AC17) + self.assertEqual(out["t.i__none"], 3.0) + self.assertFalse(isinstance(3, float)) + + def test_project_keys_never_contain_commas(self): + # Invariant: no ',' survives in any produced key (from task or metric). + payload = { + "results": { + "group,4096": {"a,b,c": 1.0, "plain": 2.0}, + "gsm8k": {"exact_match,strict-match": 0.5}, + } + } + out = project(payload) + for key in out: + with self.subTest(key=key): + self.assertNotIn(",", key) + + def test_project_one_entry_per_real_numeric_metric(self): + # Invariant (AC15): output size equals the count of (task, real-numeric, + # non-alias) metric pairs -- nothing lost or merged across tasks. + payload = { + "results": { + "t1": {"a,none": 0.1, "b,none": 0.2, "alias": "t1"}, + "t2": {"c,none": 0.3, "bad": "x", "flag": True, "alias": "t2"}, + } + } + out = project(payload) + self.assertEqual(len(out), 3) + self.assertEqual( + set(out.keys()), + {"t1.a__none", "t1.b__none", "t2.c__none"}, + ) + + def test_project_is_deterministic(self): + # Invariant: pure function -> repeated calls on equal input yield equal + # output (and distinct dict objects each call, since a NEW dict is + # returned per contract). + payload = {"results": {"mmlu": {"acc,none": 0.71, "alias": "mmlu"}}} + first = project(payload) + second = project(payload) + self.assertEqual(first, second) + self.assertIsNot(first, second) + + def test_project_returns_new_empty_dict_not_none(self): + # Output contract: never returns None; empty case is a real {} dict. + out = project({}) + self.assertIsInstance(out, dict) + self.assertEqual(out, {}) + + def test_project_does_not_mutate_payload(self): + # AC16: in-contract payload (nested dicts, mixed value kinds) is not + # mutated -- deep-equality against a pre-call snapshot. + payload = { + "results": { + "mmlu": {"acc,none": 0.71, "alias": "mmlu"}, + "gsm8k": { + "exact_match,strict-match": 0.5, + "flag": True, + "note": "text", + }, + "empty": {}, + }, + "versions": {"mmlu": 2}, + "model_name": "x", + } + snapshot = copy.deepcopy(payload) + project(payload) + self.assertEqual(payload, snapshot) + + def test_project_performs_no_file_io(self): + # AC23: with builtins.open patched to raise, project still works -> + # proves the flattener performs no file I/O. + with patch("builtins.open", side_effect=AssertionError("no I/O allowed")): + out = project({"results": {"mmlu": {"acc,none": 0.71, "alias": "mmlu"}}}) + self.assertEqual(out, {"mmlu.acc__none": 0.71}) + self.assertIs(_is_real_number(1), True) + + +class TestSignaturePreservation(unittest.TestCase): + """AC24 / Regression Constraints: signatures + annotations are frozen.""" + + def test_is_real_number_type_hints(self): + self.assertEqual( + typing.get_type_hints(_is_real_number), + {"value": typing.Any, "return": bool}, + ) + + def test_project_type_hints(self): + self.assertEqual( + typing.get_type_hints(project), + {"payload": typing.Dict[str, typing.Any], "return": typing.Dict[str, float]}, + ) + + def test_is_real_number_parameter_names(self): + self.assertEqual(list(inspect.signature(_is_real_number).parameters), ["value"]) + + def test_project_parameter_names(self): + self.assertEqual(list(inspect.signature(project).parameters), ["payload"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_vllm_config_loader.py b/cvs/lib/inference/unittests/test_vllm_config_loader.py new file mode 100644 index 000000000..7a513fcf9 --- /dev/null +++ b/cvs/lib/inference/unittests/test_vllm_config_loader.py @@ -0,0 +1,167 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.inference.utils.vllm_config_loader's gpu.*/prom.* +threshold coverage in _check_thresholds_cover_sweep. No hardware. + +_check_thresholds_cover_sweep only requires that every sweep cell have a +threshold entry -- it never requires that entry name every gated metric. +Operators may gate only the metrics they care about; test_metric/ +test_gpu_metric/test_prom_metric already treat an absent spec as +"don't gate this metric" at evaluation time. +''' + +import unittest + +from cvs.lib.inference.utils.vllm_config_loader import ( + GATED_GPU_METRICS, + GATED_PROM_METRICS, + Run, + SeqCombo, + Sweep, + VariantConfig, +) +from cvs.lib.inference.utils.vllm_parsing import GATED_METRICS + + +def _combo(name, isl="128", osl="2048"): + return SeqCombo(name=name, isl=isl, osl=osl) + + +def _full_gated_specs(): + """A spec for every gated client.*, gpu.*, and prom.* metric -- the + minimum that satisfies coverage. Values are inert so the set passes + without asserting anything; these tests pin the coverage gate, not the + numbers.""" + out = {} + for m in GATED_METRICS: + kind = "max_ms" if m.endswith("_ms") else "max" if m == "failed" else "min" + out[f"client.{m}"] = {"kind": kind, "value": 0 if kind == "min" else 1e12} + for m in GATED_GPU_METRICS: + kind = "max" if m in ("peak_gpu_memory_mb", "model_load_memory_mb", "model_load_s") else "min" + out[f"gpu.{m}"] = {"kind": kind, "value": 0 if kind == "min" else 1e12} + for m in GATED_PROM_METRICS: + out[f"prom.{m}"] = {"kind": "max_ms", "value": 1e12} + return out + + +class TestGpuGatedMetricCoverage(unittest.TestCase): + """The gpu.* axis of vllm_config_loader's _check_thresholds_cover_sweep.""" + + _CELL = "ISL=128,OSL=2048,TP=8,CONC=16" + + def _variant_with(self, thresholds, enforce): + sw = Sweep( + sequence_combinations=[_combo("a")], + runs=[Run(combo="a", concurrency=16)], + ) + return VariantConfig( + schema_version=1, + framework="vllm", + gpu_arch="mi300x", + enforce_thresholds=enforce, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "amd/Llama-3.1-70B-Instruct-FP8-KV", "remote": 0}, + params={"tensor_parallelism": "8"}, + sweep=sw, + thresholds=thresholds, + ) + + def test_full_gated_set_constructs(self): + vc = self._variant_with({self._CELL: _full_gated_specs()}, enforce=True) + self.assertEqual(vc.enforce_thresholds, True) + + def test_missing_gpu_metric_does_not_raise_when_enforced(self): + # Operators may gate only a subset of gpu.* metrics; an absent one is + # simply not gated, not an authoring error. + specs = _full_gated_specs() + del specs["gpu.peak_gpu_memory_mb"] + vc = self._variant_with({self._CELL: specs}, enforce=True) + self.assertNotIn("gpu.peak_gpu_memory_mb", vc.thresholds[self._CELL]) + + def test_no_gpu_specs_at_all_does_not_raise_when_enforced(self): + vc = self._variant_with({self._CELL: {}}, enforce=True) + self.assertEqual(vc.thresholds[self._CELL], {}) + + def test_all_five_gpu_metrics_are_gated(self): + self.assertEqual( + GATED_GPU_METRICS, + { + "peak_gpu_memory_mb", + "model_load_memory_mb", + "model_load_s", + "gpu_bandwidth_util_pct", + "gpu_compute_util_pct", + }, + ) + + +class TestPromGatedMetricCoverage(unittest.TestCase): + """The prom.* axis of vllm_config_loader's _check_thresholds_cover_sweep. + + Mirrors TestGpuGatedMetricCoverage: prom.* is a fully separate, parallel + gated family, not part of client.*'s tiering machinery, so its coverage + is proven independently here rather than in test_vllm_report_preset.py. + """ + + _CELL = "ISL=128,OSL=2048,TP=8,CONC=16" + + def _variant_with(self, thresholds, enforce): + sw = Sweep( + sequence_combinations=[_combo("a")], + runs=[Run(combo="a", concurrency=16)], + ) + return VariantConfig( + schema_version=1, + framework="vllm", + gpu_arch="mi300x", + enforce_thresholds=enforce, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "amd/Llama-3.1-70B-Instruct-FP8-KV", "remote": 0}, + params={"tensor_parallelism": "8"}, + sweep=sw, + thresholds=thresholds, + ) + + def test_full_gated_set_constructs(self): + vc = self._variant_with({self._CELL: _full_gated_specs()}, enforce=True) + self.assertEqual(vc.enforce_thresholds, True) + + def test_missing_prom_metric_does_not_raise_when_enforced(self): + # Operators may gate only a subset of prom.* metrics; an absent one is + # simply not gated, not an authoring error. + specs = _full_gated_specs() + del specs["prom.queue_time_p50_ms"] + vc = self._variant_with({self._CELL: specs}, enforce=True) + self.assertNotIn("prom.queue_time_p50_ms", vc.thresholds[self._CELL]) + + def test_only_one_prom_metric_gated_does_not_raise_when_enforced(self): + specs = {"prom.queue_time_p50_ms": {"kind": "max_ms", "value": 200}} + vc = self._variant_with({self._CELL: specs}, enforce=True) + self.assertEqual(vc.thresholds[self._CELL], specs) + + def test_all_four_prom_metrics_are_gated(self): + self.assertEqual( + GATED_PROM_METRICS, + { + "queue_time_p50_ms", + "queue_time_p95_ms", + "prefill_time_p50_ms", + "prefill_time_p95_ms", + }, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_vllm_config_loader_accuracy.py b/cvs/lib/inference/unittests/test_vllm_config_loader_accuracy.py new file mode 100644 index 000000000..2429e428b --- /dev/null +++ b/cvs/lib/inference/unittests/test_vllm_config_loader_accuracy.py @@ -0,0 +1,113 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for the `accuracy` field wired onto vllm's VariantConfig (step 5 of +the accuracy-harness plan). AccuracyConfig itself is fully covered by +test_accuracy_config.py; these tests only pin the wiring: default/opt-in +behavior and pass-through of an explicit accuracy block. +''' + +import unittest + +from cvs.lib.inference.utils.accuracy_config import AccuracyConfig +from cvs.lib.inference.utils.vllm_config_loader import VariantConfig + + +def _base_kwargs(**overrides): + kwargs = dict( + schema_version=1, + framework="vllm", + gpu_arch="mi300x", + enforce_thresholds=False, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "/models/test-model", "remote": 0}, + sweep={ + "sequence_combinations": [{"name": "a", "isl": "1024", "osl": "1024"}], + "runs": [{"combo": "a", "concurrency": 16}], + }, + thresholds={}, + ) + kwargs.update(overrides) + return kwargs + + +class TestVariantConfigAccuracyField(unittest.TestCase): + def test_defaults_to_empty_accuracy_config_when_omitted(self): + vc = VariantConfig(**_base_kwargs()) + self.assertIsInstance(vc.accuracy, AccuracyConfig) + self.assertEqual(vc.accuracy.tasks, []) + + def test_accepts_explicit_accuracy_tasks(self): + vc = VariantConfig(**_base_kwargs(accuracy={"tasks": [{"id": "mmlu", "task": "mmlu", "num_fewshot": 5}]})) + self.assertEqual(len(vc.accuracy.tasks), 1) + self.assertEqual(vc.accuracy.tasks[0].id, "mmlu") + self.assertEqual(vc.accuracy.tasks[0].num_fewshot, 5) + + def test_duplicate_task_ids_rejected_through_variant_config(self): + with self.assertRaises(ValueError): + VariantConfig( + **_base_kwargs( + accuracy={ + "tasks": [ + {"id": "mmlu", "task": "mmlu"}, + {"id": "mmlu", "task": "mmlu"}, + ] + } + ) + ) + + +class TestAccuracyThresholdKeyDoesNotTripSweepCoverage(unittest.TestCase): + """The top-level "accuracy" threshold key must not be flagged as an + unrecognized sweep-cell key by _check_thresholds_cover_sweep, now that it + delegates to the shared validate_thresholds_cover_sweep.""" + + _CELL = "ISL=1024,OSL=1024,TP=8,CONC=16" + + def _full_gated_specs(self): + from cvs.lib.inference.utils.vllm_config_loader import GATED_GPU_METRICS + from cvs.lib.inference.utils.vllm_parsing import GATED_METRICS + + out = {} + for m in GATED_METRICS: + kind = "max_ms" if m.endswith("_ms") else "max" if m == "failed" else "min" + out[f"client.{m}"] = {"kind": kind, "value": 0 if kind == "min" else 1e12} + for m in GATED_GPU_METRICS: + out[f"gpu.{m}"] = {"kind": "min", "value": 0} + return out + + def test_accuracy_key_alongside_full_sweep_coverage_constructs(self): + vc = VariantConfig( + **_base_kwargs( + enforce_thresholds=True, + thresholds={ + self._CELL: self._full_gated_specs(), + "accuracy": {"mmlu": {"mmlu.acc__none": {"kind": "min", "value": 0.5}}}, + }, + ) + ) + self.assertIn("accuracy", vc.thresholds) + + def test_typo_key_alongside_accuracy_still_raises(self): + with self.assertRaises(ValueError) as ctx: + VariantConfig( + **_base_kwargs( + enforce_thresholds=True, + thresholds={ + self._CELL: self._full_gated_specs(), + "accuracy": {"mmlu": {"mmlu.acc__none": {"kind": "min", "value": 0.5}}}, + "acuracy": {}, + }, + ) + ) + self.assertIn("threshold keys matching no sweep cell", str(ctx.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_vllm_job_ray_backend.py b/cvs/lib/inference/unittests/test_vllm_job_ray_backend.py new file mode 100644 index 000000000..6c67c13ff --- /dev/null +++ b/cvs/lib/inference/unittests/test_vllm_job_ray_backend.py @@ -0,0 +1,1385 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for the Ray distributed-executor-backend support added to +cvs.lib.inference.vllm_job.VllmJob and +cvs.lib.inference.utils.vllm_config_loader.VariantConfig. + +Impl-blind / spec-derived (greenfield): these tests are written from the +behavioral spec before the implementation exists and are committed RED. A +different agent makes them green and may NOT edit this file. + +Coverage map (spec AC -> test): + Config validator relaxation ...... AC1-6 -> TestVariantConfigRayConsistency + cell_key ray multi-node .......... AC7 -> TestCellKeyRayMultiNode + _is_ray_backend .................. AC8 -> TestIsRayBackend + _server_argv ray vs mp ........... AC16,17 + RC1,RC3 -> TestServerArgvRayVsMp + start_server bootstrap/order ..... AC9-15,26,27 -> TestStartServerRayBootstrap + stop_server teardown ............. AC18-21 -> TestStopServerRayTeardown + _check_early_failure ray skip .... AC22,23 -> TestCheckEarlyFailureRayWorkerSkip + server_signature ................. AC24,25 -> TestServerSignatureRay + lifecycle (transition table) ..... regr -> TestVllmJobRayLifecycle + +Coverage-gap additions (post-review, impl-blind against the same spec): + ray pp>1 keeps --pipeline-parallel-size ......... TestServerArgvRayVsMp + serve-launch EARLY_FAILURE (ray head, mp worker) TestStartServerRayBootstrap + bootstrap OR-branch matrix (bad-out head, exit!=0 worker) TestStartServerRayBootstrap + re-entrant start_server ......................... TestVllmJobRayLifecycle + node-rank strip strengthened (mp, non-vacuous) .. TestServerSignatureRay + worker master_addr regression (distinct from hosts[0]) TestStartServerRayBootstrap + +Round-2 coverage-gap additions (impl-blind against the same spec): + 6 empty/None exec silent-success guard ......... TestStartServerRayBootstrap + 7 nnodes=3 worker loop (all bootstrap; fail last) TestStartServerRayBootstrap + 8 ray+pp>1 through the REAL VariantConfig ....... TestVariantConfigRayConsistency + 9 stop-after-failed-start asserts teardown calls TestVllmJobRayLifecycle + 10 FATAL_LOG_RE grep -> "vllm server fatal error" TestCheckEarlyFailureRayWorkerSkip + +Post-review round additions (impl-blind against the same spec): + R1.1 bootstrap-fail return omitting 'output' key . TestStartServerRayBootstrap + R1.2 server_signature env tuple independent oracle TestServerSignatureRay + R2.1 mp dist-block flag VALUES (not just presence) TestServerArgvRayVsMp + R2.2 is_ready True/False/empty + multinode skip .. TestVllmJobIsReady + R2.3 parse_results empty/unparseable/delegation .. TestVllmJobParseResults + +Round-3 coverage-gap additions (impl-blind against the same spec): + R3.1 parse_results asserts RETURN value (not just delegation) TestVllmJobParseResults + R3.2 wait_ready poll state machine (return/timeout/order) .. TestVllmJobWaitReady + R3.3 build_server_cmd env-script + per-rank mkdir branches .. TestVllmJobBuildServerCmd + +Round-4 coverage-gap additions (impl-blind against the same spec): + R4.1 server_env pass-through pins KEY=VALUE (non-colliding) TestVllmJobBuildServerCmd + R4.2 ray server_signature invariant to nnodes (2 vs 3) .... TestServerSignatureRay + R4.3 cell_key pp>1 branch (PP= segment) + pp==1, subTest ... TestCellKeyRayMultiNode + R4.4 _flatten_serve_args list/tuple repeat branch ......... TestFlattenServeArgsBranches +''' + +import unittest +import unittest.mock as mock +from types import SimpleNamespace + +from pydantic import ValidationError + +from cvs.lib.inference.utils.vllm_config_loader import VariantConfig +from cvs.lib.inference.vllm_job import VllmJob + +RAY = {"distributed-executor-backend": "ray"} + +# EARLY_FAILURE_RE-matching / non-matching bootstrap outputs (spec Failure Modes). +_CLEAN = "Local node IP: 10.0.0.1" # confirmed NON-matching +_BAD = "command not found" # confirmed matching + + +# --------------------------------------------------------------------------- # +# Fakes / fixtures (per spec "Test fake orchestrator contract") +# --------------------------------------------------------------------------- # +class RecordingOrch: + """Records (cmd, hosts) per exec call so host-targeting and ordering ACs + are checkable. `responder(cmd, hosts, detailed) -> dict` controls returns.""" + + hosts = ["10.0.0.1", "10.0.0.2"] # index 0 = head/rank0, 1 = worker/rank1 + + def __init__(self, responder=None, hosts=None, head_responder=None): + self.calls = [] # list of (cmd, hosts) in call order + self.head_cmds = [] + self._responder = responder + # head_responder(cmd) -> return value for exec_on_head; None preserves the + # legacy {} return so existing tests that never inspect exec_on_head output + # are unaffected. Only parse_results (which fetches via exec_on_head) needs it. + self._head_responder = head_responder + if hosts is not None: + self.hosts = list(hosts) + + def exec(self, cmd, hosts=None, detailed=False, **k): + self.calls.append((cmd, hosts)) + if self._responder is not None: + return self._responder(cmd, hosts, detailed) + return {} + + def exec_on_head(self, cmd, *a, **k): + self.head_cmds.append(cmd) + if self._head_responder is not None: + return self._head_responder(cmd) + return {} + + +HEAD = RecordingOrch.hosts[0] +WORKER = RecordingOrch.hosts[1] +HOST2 = "10.0.0.3" # third host for nnodes=3 worker-loop coverage (not in default hosts) + + +def _responder_ok(): + """Every bootstrap succeeds (exit 0, clean output); serve launch is clean. + + A detailed `grep` (the _check_early_failure FATAL scan) returns exit_code 1 + = "no fatal pattern found"; every other detailed call (ray bootstrap) returns + exit_code 0 = success. Non-detailed calls (serve launch / tail) return clean + text that does not match EARLY_FAILURE_RE. + """ + + def r(cmd, hosts, detailed): + host = hosts[0] if hosts else HEAD + if detailed: + exit_code = 1 if "grep" in cmd else 0 + return {host: {"exit_code": exit_code, "output": _CLEAN, "stdout": ""}} + return {host: ""} + + return r + + +def _responder_bootstrap_fail(fail_map): + """fail_map: host -> detailed return dict for that host's bootstrap; all + other hosts succeed, serve launches are clean.""" + + def r(cmd, hosts, detailed): + host = hosts[0] if hosts else HEAD + if detailed: + return {host: fail_map.get(host, {"exit_code": 0, "output": _CLEAN, "stdout": _CLEAN})} + return {host: ""} + + return r + + +def _responder_serve_fail(bad_serve_hosts): + """Ray/mp bootstrap detailed calls all succeed (exit 0, clean output); the + NON-detailed `vllm serve` launch returns EARLY_FAILURE_RE-matching output for + hosts in `bad_serve_hosts`, clean otherwise. + + Exercises the post-bootstrap serve-launch EARLY_FAILURE check (the + "vllm server failed to launch on ... (rank N)" RuntimeError site), which is + distinct from the bootstrap failure sites covered by _responder_bootstrap_fail. + """ + + def r(cmd, hosts, detailed): + host = hosts[0] if hosts else HEAD + if detailed: + # No grep is issued by start_server; bootstrap detailed calls succeed. + exit_code = 1 if "grep" in cmd else 0 + return {host: {"exit_code": exit_code, "output": _CLEAN, "stdout": ""}} + if "vllm serve" in cmd and host in bad_serve_hosts: + return {host: _BAD} + return {host: ""} + + return r + + +def _responder_const(value): + """Every orch.exec call (bootstrap detailed AND serve non-detailed) returns + the SAME constant `value` -- used to drive the empty/None silent-success guard + (`(out or {}).items()`) in _bootstrap_ray_cluster and start_server. With + value={} or value=None the guard's iterable is empty, so no per-host failure + check runs and no false positive is raised.""" + + def r(cmd, hosts, detailed): + return value + + return r + + +# FATAL_LOG_RE-matching text confirmed by the class regex (see stub FATAL_LOG_RE: +# "...|Engine core initialization failed|..."). Distinct from EARLY_FAILURE_RE. +_FATAL = "Engine core initialization failed" + + +def _responder_fatal_grep(fatal_hosts, fatal_text=_FATAL): + """Detailed `grep` (the _check_early_failure FATAL_LOG_RE scan) returns + exit_code 0 (match found) with FATAL-matching `stdout` for hosts in + `fatal_hosts`; every other detailed call (and every host's grep otherwise) + returns exit_code 1 = no match. Non-detailed `tail` returns clean text that + does NOT match EARLY_FAILURE_RE, so the FATAL_LOG_RE branch -- not the tail + EARLY_FAILURE branch -- is the one that fires.""" + + def r(cmd, hosts, detailed): + host = hosts[0] if hosts else HEAD + if detailed: + if "grep" in cmd and host in fatal_hosts: + return {host: {"exit_code": 0, "stdout": fatal_text, "output": fatal_text}} + return {host: {"exit_code": 1, "stdout": "", "output": _CLEAN}} + return {host: ""} + + return r + + +def _responder_readiness(exit_code=0, empty=False): + """is_ready() greps each non-skipped rank's readiness log via + orch.exec(detailed=True) and returns {host: {"exit_code": ...}}; exit_code 0 + means the readiness pattern was found (server ready). empty=True returns {} + to exercise the `not out` (empty result) False path. Non-detailed calls + return clean text (unused by is_ready).""" + + def r(cmd, hosts, detailed): + if empty: + return {} + host = hosts[0] if hosts else HEAD + if detailed: + return {host: {"exit_code": exit_code, "output": "", "stdout": ""}} + return {host: ""} + + return r + + +def _variant(serve_args=None, nnodes="2", pp="2", ib_netdev="enp159s0np0", tp="8", master_addr="10.0.0.1", env=None): + """Minimal SimpleNamespace variant mirroring _variant() in the reuse suite.""" + params = SimpleNamespace( + tensor_parallelism=tp, + pipeline_parallel_size=pp, + master_addr=master_addr, + master_port="29501", + nnodes=nnodes, + port_no="8000", + random_range_ratio="0.0", + random_prefix_len="0", + burstiness="1.0", + seed="0", + request_rate="inf", + tokenizer_mode="auto", + percentile_metrics="ttft,tpot,itl,e2el", + metric_percentiles="50,90,95,99", + base_url="http://0.0.0.0", + dataset_name="random", + backend="vllm", + ) + return SimpleNamespace( + params=params, + model=SimpleNamespace(id="/models/test-model"), + paths=SimpleNamespace(log_dir="/logs", models_dir="/models"), + roles=SimpleNamespace( + server=SimpleNamespace(serve_args=dict(serve_args or {}), env=dict(env or {}), ib_netdev=ib_netdev) + ), + ) + + +def _job( + orch=None, + serve_args=None, + nnodes="2", + pp="2", + ib_netdev="enp159s0np0", + concurrency=16, + isl="1024", + osl="1024", + tp="8", + master_addr="10.0.0.1", + env=None, + ib_hcas=None, +): + orch = RecordingOrch() if orch is None else orch + return VllmJob( + orch=orch, + variant=_variant(serve_args, nnodes, pp, ib_netdev, tp, master_addr, env), + hf_token="tok", + isl=isl, + osl=osl, + concurrency=concurrency, + num_prompts="640", + ib_hcas=ib_hcas, + ) + + +def _vc(nnodes="2", pp="1", serve_args=None, ib_netdev="eth0", tp="8"): + """A real pydantic VariantConfig exercising _check_distributed_consistency. + + enforce_thresholds=False so the (independent) threshold-coverage validator + only warns and never masks the distributed-consistency error under test. + """ + return VariantConfig( + schema_version=1, + framework="vllm", + gpu_arch="mi300x", + enforce_thresholds=False, + paths={ + "shared_fs": "/home/x", + "models_dir": "/home/x/models", + "log_dir": "/home/x/LOGS", + "hf_token_file": "/home/x/.hf", + }, + model={"id": "/models/test-model", "remote": 0}, + params={"tensor_parallelism": tp, "pipeline_parallel_size": pp, "nnodes": nnodes}, + roles={"server": {"serve_args": dict(serve_args or {}), "env": {}, "ib_netdev": ib_netdev}}, + sweep={ + "sequence_combinations": [{"name": "a", "isl": "1024", "osl": "1024"}], + "runs": [{"combo": "a", "concurrency": 16}], + }, + thresholds={}, + ) + + +# --------------------------------------------------------------------------- # +# helpers for argv / call inspection +# --------------------------------------------------------------------------- # +def _value_after(argv, flag): + """Return the element following `flag` in argv, or None if flag absent.""" + for i, a in enumerate(argv): + if a == flag: + return argv[i + 1] if i + 1 < len(argv) else None + return None + + +def _calls_to(orch, host): + return [cmd for cmd, hosts in orch.calls if hosts == [host]] + + +def _first_index(orch, predicate): + for i, (cmd, hosts) in enumerate(orch.calls): + if predicate(cmd, hosts): + return i + return -1 + + +def _all_cmds(orch): + return [c for c, _ in orch.calls] + list(orch.head_cmds) + + +# --------------------------------------------------------------------------- # +# Config validator: VariantConfig._check_distributed_consistency (AC1-6) +# --------------------------------------------------------------------------- # +class TestVariantConfigRayConsistency(unittest.TestCase): + """The ray relaxation applies ONLY to the (nn>1 & pp==1) rule and ONLY for + the exact string 'ray'. ib_netdev and the (pp>1 & nn==1) rule are untouched.""" + + def test_accepts_valid(self): + # (nnodes, pp, serve_args, ib_netdev) that must construct without error. + cases = [ + ("1", "1", {}, None), # baseline: unrelaxed single-node default path + ("2", "1", RAY, "eth0"), # AC1: ray relaxation permits nn>1 & pp==1 + ("2", "2", {}, "eth0"), # AC3: mp multi-node path unchanged + ("2", "2", RAY, "eth0"), # finding 8: ray + pp>1 is legal (nn>1 & pp>1 + # is valid for ANY backend; the ray relaxation only special-cases pp==1, + # it never REJECTS ray+pp>1). Validated through the REAL VariantConfig + # validator, not just the SimpleNamespace fake used by _server_argv tests. + ] + for nn, pp, sa, ib in cases: + with self.subTest(nnodes=nn, pp=pp, serve_args=sa): + try: + _vc(nnodes=nn, pp=pp, serve_args=sa, ib_netdev=ib) + except ValidationError as e: # pragma: no cover - failure path + self.fail(f"unexpected ValidationError: {e}") + + def test_rejects_invalid(self): + # (nnodes, pp, serve_args, ib_netdev, field-token-in-message) + cases = [ + ("2", "1", {}, "eth0", "pipeline_parallel_size"), # AC2 no ray key + ("1", "2", RAY, "eth0", "pipeline_parallel_size"), # AC4 pp>1 & nn==1 never relaxed + ("2", "1", RAY, None, "ib_netdev"), # AC5 ib_netdev not relaxed by ray + ("2", "1", {"distributed-executor-backend": "RAY"}, "eth0", "pipeline_parallel_size"), # AC6 case-sensitive + ("2", "1", {"distributed-executor-backend": "Ray"}, "eth0", "pipeline_parallel_size"), # AC6 case-sensitive + ] + for nn, pp, sa, ib, token in cases: + with self.subTest(nnodes=nn, pp=pp, serve_args=sa, ib_netdev=ib): + with self.assertRaises(ValidationError) as ctx: + _vc(nnodes=nn, pp=pp, serve_args=sa, ib_netdev=ib) + self.assertIn(token, str(ctx.exception)) + + +class TestCellKeyRayMultiNode(unittest.TestCase): + """AC7: ray multi-node has pp=1, so cell_key uses the single-node format + (no PP= segment), identical to a genuine single-node cell. + + Round-4 finding 3: cell_key has two branches -- pp==1 (no PP= segment) and + pp>1 (a "PP=," segment inserted before CONC). The pp>1 branch had zero + coverage anywhere for THIS VariantConfig class, so both branches are now + pinned together in one subTest table (discipline rule B), asserting the exact + segment position/value/comma placement, not just presence.""" + + def test_cell_key_format_both_pp_branches(self): + # (nnodes, pp, serve_args, ib_netdev, expected_key) + cases = [ + # AC7: ray multi-node, pp==1 -> single-node format, NO PP= segment. + ("2", "1", RAY, "eth0", "ISL=1024,OSL=1024,TP=8,CONC=16"), + # pp>1 branch -> "PP=2," inserted immediately before CONC. pp>1 requires + # nnodes>1 (the pp>1 & nn==1 rule always fires), so this is a valid mp + # multi-node config; the PP segment is what distinguishes it from the + # pp==1 key above. + ("2", "2", {}, "eth0", "ISL=1024,OSL=1024,TP=8,PP=2,CONC=16"), + ] + for nn, pp, sa, ib, expected in cases: + with self.subTest(nnodes=nn, pp=pp): + vc = _vc(nnodes=nn, pp=pp, serve_args=sa, ib_netdev=ib, tp="8") + self.assertEqual(vc.cell_key(isl="1024", osl="1024", concurrency="16"), expected) + + +# --------------------------------------------------------------------------- # +# _is_ray_backend (AC8) +# --------------------------------------------------------------------------- # +class TestIsRayBackend(unittest.TestCase): + def test_backend_detection_is_exact_string(self): + # (serve_args, expected) + cases = [ + ({"distributed-executor-backend": "ray"}, True), + ({}, False), + ({"distributed-executor-backend": "mp"}, False), + ({"distributed-executor-backend": "RAY"}, False), + ({"distributed-executor-backend": "Ray"}, False), + ] + for sa, expected in cases: + with self.subTest(serve_args=sa): + job = _job(serve_args=sa, nnodes="1", pp="1") + self.assertIs(job._is_ray_backend, expected) + + def test_property_reflects_live_serve_args_not_a_cached_snapshot(self): + job = _job(serve_args={}, nnodes="1", pp="1") + self.assertIs(job._is_ray_backend, False) + job.serve_args["distributed-executor-backend"] = "ray" + self.assertIs(job._is_ray_backend, True) + + +# --------------------------------------------------------------------------- # +# _server_argv (AC16, AC17, RC1, RC3) +# --------------------------------------------------------------------------- # +class TestServerArgvRayVsMp(unittest.TestCase): + _DRIVER_DIST_FLAGS = [ + "--node-rank", + "--headless", + "--pipeline-parallel-size", + "--master-addr", + "--master-port", + "--nnodes", + ] + + def test_ray_multinode_omits_all_driver_dist_flags(self): + # AC16: the mp block is skipped entirely under ray. + argv = _job(serve_args=RAY, nnodes="2", pp="1")._server_argv(0) + for flag in self._DRIVER_DIST_FLAGS: + with self.subTest(flag=flag): + self.assertNotIn(flag, argv) + + def test_ray_multinode_backend_arrives_via_serve_args(self): + # AC17: --distributed-executor-backend ray comes from _flatten_serve_args. + argv = _job(serve_args=RAY, nnodes="2", pp="1")._server_argv(0) + self.assertIn("--distributed-executor-backend", argv) + self.assertEqual(_value_after(argv, "--distributed-executor-backend"), "ray") + + def test_mp_multinode_injects_full_dist_block(self): + # RC1: mp multi-node keeps the driver-injected block + hardcoded mp backend. + # Round-2 finding 1: assert the VALUE after each flag, not just presence. A + # mutant that emits the right flag names but wrong/hardcoded values -- e.g. + # swapping master_addr/master_port, hardcoding --nnodes 1, or dropping the + # pipeline width -- would pass a presence-only check while breaking the + # launch. Each expected value is pinned to the job's real attribute (read + # from variant.params, not re-read from the produced argv), so the check is + # independent of the argv it is validating. + job = _job(serve_args={}, nnodes="2", pp="2") + argv = job._server_argv(0) + expected = [ + ("--node-rank", "0"), # the rank argument passed to _server_argv(0) + ("--master-addr", job.master_addr), + ("--master-port", job.master_port), + ("--nnodes", job.nnodes), + ("--pipeline-parallel-size", job.pp), + ("--distributed-executor-backend", "mp"), # hardcoded on the mp path + ] + for flag, val in expected: + with self.subTest(flag=flag): + self.assertIn(flag, argv) + self.assertEqual(_value_after(argv, flag), val) + + def test_mp_worker_rank_is_headless(self): + # RC1: rank>0 mp worker additionally carries --headless; rank 0 does not. + # Round-2 finding 1: also pin --node-rank's VALUE to the actual rank arg, so + # a mutant that always emits "--node-rank 0" regardless of rank (breaking + # multi-node distribution) is caught -- not merely flag/--headless presence. + job = _job(serve_args={}, nnodes="2", pp="2") + argv0 = job._server_argv(0) + argv1 = job._server_argv(1) + self.assertNotIn("--headless", argv0) + self.assertIn("--headless", argv1) + self.assertEqual(_value_after(argv0, "--node-rank"), "0") + self.assertEqual(_value_after(argv1, "--node-rank"), "1") + + def test_single_node_ray_passthrough_no_driver_flags(self): + # RC3 / Edge: single-node omits all driver-injected dist flags, but the + # user's serve_args backend still passes through verbatim. + argv = _job(serve_args=RAY, nnodes="1", pp="1")._server_argv(0) + for flag in self._DRIVER_DIST_FLAGS: + with self.subTest(flag=flag): + self.assertNotIn(flag, argv) + self.assertEqual(_value_after(argv, "--distributed-executor-backend"), "ray") + + def test_ray_multinode_pp_gt_1_keeps_pipeline_parallel_size(self): + # Coverage-gap (finding 1): VariantConfig permits a ray backend with + # nnodes>1 AND pp>1 (the ray relaxation only special-cases pp==1; the + # nn>1 & pp>1 combo is legal for any backend). The mp block that normally + # carries "--pipeline-parallel-size" is skipped for every ray job, so a + # ray+pp=2 config must NOT silently drop the pipeline-parallel width: the + # head's single `vllm serve` still has to be told pp=2 (via the flag with + # value self.pp) or the cluster silently runs at pp=1. A mutant that drops + # the flag under ray (the current guard `nnodes>1 and not _is_ray_backend`) + # is caught here. + argv = _job(serve_args=RAY, nnodes="2", pp="2")._server_argv(0) + self.assertIn("--pipeline-parallel-size", argv) + self.assertEqual(_value_after(argv, "--pipeline-parallel-size"), "2") + # Backend is still ray (contributed by serve_args passthrough), not mp. + self.assertEqual(_value_after(argv, "--distributed-executor-backend"), "ray") + # Ray still manages rendezvous, so the torchrun-style mp flags stay absent + # even though pp>1 (ray does not use --node-rank/--master-*/--nnodes/--headless). + for flag in ("--node-rank", "--headless", "--master-addr", "--master-port", "--nnodes"): + with self.subTest(flag=flag): + self.assertNotIn(flag, argv) + + +# --------------------------------------------------------------------------- # +# _flatten_serve_args: the four equivalence classes (RC8) +# --------------------------------------------------------------------------- # +class TestFlattenServeArgsBranches(unittest.TestCase): + """RC8: _flatten_serve_args has four branches -- True (bare flag), False + (omitted), list/tuple (flag repeated per element), scalar (flag + str(value)). + True/False/scalar are covered in test_vllm_job_server_reuse.py; the list/tuple + repeat branch (Round-4 finding 4) is covered here so all four cells of this + pure function's table are pinned (discipline rule B). A bug in the repeat branch + (wrong flag repeated, values not str()-cast, wrong order) is caught.""" + + def test_list_and_tuple_values_repeat_the_flag_per_element(self): + # (value, expected) -- list and tuple both repeat "--" before each + # element, in order; non-string elements are str()-cast. + cases = [ + (["a.b.C", "d.e.F"], ["--middleware", "a.b.C", "--middleware", "d.e.F"]), + (("a.b.C", "d.e.F"), ["--middleware", "a.b.C", "--middleware", "d.e.F"]), + ([1, 2], ["--middleware", "1", "--middleware", "2"]), # str()-cast + ] + for value, expected in cases: + with self.subTest(value=value): + self.assertEqual(VllmJob._flatten_serve_args({"middleware": value}), expected) + + +# --------------------------------------------------------------------------- # +# start_server: ray bootstrap ordering, host targeting, failure (AC9-15,26,27) +# --------------------------------------------------------------------------- # +class TestStartServerRayBootstrap(unittest.TestCase): + def test_head_bootstrap_command_and_target(self): + # AC9 + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + head_ray = [c for c in _calls_to(orch, HEAD) if "ray start" in c] + self.assertTrue(head_ray, "expected a ray start command targeting the head") + cmd = head_ray[0] + for token in ("ray start", "--head", "--port=29501"): + with self.subTest(token=token): + self.assertIn(token, cmd) + + def test_worker_bootstrap_command_and_target(self): + # AC10 + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + worker_ray = [c for c in _calls_to(orch, WORKER) if "ray start" in c] + self.assertTrue(worker_ray, "expected a ray start command targeting the worker") + cmd = worker_ray[0] + self.assertIn("ray start", cmd) + self.assertIn("--address=10.0.0.1:29501", cmd) + + def test_worker_bootstrap_targets_master_addr_not_head_host(self): + # AC10 disambiguation / REGRESSION: the worker's Ray rendezvous --address + # must be self.master_addr (the data-plane IP the head actually started + # with via `ray start --head --port=...`), NOT self.orch.hosts[0] (the + # SSH/management host). The default fixture sets master_addr == hosts[0] + # ("10.0.0.1"), so the plain AC10 test above passes regardless of which + # field the impl uses. Here master_addr is DISTINCT from hosts[0] + # (hosts=["10.0.0.1","10.0.0.2"], master_addr="172.16.0.1"), so only an + # impl that targets master_addr passes; one that targets hosts[0] fails. + orch = RecordingOrch(responder=_responder_ok()) # hosts[0]=HEAD=10.0.0.1 + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1", master_addr="172.16.0.1").start_server() + worker_ray = [c for c in _calls_to(orch, WORKER) if "ray start" in c] + self.assertTrue(worker_ray, "expected a ray start command targeting the worker") + cmd = worker_ray[0] + self.assertIn( + "--address=172.16.0.1:29501", + cmd, + "worker rendezvous must target master_addr (data-plane IP), not hosts[0]", + ) + self.assertNotIn( + "--address=10.0.0.1:29501", + cmd, + "worker must NOT rendezvous against the SSH/management host hosts[0]", + ) + + def test_bootstrap_precedes_serve_launch(self): + # AC11: every ray start bootstrap (head AND worker) precedes the vllm serve launch. + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + first_head_bootstrap = _first_index(orch, lambda c, h: "ray start" in c and h == [HEAD]) + first_worker_bootstrap = _first_index(orch, lambda c, h: "ray start" in c and h == [WORKER]) + first_serve = _first_index(orch, lambda c, h: "vllm serve" in c) + self.assertNotEqual(first_head_bootstrap, -1, "no head ray start call recorded") + self.assertNotEqual(first_worker_bootstrap, -1, "no worker ray start call recorded") + self.assertNotEqual(first_serve, -1, "no vllm serve call recorded") + self.assertLess(first_head_bootstrap, first_serve) + self.assertLess(first_worker_bootstrap, first_serve) + + def test_no_serve_on_worker_under_ray(self): + # AC12: vllm serve runs only on the head under ray. + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + self.assertEqual([c for c in _calls_to(orch, WORKER) if "vllm serve" in c], []) + self.assertTrue([c for c in _calls_to(orch, HEAD) if "vllm serve" in c]) + + def test_mp_serves_every_host_no_ray_start(self): + # AC13: mp multi-node serves on every host (incl. worker), no ray start. + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args={}, nnodes="2", pp="2").start_server() + self.assertTrue([c for c in _calls_to(orch, HEAD) if "vllm serve" in c]) + self.assertTrue([c for c in _calls_to(orch, WORKER) if "vllm serve" in c]) + self.assertEqual([c for c, _ in orch.calls if "ray start" in c], []) + + def test_single_node_ray_no_bootstrap(self): + # AC14: 1-node ray issues no ray start and exactly one vllm serve launch. + orch = RecordingOrch(responder=_responder_ok(), hosts=[HEAD]) + _job(orch=orch, serve_args=RAY, nnodes="1", pp="1", ib_netdev=None).start_server() + self.assertEqual([c for c in _all_cmds(orch) if "ray start" in c], []) + serves = [c for c in _all_cmds(orch) if "vllm serve" in c] + self.assertEqual(len(serves), 1, f"expected exactly one serve launch, got {serves}") + + def test_happy_path_launches_serve_on_head(self): + # AC15: clean bootstrap -> no exception + serve on head via exec(hosts=[head]). + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + self.assertTrue([c for c in _calls_to(orch, HEAD) if "vllm serve" in c]) + + def test_head_bootstrap_failure_aborts_before_serve(self): + # AC26: head exit_code!=0 -> RuntimeError(rank 0), no serve, no worker bootstrap. + orch = RecordingOrch( + responder=_responder_bootstrap_fail({HEAD: {"exit_code": 1, "output": "something went wrong"}}) + ) + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + msg = str(ctx.exception) + self.assertIn("ray bootstrap failed on", msg) + self.assertIn("rank 0", msg) + self.assertEqual([c for c, _ in orch.calls if "vllm serve" in c], []) + self.assertEqual([c for c in _calls_to(orch, WORKER) if "ray start" in c], []) + + def test_worker_bootstrap_failure_bad_output(self): + # AC27: head ok, worker exit 0 but output matches EARLY_FAILURE_RE -> rank 1. + orch = RecordingOrch(responder=_responder_bootstrap_fail({WORKER: {"exit_code": 0, "output": _BAD}})) + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + msg = str(ctx.exception) + self.assertIn("ray bootstrap failed on", msg) + self.assertIn("rank 1", msg) + self.assertEqual([c for c, _ in orch.calls if "vllm serve" in c], []) + + # ---- Coverage-gap (finding 5): the bootstrap failure check is an OR of + # (exit_code != 0) OR (EARLY_FAILURE_RE matches output), applied to BOTH head + # and worker. Existing tests cover only exit!=0-on-head and bad-output-on-worker; + # the two mirror combinations below (bad-output-on-head, exit!=0-on-worker) close + # the OR-branch matrix so a mutant dropping either half on either host is killed. + def test_head_bootstrap_failure_bad_output_exit0(self): + # head: exit_code 0 but output matches EARLY_FAILURE_RE -> RuntimeError rank 0, + # aborting before any worker bootstrap and before serve launch. + orch = RecordingOrch(responder=_responder_bootstrap_fail({HEAD: {"exit_code": 0, "output": _BAD}})) + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + msg = str(ctx.exception) + self.assertIn("ray bootstrap failed on", msg) + self.assertIn("rank 0", msg) + self.assertEqual([c for c in _calls_to(orch, WORKER) if "ray start" in c], []) + self.assertEqual([c for c, _ in orch.calls if "vllm serve" in c], []) + + def test_worker_bootstrap_failure_nonzero_exit_clean_output(self): + # worker: exit_code != 0 with CLEAN (non-EARLY_FAILURE) output -> RuntimeError + # rank 1. The head bootstrap succeeded, so the failure is attributed to rank 1. + orch = RecordingOrch(responder=_responder_bootstrap_fail({WORKER: {"exit_code": 1, "output": _CLEAN}})) + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + msg = str(ctx.exception) + self.assertIn("ray bootstrap failed on", msg) + self.assertIn("rank 1", msg) + + # ---- Coverage-gap (Round-1 finding 1): the bootstrap-failure detailed return + # dict may OMIT the content key entirely. The spec's Failure Modes section is + # explicit: the check reads r.get("output", ""), so a return dict missing the + # content key is treated as empty string -- "no KeyError, no false positive". + # Every other responder in this file always supplies an "output" key, so a + # regression that read r["output"] (KeyError on a real orchestrator response + # missing that key) would slip through. Here the failing head returns a dict + # with exit_code=1 and NO "output" key at all: start_server() must still raise + # the normal RuntimeError naming rank 0 (the empty-output path), NOT a KeyError. + # assertRaises(RuntimeError) does not catch KeyError, so an r["output"] mutant + # surfaces as a test error/failure rather than a false pass. + def test_head_bootstrap_failure_output_key_absent_no_keyerror(self): + orch = RecordingOrch(responder=_responder_bootstrap_fail({HEAD: {"exit_code": 1}})) + try: + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + except KeyError as e: # pragma: no cover - regression guard + self.fail(f"missing 'output' key must be treated as empty string, not raise KeyError: {e!r}") + msg = str(ctx.exception) + self.assertIn("ray bootstrap failed on", msg) + self.assertIn("rank 0", msg) + # Aborts before any serve launch, exactly like the with-output failure path. + self.assertEqual([c for c, _ in orch.calls if "vllm serve" in c], []) + self.assertEqual([c for c, _ in orch.calls if "vllm serve" in c], []) + + # ---- Coverage-gap (finding 4): the post-bootstrap `vllm serve` launch has its + # own EARLY_FAILURE_RE check (RuntimeError "vllm server failed to launch on ... + # (rank N)"), distinct from the bootstrap checks above. No existing test returns + # EARLY_FAILURE output for the non-detailed serve launch, so these two sites -- + # the ray head launch and the mp non-head-rank launch -- were never exercised. + def test_ray_head_serve_launch_failure_raises_rank0(self): + # ray path: bootstrap (head + worker) succeeds, but the head's post-bootstrap + # vllm serve launch output matches EARLY_FAILURE_RE -> RuntimeError rank 0. + orch = RecordingOrch(responder=_responder_serve_fail({HEAD})) + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + msg = str(ctx.exception) + self.assertIn("vllm server failed to launch on", msg) + self.assertIn("rank 0", msg) + + def test_mp_worker_serve_launch_failure_raises_rank1(self): + # mp path (else branch): the non-head-rank (rank 1) vllm serve launch output + # matches EARLY_FAILURE_RE -> RuntimeError rank 1. Confirms the serve-launch + # failure check fires for a worker on the mp path, not just the head. + orch = RecordingOrch(responder=_responder_serve_fail({WORKER})) + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args={}, nnodes="2", pp="2").start_server() + msg = str(ctx.exception) + self.assertIn("vllm server failed to launch on", msg) + self.assertIn("rank 1", msg) + + # ---- Coverage-gap (finding 6): the per-host failure check iterates + # `(out or {}).items()` in BOTH _bootstrap_ray_cluster and the serve-launch + # scan in start_server. When orch.exec returns {} or None (an empty/omitted + # result -- which the real orchestrator can produce, and which the spec says + # must be treated as empty output "no KeyError, no false positive"), the + # iterable is empty, no host entry is examined, and the code proceeds as a + # silent success. No prior responder ever returned {}/None, so this guard was + # never exercised. Intended behavior: start_server does NOT raise and the ray + # start + vllm serve calls are still issued (returns are recorded regardless). + def test_empty_or_none_bootstrap_result_is_silent_success(self): + for value in ({}, None): + with self.subTest(exec_return=value): + orch = RecordingOrch(responder=_responder_const(value)) + try: + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").start_server() + except Exception as e: # pragma: no cover - failure path + self.fail(f"empty/None exec return must be silent-success, raised: {e!r}") + # The calls were still dispatched (their empty returns just yield no + # failure to detect): head+worker ray start and a head vllm serve. + self.assertTrue([c for c in _calls_to(orch, HEAD) if "ray start" in c]) + self.assertTrue([c for c in _calls_to(orch, WORKER) if "ray start" in c]) + self.assertTrue([c for c in _calls_to(orch, HEAD) if "vllm serve" in c]) + + # ---- Coverage-gap (finding 7): the worker bootstrap loop was only ever + # exercised with exactly ONE worker (nnodes=2). A loop that bootstraps only + # hosts[1] (off-by-one / no loop) would pass every nnodes=2 test. These two + # nnodes=3 cases pin the loop: (a) EVERY worker is bootstrapped on the happy + # path; (b) a failure injected on the LAST worker still aborts with the + # correct rank, proving the loop reaches it (no short-circuit after rank 1). + def test_three_node_ray_bootstraps_every_worker(self): + orch = RecordingOrch(responder=_responder_ok(), hosts=[HEAD, WORKER, HOST2]) + _job(orch=orch, serve_args=RAY, nnodes="3", pp="1").start_server() + # Both workers (rank 1 and rank 2) get a ray start; the head gets one too. + self.assertTrue([c for c in _calls_to(orch, HEAD) if "ray start" in c], "head ray start missing") + self.assertTrue([c for c in _calls_to(orch, WORKER) if "ray start" in c], "rank-1 worker ray start missing") + self.assertTrue([c for c in _calls_to(orch, HOST2) if "ray start" in c], "rank-2 worker ray start missing") + # Ray still serves only on the head; no serve on either worker. + self.assertTrue([c for c in _calls_to(orch, HEAD) if "vllm serve" in c]) + self.assertEqual([c for c in _calls_to(orch, WORKER) if "vllm serve" in c], []) + self.assertEqual([c for c in _calls_to(orch, HOST2) if "vllm serve" in c], []) + + def test_three_node_ray_failure_on_last_worker_aborts_rank2(self): + # Failure on the LAST worker (rank 2), not the first -- confirms the loop + # does not short-circuit at rank 1. The head and rank-1 worker bootstrap + # cleanly; rank-2 fails, so the RuntimeError names rank 2 and no serve runs. + orch = RecordingOrch( + responder=_responder_bootstrap_fail({HOST2: {"exit_code": 1, "output": _BAD}}), + hosts=[HEAD, WORKER, HOST2], + ) + with self.assertRaises(RuntimeError) as ctx: + _job(orch=orch, serve_args=RAY, nnodes="3", pp="1").start_server() + msg = str(ctx.exception) + self.assertIn("ray bootstrap failed on", msg) + self.assertIn("rank 2", msg) + # The loop DID reach the earlier ranks before failing at the last worker. + self.assertTrue([c for c in _calls_to(orch, HEAD) if "ray start" in c], "head must have bootstrapped") + self.assertTrue( + [c for c in _calls_to(orch, WORKER) if "ray start" in c], "rank-1 worker must have bootstrapped" + ) + # No serve launched anywhere after the abort. + self.assertEqual([c for c, _ in orch.calls if "vllm serve" in c], []) + + +# --------------------------------------------------------------------------- # +# stop_server: ray teardown (AC18-21) +# --------------------------------------------------------------------------- # +@mock.patch("cvs.lib.inference.vllm_job.time.sleep") +class TestStopServerRayTeardown(unittest.TestCase): + def test_ray_multinode_broadcasts_single_ray_stop(self, mock_sleep): + # AC18: exactly one broadcast (hosts=None) ray stop. + orch = RecordingOrch() + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").stop_server() + ray_stops = [(c, h) for c, h in orch.calls if "ray stop" in c] + self.assertEqual(len(ray_stops), 1, f"expected one ray stop, got {ray_stops}") + self.assertIsNone(ray_stops[0][1], "ray stop must be broadcast (hosts=None)") + + def test_pkill_precedes_ray_stop(self, mock_sleep): + # AC19: pkill vllm serve broadcast comes before ray stop. + orch = RecordingOrch() + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1").stop_server() + pkill_idx = _first_index(orch, lambda c, h: "pkill" in c and "vllm serve" in c) + ray_idx = _first_index(orch, lambda c, h: "ray stop" in c) + self.assertNotEqual(pkill_idx, -1, "no pkill vllm serve call recorded") + self.assertNotEqual(ray_idx, -1, "no ray stop call recorded") + self.assertLess(pkill_idx, ray_idx) + + def test_mp_multinode_no_ray_stop(self, mock_sleep): + # AC20 / RC4 + orch = RecordingOrch() + _job(orch=orch, serve_args={}, nnodes="2", pp="2").stop_server() + self.assertEqual([c for c, _ in orch.calls if "ray stop" in c], []) + + def test_single_node_ray_no_ray_stop(self, mock_sleep): + # AC21 + orch = RecordingOrch(hosts=[HEAD]) + _job(orch=orch, serve_args=RAY, nnodes="1", pp="1", ib_netdev=None).stop_server() + self.assertEqual([c for c, _ in orch.calls if "ray stop" in c], []) + + +# --------------------------------------------------------------------------- # +# _check_early_failure: Ray worker skip (AC22, AC23) +# --------------------------------------------------------------------------- # +class TestCheckEarlyFailureRayWorkerSkip(unittest.TestCase): + def test_ray_worker_is_skipped(self): + # AC22: ray workers have no per-rank server log -> no tail/grep on worker. + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args=RAY, nnodes="2", pp="1")._check_early_failure() + self.assertEqual(_calls_to(orch, WORKER), [], "ray worker must not be tailed/grepped") + self.assertTrue(_calls_to(orch, HEAD), "rank 0 head must still be checked") + + def test_mp_worker_is_checked(self): + # AC23: mp workers DO produce a per-rank log -> rank-1 worker is checked. + orch = RecordingOrch(responder=_responder_ok()) + _job(orch=orch, serve_args={}, nnodes="2", pp="2")._check_early_failure() + self.assertTrue(_calls_to(orch, WORKER), "mp rank-1 worker must be tailed/grepped") + + def test_fatal_log_match_raises_with_rank(self): + # Coverage-gap (finding 10): the FATAL_LOG_RE grep branch (detailed grep + # returns exit_code 0 with stdout matching FATAL_LOG_RE) raises a + # RuntimeError "vllm server fatal error". Every prior fixture returned + # exit_code 1 / no-match for the grep, so this RuntimeError site was never + # exercised. Single-host job so exactly rank 0 is inspected; the tail + # returns clean text so the EARLY_FAILURE_RE branch does NOT pre-empt the + # FATAL_LOG_RE branch under test. + orch = RecordingOrch(responder=_responder_fatal_grep({HEAD}), hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp="1", ib_netdev=None) + with self.assertRaises(RuntimeError) as ctx: + job._check_early_failure() + msg = str(ctx.exception) + self.assertIn("vllm server fatal error", msg) + self.assertIn("rank 0", msg) + + def test_fatal_log_match_on_mp_worker_reports_rank1(self): + # Companion to finding 10: the FATAL_LOG_RE match on an mp rank-1 worker + # (which IS inspected, unlike a ray worker) must attribute the fatal error + # to rank 1 -- confirming the rank is threaded into the message, not + # hard-coded to 0. Head grep is clean; only the worker's grep matches. + orch = RecordingOrch(responder=_responder_fatal_grep({WORKER})) + job = _job(orch=orch, serve_args={}, nnodes="2", pp="2") + with self.assertRaises(RuntimeError) as ctx: + job._check_early_failure() + msg = str(ctx.exception) + self.assertIn("vllm server fatal error", msg) + self.assertIn("rank 1", msg) + + +# --------------------------------------------------------------------------- # +# server_signature (AC24, AC25) +# --------------------------------------------------------------------------- # +class TestServerSignatureRay(unittest.TestCase): + def test_hashable_and_stable(self): + # AC24 + job = _job(serve_args=RAY, nnodes="2", pp="1") + sig = job.server_signature() + self.assertEqual(hash(sig), hash(job.server_signature())) + self.assertEqual(sig, job.server_signature()) + + def test_invariant_to_concurrency(self): + # AC25: concurrency is client-only; two ray jobs differing only in it match. + self.assertEqual( + _job(serve_args=RAY, nnodes="2", pp="1", concurrency=4).server_signature(), + _job(serve_args=RAY, nnodes="2", pp="1", concurrency=64).server_signature(), + ) + + def test_ray_signature_invariant_to_nnodes(self): + # Round-4 finding 2 / spec Edge Cases (line 299, called out as "intentional"): + # two ray jobs differing ONLY in nnodes (2 vs 3) must produce EQUAL + # server_signature() values, because ray's _server_argv(0) never emits + # --nnodes (ray manages cluster size, not the `vllm serve` command). This + # is what lets a reused server span differently-sized ray clusters. A + # regression that leaks nnodes / worker count into the ray argv (e.g. a + # future _bootstrap change reusing _server_argv) would break server reuse + # and is caught here -- the mirror of the concurrency-invariance test above. + self.assertEqual( + _job(serve_args=RAY, nnodes="2", pp="1").server_signature(), + _job(serve_args=RAY, nnodes="3", pp="1").server_signature(), + ) + + def test_node_rank_strip_removes_flag_and_value(self): + # Strengthened (finding 3): the old test only asserted --node-rank absent + # from a ray signature, which is vacuous (ray argv never contains it, so no + # implementation could fail it -- redundant with AC16). Here we exercise the + # strip loop where it CAN fail: an mp multi-node job's _server_argv(0) DOES + # contain "--node-rank ", and server_signature() must remove exactly that + # flag+value pair (two elements) while leaving every other token intact. A + # mutant that strips nothing, strips only the flag, or strips extra tokens + # is caught. The ray no-op (no --node-rank to strip) is also re-asserted. + mp_job = _job(serve_args={}, nnodes="2", pp="2") + argv = list(mp_job._server_argv(0)) + self.assertIn("--node-rank", argv) # precondition: mp argv has it + i = argv.index("--node-rank") + expected = argv[:i] + argv[i + 2 :] # argv minus the flag+value pair + sig_argv = list(mp_job.server_signature()[0]) + self.assertNotIn("--node-rank", sig_argv) + self.assertEqual(sig_argv, expected) + self.assertEqual(len(sig_argv), len(argv) - 2) + # Ray path: no --node-rank is ever present, so the strip is a documented no-op. + ray_sig = _job(serve_args=RAY, nnodes="2", pp="1").server_signature() + self.assertNotIn("--node-rank", ray_sig[0]) + + def test_signature_pins_actual_argv_content_not_a_constant(self): + # Rejects a hard-coded/degenerate server_signature(): the signature must + # actually contain the job's real server argv (rank-0, --node-rank + # stripped) and the real env map, not an opaque constant. + job = _job(serve_args=RAY, nnodes="2", pp="1") + expected_argv = list(job._server_argv(0)) + if "--node-rank" in expected_argv: + i = expected_argv.index("--node-rank") + del expected_argv[i : i + 2] + expected_env = tuple(sorted((str(k), str(v)) for k, v in job.server_env.items())) + sig = job.server_signature() + self.assertEqual(sig, (tuple(expected_argv), expected_env)) + self.assertIn("--tensor-parallel-size", sig[0]) + self.assertIn("--distributed-executor-backend", sig[0]) + + def test_signature_env_is_independently_sorted_and_str_cast(self): + # Round-1 finding 2: the pin test above derives its expected env tuple with + # the SAME sorted((str(k),str(v)) ...) expression as production, so the env + # half of that assertion is tautological -- a bug in that exact transform + # (wrong sort key, missing str() cast, unsorted output) would reproduce in + # both sides and still pass. Here the expected env is an INDEPENDENTLY + # hard-coded literal, and server_env is populated with multiple out-of-order + # keys plus a non-string value, so the assertion actually verifies: (a) keys + # are sorted, (b) both key and value are str()-cast (the int 3 -> "3"), (c) + # the result is a tuple of (str, str) pairs. No other test in the suite + # exercises server_env with >1 entry, so this is the sole real coverage of + # that transform. + job = _job(serve_args=RAY, nnodes="2", pp="1") + # Deliberately out-of-order insertion order; "MID" maps to an int to force str(). + job.server_env = {"ZEBRA": "z1", "ALPHA": "a1", "MID": 3} + # Independently-constructed literal oracle (not re-derived from server_env). + expected_env = (("ALPHA", "a1"), ("MID", "3"), ("ZEBRA", "z1")) + sig = job.server_signature() + self.assertEqual(sig[1], expected_env) + + def test_differing_tensor_parallelism_yields_different_ray_signature(self): + # A ray job differing in a server-affecting field (tp) must NOT share a + # signature with another ray job — otherwise an incompatible server + # would be wrongly reused across cells. + self.assertNotEqual( + _job(serve_args=RAY, nnodes="2", pp="1", tp="4").server_signature(), + _job(serve_args=RAY, nnodes="2", pp="1", tp="8").server_signature(), + ) + + def test_differing_model_id_yields_different_ray_signature(self): + # model_id is always in argv regardless of backend (unlike master_addr, + # which ray legitimately omits per AC16 / _DRIVER_DIST_FLAGS above). + job_a = _job(serve_args=RAY, nnodes="2", pp="1") + job_b = _job(serve_args=RAY, nnodes="2", pp="1") + job_b.model_id = "/models/a-different-model" + self.assertNotEqual(job_a.server_signature(), job_b.server_signature()) + + +# --------------------------------------------------------------------------- # +# is_ready (Round-2 finding 2: previously zero direct coverage) +# --------------------------------------------------------------------------- # +class TestVllmJobIsReady(unittest.TestCase): + """is_ready() greps rank-0's readiness log via orch.exec(detailed=True) and + returns True iff the collected result is non-empty AND every grepped rank's + exit_code == 0 (exit 0 = readiness pattern found). rank>0 workers are skipped + when int(nnodes) > 1 (the pre-existing guard, NOT ray-gated -- spec RC9). + These tests pin the True path, both False paths (non-zero exit, empty result), + and the multi-node worker-skip; none of it was exercised before (is_ready is + never called by the start_server tests).""" + + def test_true_when_readiness_found(self): + orch = RecordingOrch(responder=_responder_readiness(exit_code=0), hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp="1", ib_netdev=None) + self.assertTrue(job.is_ready()) + + def test_false_when_readiness_absent(self): + orch = RecordingOrch(responder=_responder_readiness(exit_code=1), hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp="1", ib_netdev=None) + self.assertFalse(job.is_ready()) + + def test_false_when_result_empty(self): + # `not out` branch: an empty/None exec result must read as NOT ready. + orch = RecordingOrch(responder=_responder_readiness(empty=True), hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp="1", ib_netdev=None) + self.assertFalse(job.is_ready()) + + def test_multinode_skips_workers_and_only_checks_rank0(self): + # nnodes=2: only rank 0 (head) is grepped; the rank-1 worker is skipped + # (rank>0 & nnodes>1 guard), so readiness is decided from rank 0 alone. + orch = RecordingOrch(responder=_responder_readiness(exit_code=0)) + job = _job(orch=orch, serve_args={}, nnodes="2", pp="2") + self.assertTrue(job.is_ready()) + self.assertEqual(_calls_to(orch, WORKER), [], "rank>0 worker must be skipped by is_ready under nnodes>1") + self.assertTrue(_calls_to(orch, HEAD), "rank 0 head readiness log must be grepped") + + +# --------------------------------------------------------------------------- # +# parse_results (Round-2 finding 3: previously zero coverage) +# --------------------------------------------------------------------------- # +class TestVllmJobParseResults(unittest.TestCase): + """parse_results() fetches the client results artifact via orch.exec_on_head + (which returns {host: content}), json-loads it, and returns + to_client_metrics(raw, tp=self.tp, isl=self.isl, pp=self.pp) per host. Two + documented exception modes: empty/missing artifact -> RuntimeError; + unparseable JSON -> RuntimeError. Exception assertions pin the TYPE only + (message text is an implementation detail per the authoring anti-patterns). + The happy path pins the delegation to to_client_metrics with the correct + keyword-only tp/isl/pp.""" + + def test_empty_artifact_raises_runtimeerror(self): + orch = RecordingOrch(head_responder=lambda cmd: {HEAD: ""}, hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp="1", ib_netdev=None) + with self.assertRaises(RuntimeError): + job.parse_results() + + def test_unparseable_json_raises_runtimeerror(self): + orch = RecordingOrch(head_responder=lambda cmd: {HEAD: "not-json{"}, hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp="1", ib_netdev=None) + with self.assertRaises(RuntimeError): + job.parse_results() + + def test_unparseable_artifact_snippet_is_single_line(self): + # Deliberate departure from this class's "pin the TYPE only" rule: the + # snippet is carried into the message precisely so a failure stays + # diagnosable once print_console=False keeps the artifact out of the + # log, so its SHAPE is the behaviour under test, not incidental + # phrasing. A raw multi-line artifact (a stack trace, or HTML from a + # proxy error page) would otherwise inject newlines straight into CI + # output and Jira ticket bodies, where the failure text is pasted. + artifact = 'oh no\nline two\rline three\tand a tab' + orch = RecordingOrch(head_responder=lambda cmd: {HEAD: artifact}, hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp="1", ib_netdev=None) + with self.assertRaises(RuntimeError) as ctx: + job.parse_results() + + message = str(ctx.exception) + self.assertNotIn("\n", message, f"raw newline leaked into the error text: {message!r}") + self.assertNotIn("\r", message, f"raw carriage return leaked into the error text: {message!r}") + # Escaped, not dropped -- the content still has to be recoverable. + self.assertIn("line two", message) + + def test_valid_artifact_delegates_to_to_client_metrics_with_tp_isl_pp(self): + # tp, isl, and pp are keyword-only in to_client_metrics, so they MUST + # arrive as kwargs; raw (the json-loaded artifact) arrives positionally. + # Patching the symbol as imported into vllm_job keeps this impl-blind on + # the metric math. + # + # Round-3 finding 1: capture and assert the RETURN VALUE, not just that the + # mock was called with the right args. Production threads the metric result + # back out as {host: to_client_metrics(...)}; a mutant that calls + # to_client_metrics for its side effect but then stores `raw` (or the wrong + # host key, or returns early) would satisfy a call-args-only check while + # breaking the actual output. The mock's return_value is the independent + # oracle for what must appear under the head host key. + # + # Post-mortem finding (Spec A1, loop 1): the prior version of this test + # asserted tp/isl only, so a broken pp passthrough at this call site + # (e.g. AC6's pp=self.pp regressing to a hardcoded value) would slip + # through silently. Assert pp explicitly, and vary it across a subTest + # so a mutant that ignores job.pp entirely is also caught. + import json as _json + + raw = {"output_throughput": 1234.0, "request_goodput": 10.0} + sentinel = {"client.sentinel": 1} + for pp in ("1", "2"): + with self.subTest(pp=pp): + orch = RecordingOrch(head_responder=lambda cmd: {HEAD: _json.dumps(raw)}, hosts=[HEAD]) + job = _job(orch=orch, serve_args={}, nnodes="1", pp=pp, ib_netdev=None, isl="1024") + with mock.patch("cvs.lib.inference.vllm_job.to_client_metrics") as m_tcm: + m_tcm.return_value = sentinel + result = job.parse_results() + self.assertTrue(m_tcm.called, "parse_results must delegate to to_client_metrics") + args, kwargs = m_tcm.call_args + self.assertEqual(kwargs.get("tp"), job.tp) + self.assertEqual(kwargs.get("isl"), job.isl) + self.assertEqual(kwargs.get("pp"), job.pp) + self.assertEqual(args[0], raw, "raw must be the json-loaded artifact passed positionally") + # The metric result must be threaded back out under the head host key -- + # NOT the raw artifact, and NOT dropped/re-keyed. + self.assertEqual(result, {HEAD: sentinel}) + + +# --------------------------------------------------------------------------- # +# wait_ready (Round-3 finding 2: the readiness polling state machine had zero +# coverage -- it is the real caller of is_ready() and _check_early_failure()) +# --------------------------------------------------------------------------- # +@mock.patch("cvs.lib.inference.vllm_job.time.sleep") +class TestVllmJobWaitReady(unittest.TestCase): + """wait_ready() drives the sequence: precheck-wait -> early-failure-check -> + warmup-wait -> early-failure-check -> poll-loop(is_ready) -> RuntimeError on + timeout. is_ready() and _check_early_failure() are unit-tested in isolation + elsewhere; here they are mocked on the instance so the ORCHESTRATION itself is + what is exercised: that is_ready is actually polled, that an exhausted poll + budget raises (not swallowed), that the early-failure check runs before the + poll loop, and that a failure surfaced during warmup aborts before polling. + time.sleep is patched at the module seam so no real waiting occurs.""" + + def test_returns_when_ready_and_stops_polling(self, mock_sleep): + # Happy path: is_ready flips True on the 3rd poll; wait_ready must return + # (no raise) and must stop polling immediately once ready (the side_effect + # list has no 4th element, so a spurious extra poll raises StopIteration). + job = _job(serve_args={}, nnodes="1", pp="1", ib_netdev=None) + job._check_early_failure = mock.Mock() + job.is_ready = mock.Mock(side_effect=[False, False, True]) + try: + job.wait_ready() + except Exception as e: # pragma: no cover - failure path + self.fail(f"wait_ready must return once is_ready() is True, raised: {e!r}") + self.assertEqual(job.is_ready.call_count, 3, "wait_ready must poll is_ready until it returns True, then stop") + self.assertTrue(job._check_early_failure.called, "wait_ready must run the early-failure check") + + def test_timeout_raises_after_exhausting_poll_budget(self, mock_sleep): + # Liveness/termination: is_ready never becomes True. wait_ready must NOT + # spin forever and must NOT swallow the failure -- it raises RuntimeError + # once the poll budget (server_poll_count) is exhausted, having polled + # is_ready exactly server_poll_count times. + # server_poll_count is bound at construction, so set it via the documented + # constructor parameter (a small budget keeps the test fast and pins the + # expected poll count without depending on the internal attribute name). + poll_count = 3 + job = VllmJob( + orch=RecordingOrch(responder=_responder_ok(), hosts=[HEAD]), + variant=_variant(serve_args={}, nnodes="1", pp="1", ib_netdev=None), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=16, + num_prompts="640", + server_poll_count=poll_count, + ) + job._check_early_failure = mock.Mock() + job.is_ready = mock.Mock(return_value=False) + with self.assertRaises(RuntimeError): + job.wait_ready() + self.assertEqual( + job.is_ready.call_count, + poll_count, + "on timeout wait_ready must have polled is_ready exactly server_poll_count times", + ) + + def test_early_failure_check_runs_before_polling(self, mock_sleep): + # Ordering: the early-failure check must precede the is_ready poll loop, so a + # crash detectable in the log is surfaced before spending the poll budget. + job = _job(serve_args={}, nnodes="1", pp="1", ib_netdev=None) + order = [] + job._check_early_failure = mock.Mock(side_effect=lambda *a, **k: order.append("check")) + job.is_ready = mock.Mock(side_effect=lambda: (order.append("ready"), True)[1]) + job.wait_ready() + self.assertIn("check", order, "the early-failure check must be invoked") + self.assertIn("ready", order, "is_ready must be polled") + self.assertEqual(order[0], "check", "early-failure check must run before the first is_ready poll") + + def test_failure_detected_during_warmup_aborts_before_polling(self, mock_sleep): + # If _check_early_failure raises (a fatal log line found during precheck/ + # warmup), wait_ready must propagate it and NOT proceed to poll is_ready -- + # the server is already known dead. + job = _job(serve_args={}, nnodes="1", pp="1", ib_netdev=None) + job._check_early_failure = mock.Mock(side_effect=RuntimeError("vllm server fatal error (rank 0)")) + job.is_ready = mock.Mock(return_value=True) + with self.assertRaises(RuntimeError): + job.wait_ready() + self.assertFalse( + job.is_ready.called, + "a failure surfaced by _check_early_failure must abort wait_ready before the poll loop", + ) + + +# --------------------------------------------------------------------------- # +# build_server_cmd (Round-3 finding 3: env-script construction had zero direct +# coverage -- multiple conditional branches forming clear equivalence classes) +# --------------------------------------------------------------------------- # +class TestVllmJobBuildServerCmd(unittest.TestCase): + """build_server_cmd() writes an env-script (broadcast to all nodes) and issues + per-rank mkdir commands. Its documented equivalence classes (per discipline + rule B, driven by subTest tables): ib_hcas present vs empty/None (the + NCCL_IB_HCA line is emitted or not), ib_netdev present vs None (the socket + interface exports are emitted or not), the server_env pass-through loop, and the + per-rank mkdir loop bounded by nnodes. Assertions target stable, named tokens + (NCCL_IB_HCA, SOCKET_IFNAME, the env keys, mkdir) rather than exact shell + formatting, and are collected from every command the method emits (whether + issued via orch.exec or returned) so the test does not couple to the transport + detail of how the script is delivered.""" + + @staticmethod + def _script(orch, ret): + parts = list(_all_cmds(orch)) + if isinstance(ret, str): + parts.append(ret) + elif isinstance(ret, (list, tuple)): + parts.extend(str(x) for x in ret) + return "\n".join(parts) + + def test_nccl_ib_hca_line_present_only_when_ib_hcas_supplied(self): + # (ib_hcas, hca_present) -- the NCCL_IB_HCA export is gated on a non-empty + # ib_hcas list; empty list and None must NOT emit it. + cases = [ + (["mlx5_0", "mlx5_1"], True), + ([], False), + (None, False), + ] + for ib_hcas, present in cases: + with self.subTest(ib_hcas=ib_hcas): + orch = RecordingOrch() + job = _job(orch=orch, serve_args={}, nnodes="2", pp="2", ib_hcas=ib_hcas) + ret = job.build_server_cmd() + script = self._script(orch, ret) + if present: + self.assertIn("NCCL_IB_HCA", script) + # The supplied HCA name must actually reach the export value. + self.assertIn("mlx5_0", script) + else: + self.assertNotIn("NCCL_IB_HCA", script) + + def test_socket_ifname_exports_present_only_when_ib_netdev_set(self): + # ib_netdev set -> the socket-interface exports are emitted (all three name + # the device); ib_netdev None -> none are emitted. + orch_set = RecordingOrch() + _job(orch=orch_set, serve_args={}, nnodes="2", pp="2", ib_netdev="eth0").build_server_cmd() + script_set = self._script(orch_set, None) + self.assertEqual( + script_set.count("SOCKET_IFNAME"), + 3, + "ib_netdev must emit exactly the three socket-ifname exports", + ) + self.assertIn("eth0", script_set, "the configured ib_netdev must reach the export value") + + orch_none = RecordingOrch() + _job(orch=orch_none, serve_args={}, nnodes="2", pp="2", ib_netdev=None).build_server_cmd() + script_none = self._script(orch_none, None) + self.assertNotIn("SOCKET_IFNAME", script_none, "no ib_netdev -> no socket-ifname exports") + + def test_server_env_entries_passed_through(self): + # Every server_env key/value must appear in the emitted env-script (the + # pass-through loop). Two entries so a single-entry short-circuit is caught. + # + # Round-4 finding 1: the values MUST be distinctive strings that cannot + # collide with any boilerplate line the env-script also emits. A bare value + # like "1" trivially matches elsewhere (e.g. "...AITER_UNIFIED_ATTENTION=1"), + # so assertIn("1", script) is vacuous -- a mutant that hard-codes a wrong + # value or drops the CUSTOM_A line entirely still passes. Using unique + # values AND asserting the "KEY=VALUE" pairing (not the bare value) pins + # both the presence and the key/value association without coupling to the + # exact "export " prefix formatting. + orch = RecordingOrch() + job = _job( + orch=orch, + serve_args={}, + nnodes="2", + pp="2", + env={"CUSTOM_A": "CUSTOM_A_VALUE_XYZ", "CUSTOM_B": "CUSTOM_B_VALUE_QRS"}, + ) + ret = job.build_server_cmd() + script = self._script(orch, ret) + for key, val in (("CUSTOM_A", "CUSTOM_A_VALUE_XYZ"), ("CUSTOM_B", "CUSTOM_B_VALUE_QRS")): + with self.subTest(key=key): + self.assertIn(f"{key}={val}", script, f"server_env {key} must be exported paired with its value") + + def test_mkdir_count_scales_with_nnodes(self): + # The per-rank mkdir loop is bounded by nnodes: a 3-node job must issue + # strictly more mkdir commands than a single-node job (all else equal). + orch1 = RecordingOrch(hosts=[HEAD]) + _job(orch=orch1, serve_args={}, nnodes="1", pp="1", ib_netdev=None).build_server_cmd() + orch3 = RecordingOrch(hosts=[HEAD, WORKER, HOST2]) + _job(orch=orch3, serve_args={}, nnodes="3", pp="1", ib_netdev="eth0").build_server_cmd() + mk1 = self._script(orch1, None).count("mkdir") + mk3 = self._script(orch3, None).count("mkdir") + self.assertGreater(mk1, 0, "build_server_cmd must create at least the rank-0 log dir") + self.assertGreater(mk3, mk1, "per-rank mkdir loop must scale with nnodes") + + +# --------------------------------------------------------------------------- # +# Lifecycle (transition table) +# --------------------------------------------------------------------------- # +# | from state | event | to state / effect | +# |----------------------|---------------------------|---------------------------------------| +# | constructed | start_server() [ok ray] | bootstrap head+worker, serve on head | +# | started | start_server() again | re-entrant: no raise, re-launch serve | +# | started | stop_server() | pkill + ray stop broadcast -> down | +# | bootstrap-failed | start_server() [head bad] | RuntimeError, no serve (illegal txn) | +# | bootstrap-failed | stop_server() | must NOT raise (partial cleanup) | +# | down | stop_server() again | idempotent no-op, no raise | +@mock.patch("cvs.lib.inference.vllm_job.time.sleep") +class TestVllmJobRayLifecycle(unittest.TestCase): + def test_legal_start_then_stop(self, mock_sleep): + orch = RecordingOrch(responder=_responder_ok()) + job = _job(orch=orch, serve_args=RAY, nnodes="2", pp="1") + job.start_server() # constructed -> started + job.stop_server() # started -> down + self.assertTrue([c for c in _calls_to(orch, HEAD) if "vllm serve" in c]) + self.assertTrue([c for c, _ in orch.calls if "ray stop" in c]) + + def test_illegal_start_on_bad_bootstrap_is_rejected(self, mock_sleep): + orch = RecordingOrch(responder=_responder_bootstrap_fail({HEAD: {"exit_code": 1, "output": _BAD}})) + job = _job(orch=orch, serve_args=RAY, nnodes="2", pp="1") + with self.assertRaises(RuntimeError): + job.start_server() + + def test_stop_after_failed_start_does_not_raise(self, mock_sleep): + # Regression: partial bootstrap -> caller must be able to stop_server safely. + orch = RecordingOrch(responder=_responder_bootstrap_fail({HEAD: {"exit_code": 1, "output": _BAD}})) + job = _job(orch=orch, serve_args=RAY, nnodes="2", pp="1") + with self.assertRaises(RuntimeError): + job.start_server() + calls_before_stop = len(orch.calls) + try: + job.stop_server() # must be robust after a failed/partial start + except Exception as e: # pragma: no cover - failure path + self.fail(f"stop_server after failed start raised: {e!r}") + # Finding 9: not merely "no exception" -- assert stop_server actually did + # the full teardown after the partial start. Both broadcasts (hosts=None) + # must be issued: the pkill vllm serve, then (nnodes>1 & ray) ray stop. + stop_calls = orch.calls[calls_before_stop:] + pkill = [(c, h) for c, h in stop_calls if "pkill" in c and "vllm serve" in c] + ray_stop = [(c, h) for c, h in stop_calls if "ray stop" in c] + self.assertEqual(len(pkill), 1, f"expected one pkill broadcast on stop, got {pkill}") + self.assertIsNone(pkill[0][1], "pkill must be a broadcast (hosts=None)") + self.assertEqual(len(ray_stop), 1, f"expected one ray stop broadcast on stop, got {ray_stop}") + self.assertIsNone(ray_stop[0][1], "ray stop must be a broadcast (hosts=None)") + + def test_reentrant_start_is_not_rejected(self, mock_sleep): + # Coverage-gap (finding 2): the transition table had a legal start and an + # idempotent stop-reentry, but no started -> start_server()-again case. The + # spec documents no idempotency guard / no documented error on re-entrant + # start, so a second start_server() must not raise and re-runs the bootstrap + # + serve sequence (mirroring the re-run semantics of the stop reentry test: + # each teardown issues its own ray stop, so each start issues its own serve). + orch = RecordingOrch(responder=_responder_ok()) + job = _job(orch=orch, serve_args=RAY, nnodes="2", pp="1") + job.start_server() # constructed -> started + try: + job.start_server() # started -> start again (re-entrant) + except Exception as e: # pragma: no cover - failure path + self.fail(f"re-entrant start_server raised: {e!r}") + head_serves = [c for c in _calls_to(orch, HEAD) if "vllm serve" in c] + self.assertEqual( + len(head_serves), 2, f"each start_server must (re-)launch vllm serve on the head; got {head_serves}" + ) + + def test_idempotent_stop_reentry(self, mock_sleep): + orch = RecordingOrch(responder=_responder_ok()) + job = _job(orch=orch, serve_args=RAY, nnodes="2", pp="1") + job.start_server() + job.stop_server() + try: + job.stop_server() # cleanup twice must not raise + except Exception as e: # pragma: no cover - failure path + self.fail(f"second stop_server raised: {e!r}") + # Each teardown issues its own ray stop broadcast. + self.assertEqual(len([c for c, _ in orch.calls if "ray stop" in c]), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_vllm_job_server_reuse.py b/cvs/lib/inference/unittests/test_vllm_job_server_reuse.py new file mode 100644 index 000000000..d35acf06f --- /dev/null +++ b/cvs/lib/inference/unittests/test_vllm_job_server_reuse.py @@ -0,0 +1,444 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.inference.vllm_job.VllmJob server-command construction: + - the duplicate --max-model-len fix (config-pin suppresses the derived value) + - server_signature(), which gates cross-cell server reuse + - _flatten_serve_args boolean handling and log-level pass-through + - _check_early_failure tail emission and CLI parse error detection + - RoleServer.serve_args log-level validator + - probe_openai_endpoints(), the OpenAI-compatible HTTP smoke probe +''' + +import json +import unittest +import unittest.mock as mock +from types import SimpleNamespace + +import pydantic + +from cvs.lib.inference.utils.vllm_config_loader import RoleServer +from cvs.lib.inference.vllm_job import VllmJob + +_TP = 8 +_PP = 2 +_NNODES = 2 + + +class FakeOrch: + hosts = ["10.0.0.1", "10.0.0.2"] + + def __init__(self): + self.head_cmds = [] + + def exec(self, *a, **k): + return {} + + def exec_on_head(self, cmd, *a, **k): + self.head_cmds.append(cmd) + return {} + + +class FakeOrchWithOutput: + """Single-rank fake orch that returns controllable tail/grep output.""" + + hosts = ["10.0.0.1"] + + def __init__(self, tail_output="", grep_exit=1): + self.head_cmds = [] + self._tail_output = tail_output + self._grep_exit = grep_exit # 1 = no match (safe), 0 = match found + + def exec(self, cmd, hosts=None, detailed=False, print_console=True): + if detailed: + return {"10.0.0.1": {"exit_code": self._grep_exit, "stdout": ""}} + return {"10.0.0.1": self._tail_output} + + def exec_on_head(self, cmd, *a, **k): + self.head_cmds.append(cmd) + return {} + + +class FakeOrchMultiHost: + """Two-host fake orch that records which hosts each exec() call targeted.""" + + hosts = ["10.0.0.1", "10.0.0.2"] + + def __init__(self): + self.exec_calls = [] # list of (cmd, hosts) actually issued + + def exec(self, cmd, hosts=None, detailed=False, print_console=True): + self.exec_calls.append((cmd, hosts)) + host = hosts[0] + return {host: f"content for {host}"} + + def exec_on_head(self, cmd, *a, **k): + return {} + + +def _make_job_for_check(tail_output="", grep_exit=1): + """Construct a VllmJob suitable for testing _check_early_failure.""" + variant = mock.MagicMock() + variant.params.tensor_parallelism = "8" + variant.params.pipeline_parallel_size = "1" + variant.params.master_addr = "localhost" + variant.params.master_port = "29501" + variant.params.nnodes = "1" + variant.params.port_no = "8000" + variant.params.random_range_ratio = "0.0" + variant.params.random_prefix_len = "0" + variant.params.burstiness = "1.0" + variant.params.seed = "0" + variant.params.request_rate = "inf" + variant.params.tokenizer_mode = "auto" + variant.params.percentile_metrics = "ttft,tpot,itl,e2el" + variant.params.metric_percentiles = "50,90,95,99" + variant.params.base_url = "http://0.0.0.0" + variant.params.dataset_name = "random" + variant.params.backend = "vllm" + variant.model.id = "/models/test-model" + variant.paths.log_dir = "/tmp/test_logs" + variant.paths.models_dir = "/tmp/models" + variant.roles.server.serve_args = {} + variant.roles.server.env = {} + variant.roles.server.ib_netdev = None + orch = FakeOrchWithOutput(tail_output=tail_output, grep_exit=grep_exit) + return VllmJob( + orch=orch, + variant=variant, + hf_token="tok", + isl="1024", + osl="1024", + concurrency="8", + num_prompts="100", + ) + + +def _variant(serve_args=None): + params = SimpleNamespace( + tensor_parallelism=str(_TP), + pipeline_parallel_size=str(_PP), + master_addr="10.0.0.1", + master_port="29501", + nnodes=str(_NNODES), + port_no="8000", + random_range_ratio="0.8", + random_prefix_len="0", + burstiness="1.0", + seed="0", + request_rate="inf", + tokenizer_mode="auto", + percentile_metrics="ttft,tpot,itl,e2el", + metric_percentiles="50,90,95,99", + base_url="http://0.0.0.0", + dataset_name="random", + backend="vllm", + ) + return SimpleNamespace( + params=params, + model=SimpleNamespace(id="/models/Kimi-K2.5-W4A8"), + paths=SimpleNamespace(log_dir="/logs", models_dir="/models"), + roles=SimpleNamespace( + server=SimpleNamespace( + serve_args=dict(serve_args or {}), + env={"VLLM_ROCM_USE_AITER": "1"}, + ib_netdev="enp159s0np0", + ) + ), + ) + + +def _job(isl, osl, conc, serve_args=None): + return VllmJob( + orch=FakeOrch(), + variant=_variant(serve_args), + hf_token="tok", + isl=isl, + osl=osl, + concurrency=conc, + num_prompts="640", + ) + + +class TestMaxModelLenNoDuplicate(unittest.TestCase): + def test_config_pin_wins_and_no_duplicate(self): + argv = _job("1024", "1024", 16, serve_args={"max-model-len": "16384"})._server_argv(0) + idxs = [i for i, a in enumerate(argv) if a == "--max-model-len"] + self.assertEqual(len(idxs), 1, "config-pinned max-model-len must appear exactly once") + self.assertEqual(argv[idxs[0] + 1], "16384", "config value must win") + + def test_derived_emitted_when_not_pinned(self): + argv = _job("1024", "1024", 16, serve_args={})._server_argv(0) + idxs = [i for i, a in enumerate(argv) if a == "--max-model-len"] + self.assertEqual(len(idxs), 1, "derived max-model-len must still be emitted when unpinned") + # 1024+1024 worst-case derived value, definitely not the 16384 config value + self.assertNotEqual(argv[idxs[0] + 1], "16384") + + +class TestServerSignatureReuse(unittest.TestCase): + def test_invariant_to_concurrency(self): + # Pinned max-model-len: cells differing only in concurrency share a server. + sa = {"max-model-len": "16384"} + self.assertEqual( + _job("1024", "1024", 4, sa).server_signature(), + _job("1024", "1024", 64, sa).server_signature(), + ) + + def test_pinned_mml_shares_across_isl_osl(self): + # With a fixed max-model-len, ISL/OSL never reach the server argv, so all + # cells legitimately share one server (ISL/OSL are client-only knobs). + sa = {"max-model-len": "16384"} + self.assertEqual( + _job("1024", "1024", 16, sa).server_signature(), + _job("8192", "1024", 16, sa).server_signature(), + ) + + def test_derived_mml_distinguishes_osl(self): + # Without a pin, max-model-len is derived per (isl+osl); different OSL must + # change the signature so a real restart happens. + self.assertNotEqual( + _job("1024", "1024", 16, serve_args={}).server_signature(), + _job("1024", "8192", 16, serve_args={}).server_signature(), + ) + + def test_signature_strips_node_rank_and_is_hashable(self): + job = _job("1024", "1024", 16, serve_args={"max-model-len": "16384"}) + self.assertIn("--node-rank", job._server_argv(0)) + sig = job.server_signature() + self.assertNotIn("--node-rank", sig[0]) + # hashable + stable + self.assertEqual(hash(sig), hash(job.server_signature())) + + +class TestRunClientEnsuresOutDir(unittest.TestCase): + """The server-reuse path skips build_server_cmd (which creates the per-cell + out_dir), so run_client must create its own out_dir or the client's + client.log/results writes fail with 'No such file or directory'.""" + + def test_run_client_mkdirs_out_dir(self): + job = _job("1024", "1024", 8, serve_args={"max-model-len": "16384"}) + job.run_client() + mkdir_cmds = [c for c in job.orch.head_cmds if "mkdir -p" in c and job.out_dir in c] + self.assertTrue( + mkdir_cmds, + f"run_client must mkdir -p its out_dir ({job.out_dir}) so the reuse path " + f"(which skips build_server_cmd) can still write client.log; head cmds: {job.orch.head_cmds}", + ) + + +class TestRunClientTrustRemoteCode(unittest.TestCase): + """Models with a custom tokenizer (e.g. Kimi-K2.6's auto_map) need the bench + client to pass --trust-remote-code, mirroring the server's serve_args, or the + client's tokenizer load raises ValueError before any request is sent.""" + + def _bench_cmd(self, job): + job.run_client() + bench = [c for c in job.orch.head_cmds if "vllm" in c and "bench" in c] + self.assertTrue(bench, f"no bench client command issued; head cmds: {job.orch.head_cmds}") + return bench[-1] + + def test_trust_remote_code_passed_when_server_enables_it(self): + job = _job("1024", "1024", 8, serve_args={"max-model-len": "16384", "trust-remote-code": True}) + self.assertIn("--trust-remote-code", self._bench_cmd(job)) + + def test_trust_remote_code_absent_when_server_omits_it(self): + job = _job("1024", "1024", 8, serve_args={"max-model-len": "16384"}) + self.assertNotIn("--trust-remote-code", self._bench_cmd(job)) + + +class TestFlattenServeArgsFalse(unittest.TestCase): + def test_false_value_omitted(self): + result = VllmJob._flatten_serve_args({"enable-prefix-caching": False, "tensor-parallel-size": "8"}) + self.assertNotIn("--enable-prefix-caching", result) + self.assertNotIn("False", result) + self.assertEqual(result, ["--tensor-parallel-size", "8"]) + + def test_true_value_emits_flag_only(self): + result = VllmJob._flatten_serve_args({"enforce-eager": True}) + self.assertEqual(result, ["--enforce-eager"]) + + def test_log_level_passed_through(self): + result = VllmJob._flatten_serve_args({"log-level": "debug"}) + self.assertEqual(result, ["--log-level", "debug"]) + + +class TestCheckEarlyFailureEmitTail(unittest.TestCase): + def test_emit_tail_true_logs_content(self): + job = _make_job_for_check(tail_output="INFO engine loading\nINFO weights done") + with mock.patch("cvs.lib.inference.vllm_job.log") as mock_log: + job._check_early_failure(emit_tail=True) + logged_lines = [call.args[3] for call in mock_log.info.call_args_list if len(call.args) >= 4] + self.assertIn("INFO engine loading", logged_lines) + self.assertIn("INFO weights done", logged_lines) + + def test_raises_on_cli_parse_error(self): + job = _make_job_for_check(tail_output="vllm: error: unrecognized arguments: False") + with self.assertRaises(RuntimeError): + job._check_early_failure() + + +class TestDumpServerLog(unittest.TestCase): + def test_logs_full_content_per_rank(self): + job = _make_job_for_check(tail_output="line one\nline two") + with mock.patch("cvs.lib.inference.vllm_job.log") as mock_log: + job.dump_server_log() + logged_lines = [call.args[3] for call in mock_log.info.call_args_list if len(call.args) >= 4] + self.assertIn("line one", logged_lines) + self.assertIn("line two", logged_lines) + + def test_mp_multinode_dumps_every_rank(self): + """mp backend: every rank runs its own vllm serve, so every rank is dumped.""" + orch = FakeOrchMultiHost() + job = VllmJob( + orch=orch, + variant=_variant({"distributed-executor-backend": "mp"}), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=8, + num_prompts="640", + ) + with mock.patch("cvs.lib.inference.vllm_job.log") as mock_log: + job.dump_server_log() + ranks_dumped = {call.args[2] for call in mock_log.info.call_args_list if len(call.args) >= 4} + self.assertEqual(ranks_dumped, {0, 1}) + self.assertEqual(len(orch.exec_calls), 2, "one cat per rank") + + def test_ray_multinode_skips_worker_ranks(self): + """Ray multinode: only rank 0 runs vllm serve, so only rank 0 is dumped.""" + orch = FakeOrchMultiHost() + job = VllmJob( + orch=orch, + variant=_variant({"distributed-executor-backend": "ray"}), + hf_token="tok", + isl="1024", + osl="1024", + concurrency=8, + num_prompts="640", + ) + with mock.patch("cvs.lib.inference.vllm_job.log") as mock_log: + job.dump_server_log() + ranks_dumped = {call.args[2] for call in mock_log.info.call_args_list if len(call.args) >= 4} + self.assertEqual(ranks_dumped, {0}, "worker rank 1 has no server log under ray and must be skipped") + self.assertEqual(len(orch.exec_calls), 1, "only rank 0's cat should be issued") + + +class TestRoleServerLogLevelValidator(unittest.TestCase): + def test_invalid_log_level_rejected(self): + with self.assertRaises(pydantic.ValidationError) as ctx: + RoleServer(serve_args={"log-level": "verbose"}) + msg = str(ctx.exception) + self.assertIn("log-level", msg) + self.assertIn("verbose", msg) + + def test_valid_log_level_accepted(self): + rs = RoleServer(serve_args={"log-level": "debug"}) + self.assertEqual(rs.serve_args["log-level"], "debug") + + +class FakeOrchWithHeadOutput: + """Single-rank fake orch whose exec_on_head returns a controllable string, + mirroring what `orch.exec_on_head` would ship back from the container.""" + + hosts = ["10.0.0.1"] + + def __init__(self, head_output=""): + self.head_cmds = [] + self._head_output = head_output + + def exec(self, *a, **k): + return {} + + def exec_on_head(self, cmd, *a, **k): + self.head_cmds.append(cmd) + return {"10.0.0.1": self._head_output} + + +class TestProbeOpenAIEndpoints(unittest.TestCase): + """Unit tests for VllmJob.probe_openai_endpoints. No hardware: FakeOrchWithHeadOutput + returns a canned base64-decoded-script's stdout line (the JSON dict the + stdlib probe script prints), mirroring what `orch.exec_on_head` would ship back + from the container.""" + + _GOOD_BODY = { + "model": "amd/Llama-3.1-70B-Instruct-FP8-KV", + "choices": [{"message": {"content": "OK"}, "text": "Paris"}], + } + _BOOK_CONTENT = json.dumps({"title": "T", "author": "A", "year": 2000, "genre": "G"}) + + def _raw(self, results): + return json.dumps(results) + + def _all_pass_results(self): + return { + "model_endpoint": [200, {"data": [{"id": "amd/Llama-3.1-70B-Instruct-FP8-KV"}]}], + "chat_completion_endpoint": [200, {**self._GOOD_BODY, "choices": [{"message": {"content": "OK"}}]}], + "completion_endpoint": [200, {**self._GOOD_BODY, "choices": [{"text": "Paris"}]}], + "structured_output_book": [ + 200, + {**self._GOOD_BODY, "choices": [{"message": {"content": self._BOOK_CONTENT}}]}, + ], + } + + def test_issues_single_head_exec_with_port_and_model(self): + orch = FakeOrchWithHeadOutput(head_output=self._raw(self._all_pass_results())) + job = _job("1024", "1024", 1, serve_args={"max-model-len": "16384"}) + job.orch = orch + job.probe_openai_endpoints() + self.assertEqual(len(orch.head_cmds), 1) + cmd = orch.head_cmds[0] + self.assertIn("base64 -d", cmd) + self.assertIn("python3", cmd) + + def test_all_pass_returns_summary_lines(self): + orch = FakeOrchWithHeadOutput(head_output=self._raw(self._all_pass_results())) + job = _job("1024", "1024", 1, serve_args={"max-model-len": "16384"}) + job.orch = orch + summary = job.probe_openai_endpoints() + self.assertEqual(len(summary), 4) + for line in summary: + self.assertIn("-> Pass (200)", line) + + def test_http_failure_raises(self): + results = self._all_pass_results() + results["model_endpoint"] = [500, {"error": "boom"}] + orch = FakeOrchWithHeadOutput(head_output=self._raw(results)) + job = _job("1024", "1024", 1, serve_args={"max-model-len": "16384"}) + job.orch = orch + with self.assertRaises(RuntimeError): + job.probe_openai_endpoints() + + def test_empty_content_raises(self): + results = self._all_pass_results() + results["chat_completion_endpoint"][1]["choices"] = [{"message": {"content": ""}}] + orch = FakeOrchWithHeadOutput(head_output=self._raw(results)) + job = _job("1024", "1024", 1, serve_args={"max-model-len": "16384"}) + job.orch = orch + with self.assertRaises(RuntimeError): + job.probe_openai_endpoints() + + def test_no_output_raises(self): + orch = FakeOrchWithHeadOutput(head_output="") + job = _job("1024", "1024", 1, serve_args={"max-model-len": "16384"}) + job.orch = orch + with self.assertRaises(RuntimeError): + job.probe_openai_endpoints() + + def test_unparseable_output_raises(self): + orch = FakeOrchWithHeadOutput(head_output="not json {{{") + job = _job("1024", "1024", 1, serve_args={"max-model-len": "16384"}) + job.orch = orch + with self.assertRaises(RuntimeError): + job.probe_openai_endpoints() + + def test_bad_shape_raises(self): + orch = FakeOrchWithHeadOutput(head_output=json.dumps({"model_endpoint": "not-a-pair"})) + job = _job("1024", "1024", 1, serve_args={"max-model-len": "16384"}) + job.orch = orch + with self.assertRaises(RuntimeError): + job.probe_openai_endpoints() + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_vllm_parsing.py b/cvs/lib/inference/unittests/test_vllm_parsing.py new file mode 100644 index 000000000..967369035 --- /dev/null +++ b/cvs/lib/inference/unittests/test_vllm_parsing.py @@ -0,0 +1,414 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs/lib/inference/utils/vllm_parsing.py. + +Impl-blind, spec-derived (Spec A1: per_gpu_throughput accounts for pipeline- +parallel size). Every case drives `to_client_metrics` / `_gpu_count` / +`_safe_div` directly with plain dict fixtures -- no orchestrator, no VllmJob, +no hardware. Written greenfield (RED) before the implementation adds the +required `pp` kwarg and the `_gpu_count` helper; the implementer makes them +green and cannot edit this file. +''' + +import unittest + +from cvs.lib.inference.utils import vllm_parsing + + +# --------------------------------------------------------------------------- +# Fixtures: a fresh, comprehensive raw benchmark artifact per call so no test +# mutates state another test reads (avoids the shared-mutable-fixture pitfall). +# Values mirror the shape of a real `vllm bench serve` `results` artifact. +# total_token_throughput defaults to a round 4800.0 so /8 and /16 are exact. +# --------------------------------------------------------------------------- +def _raw(**overrides): + base = { + "num_prompts": 3200, + "completed": 1791, + "failed": 1409, + "duration": 564.1147056743503, + "request_throughput": 3.174886210170704, + "request_goodput": 3.174886210170704, + "output_throughput": 4099.180497760143, + "total_token_throughput": 4800.0, + "max_output_tokens_per_s": 4590.0, + "max_concurrency": 64, + "max_concurrent_requests": 76, + "total_input_tokens": 229974, + "total_output_tokens": 2312408, + "rtfx": 0.0, + "mean_ttft_ms": 291.88843878398956, + "median_ttft_ms": 73.2587007805705, + "p90_ttft_ms": 85.64653992652893, + "p95_ttft_ms": 91.81479038670659, + "p99_ttft_ms": 6259.152442589402, + "mean_tpot_ms": 15.032167084785439, + "median_tpot_ms": 15.03020008988302, + "p90_tpot_ms": 15.209271169796184, + "p95_tpot_ms": 15.24944565870449, + "p99_tpot_ms": 15.943741001209947, + "mean_itl_ms": 15.019441688915942, + "median_itl_ms": 14.628257602453232, + "p50_itl_ms": 14.628257602453232, + "p95_itl_ms": 17.392832040786708, + "p99_itl_ms": 27.511265948414778, + "mean_e2el_ms": 19668.871285726116, + "median_e2el_ms": 19835.17629932612, + "p90_e2el_ms": 30145.559770055115, + "p95_e2el_ms": 31765.095902141184, + "p99_e2el_ms": 34034.53387795014, + } + base.update(overrides) + return base + + +_ISL = "128" # str, matching production (VllmJob stores self.isl = str(isl)) + + +def _metrics(raw=None, tp="8", isl=_ISL, pp="1"): + if raw is None: + raw = _raw() + return vllm_parsing.to_client_metrics(raw, tp=tp, isl=isl, pp=pp) + + +# =========================================================================== +# _gpu_count -- pure helper (Spec AC2). int(tp)*int(pp) for valid numeric +# input (str or int), None for missing/None/non-numeric, never raises. +# Range/equivalence table + zero boundary + no-raise + commutativity invariant. +# =========================================================================== +class TestGpuCount(unittest.TestCase): + def test_gpu_count_grid(self): + cases = [ + # (tp, pp, expected) + ("8", "2", 16), # both numeric strings -> product + (8, 2, 16), # both ints + (8, "2", 16), # mixed int/str (no str-repetition trap) + ("8", 2, 16), # mixed str/int + ("1", "8", 8), # single-node style + ("16", "1", 16), + ("0", "8", 0), # zero is a real product, not None + ("8", "0", 0), + (None, "8", None), # missing/None -> None + ("8", None, None), + (None, None, None), + ("auto", "8", None), # non-numeric -> None (int('auto') raises) + ("8", "auto", None), + ("", "8", None), # empty string -> None + ("2.5", "8", None), # int('2.5') raises ValueError -> None + ] + for tp, pp, expected in cases: + with self.subTest(tp=tp, pp=pp): + self.assertEqual(vllm_parsing._gpu_count(tp, pp), expected) + + def test_gpu_count_zero_is_int_not_none(self): + # 0 (a degenerate but real count) must be distinct from None so that + # _safe_div can then apply its zero-divisor guard downstream. + result = vllm_parsing._gpu_count("0", "8") + self.assertEqual(result, 0) + self.assertIsNotNone(result) + + def test_gpu_count_never_raises_on_bad_input(self): + # Degrade-to-None contract: must not raise out on any bad input. + for tp, pp in [(None, None), ("auto", "auto"), ("", ""), (object(), 8), (8, [1])]: + with self.subTest(tp=tp, pp=pp): + try: + self.assertIsNone(vllm_parsing._gpu_count(tp, pp)) + except Exception as exc: # noqa: BLE001 - the whole point is no raise + self.fail(f"_gpu_count({tp!r}, {pp!r}) raised {exc!r}") + + def test_gpu_count_commutative_invariant(self): + # int(tp)*int(pp) == int(pp)*int(tp): the helper must be symmetric. + for a, b in [("8", "2"), (4, 3), ("1", "16"), ("0", "8"), ("auto", "2")]: + with self.subTest(a=a, b=b): + self.assertEqual( + vllm_parsing._gpu_count(a, b), + vllm_parsing._gpu_count(b, a), + ) + + +# =========================================================================== +# _safe_div -- pure helper underpinning every derived metric's None-degrade. +# Spec: None/0 divisors -> None; None numerator -> None; zero numerator with +# a real divisor is a real 0.0 result (not None). +# =========================================================================== +class TestSafeDiv(unittest.TestCase): + def test_safe_div_grid(self): + cases = [ + (10, 2, 5.0), + (9, 4, 2.25), + (0, 5, 0.0), # zero numerator -> real 0.0, NOT None + (10, 0, None), # zero divisor -> None + (10, None, None), # None divisor -> None + (None, 5, None), # None numerator -> None + (None, None, None), + ] + for num, den, expected in cases: + with self.subTest(num=num, den=den): + result = vllm_parsing._safe_div(num, den) + if expected is None: + self.assertIsNone(result) + else: + self.assertIsNotNone(result) + self.assertAlmostEqual(result, expected) + + def test_safe_div_zero_numerator_is_real_zero(self): + result = vllm_parsing._safe_div(0, 5) + self.assertIsNotNone(result) + self.assertAlmostEqual(result, 0.0) + + +# =========================================================================== +# per_gpu_throughput -- the spec's focus (AC3/AC4/AC5). Pure; value grid over +# the (tp, pp) space + None-degradation + numeric invariants. +# =========================================================================== +class TestPerGpuThroughput(unittest.TestCase): + KEY = "client.per_gpu_throughput" + + def test_per_gpu_throughput_over_tp_pp_grid(self): + T = 4800.0 + cases = [ + # (tp, pp, expected) -- expected None means degrade-to-None + ("8", "1", T / 8), # AC4: single-node, == pre-fix ttot/tp + ("8", "2", T / 16), # AC3/AC5: pp accounted -> ttot/(tp*pp) + (8, "2", T / 16), # mixed int tp + ("8", 2, T / 16), # mixed int pp + (8, 2, T / 16), # both int + ("16", "1", T / 16), + ("4", "4", T / 16), + ("8", None, None), # pp None -> None + ("8", "auto", None), # pp non-numeric -> None + ("auto", "1", None), # tp non-numeric -> None + ("0", "8", None), # zero gpu count -> _safe_div guards -> None + ("8", "0", None), # zero gpu count -> None + ] + for tp, pp, expected in cases: + with self.subTest(tp=tp, pp=pp): + m = vllm_parsing.to_client_metrics(_raw(total_token_throughput=T), tp=tp, isl=_ISL, pp=pp) + if expected is None: + self.assertIsNone(m[self.KEY]) + else: + self.assertIsNotNone(m[self.KEY]) + self.assertAlmostEqual(m[self.KEY], expected) + + def test_pp1_equals_ttot_over_tp(self): + # AC4: single-node (pp="1") is byte-identical to the pre-fix formula. + raw = _raw() + m = _metrics(raw, tp="8", pp="1") + self.assertAlmostEqual(m[self.KEY], raw["total_token_throughput"] / 8) + + def test_pp2_is_exactly_half_of_pp1(self): + # AC5: a pp="2" cell yields exactly half of the pre-fix (ttot/tp) value. + v1 = _metrics(_raw(), tp="8", pp="1")[self.KEY] + v2 = _metrics(_raw(), tp="8", pp="2")[self.KEY] + self.assertAlmostEqual(v2, v1 / 2) + + def test_per_gpu_throughput_monotonic_decreasing_in_pp(self): + # Invariant: with tp and ttot fixed, more pipeline stages -> strictly + # lower per-GPU throughput. + vals = [_metrics(_raw(), tp="8", pp=str(pp))[self.KEY] for pp in (1, 2, 4, 8)] + for higher, lower in zip(vals, vals[1:]): + self.assertGreater(higher, lower) + + def test_none_when_total_token_throughput_missing_or_none(self): + # AC3: unavailable ttot -> None regardless of tp/pp (unchanged _safe_div). + raw_missing = _raw() + del raw_missing["total_token_throughput"] + self.assertIsNone(_metrics(raw_missing, tp="8", pp="2")[self.KEY]) + raw_none = _raw(total_token_throughput=None) + self.assertIsNone(_metrics(raw_none, tp="8", pp="2")[self.KEY]) + + def test_pp_defaults_to_one(self): + # Callers with no pipeline-parallel concept (e.g. InferenceX ATOM) omit + # `pp` entirely; it must silently behave as pp="1", not raise. + omitted = vllm_parsing.to_client_metrics(_raw(), tp="8", isl=_ISL) + explicit = _metrics(_raw(), tp="8", pp="1") + self.assertEqual(omitted[self.KEY], explicit[self.KEY]) + + +# =========================================================================== +# The other four _safe_div-guarded derived metrics + goodput alias. +# Restores TestToClientMetricsPure coverage: value + None-degradation per +# metric, table-driven. +# =========================================================================== +class TestDerivedMetrics(unittest.TestCase): + def test_derived_metric_values(self): + raw = _raw() + m = _metrics(raw, tp="8", isl=_ISL, pp="2") + cases = [ + ("client.normalized_ttft_ms_per_tok", raw["mean_ttft_ms"] / 128), + ("client.decode_latency_ratio", raw["p99_itl_ms"] / raw["p50_itl_ms"]), + ("client.decode_throughput_p50", 1000.0 / raw["median_tpot_ms"]), + ("client.success_rate", raw["completed"] / (raw["completed"] + raw["failed"])), + ] + for key, expected in cases: + with self.subTest(metric=key): + self.assertIsNotNone(m[key]) + self.assertAlmostEqual(m[key], expected) + + def test_derived_metric_none_degradation(self): + # Drop the raw scalar each derived metric depends on -> it degrades to + # None (does not raise, does not compute a wrong number). + cases = [ + ("client.normalized_ttft_ms_per_tok", "mean_ttft_ms"), + ("client.decode_latency_ratio", "p50_itl_ms"), + ("client.decode_throughput_p50", "median_tpot_ms"), + ] + for key, drop in cases: + with self.subTest(metric=key, dropped=drop): + raw = _raw() + del raw[drop] + m = _metrics(raw, tp="8", isl=_ISL, pp="2") + self.assertIsNone(m[key]) + + def test_success_rate_none_when_denominator_zero(self): + # completed=0, failed=0 -> _safe_div(0, 0) -> None (not a crash, not 0). + m = _metrics(_raw(completed=0, failed=0), tp="8", pp="1") + self.assertIsNone(m["client.success_rate"]) + + def test_goodput_alias_value_passthrough(self): + m = _metrics(_raw(request_goodput=42.5), tp="8", pp="1") + self.assertEqual(m["client.goodput"], 42.5) + + def test_goodput_alias_none_passthrough(self): + # Ran without --goodput -> request_goodput is null -> client.goodput None. + m = _metrics(_raw(request_goodput=None), tp="8", pp="1") + self.assertIsNone(m["client.goodput"]) + + +# =========================================================================== +# 1:1 stock-scalar namespacing (client. == raw[key]) and AC7 isolation. +# =========================================================================== +class TestStockScalarNamespacing(unittest.TestCase): + def test_returns_a_dict(self): + # Contract: to_client_metrics always returns a dict, never None/other. + # (A type-level assertion so a no-op stub is caught as a genuine + # assertion FAILURE rather than a downstream TypeError/ERROR.) + m = _metrics(_raw(), tp="8", pp="1") + self.assertIsInstance(m, dict) + + def test_every_stock_scalar_namespaced_one_to_one(self): + raw = _raw() + m = _metrics(raw, tp="8", pp="1") + for key, value in raw.items(): + with self.subTest(key=key): + nk = f"client.{key}" + self.assertIn(nk, m) + self.assertEqual(m[nk], value) + + def test_zero_valued_scalar_preserved_not_dropped(self): + # 0.0 is a real measurement; it must survive namespacing as 0.0, not be + # coerced to None or dropped. + m = _metrics(_raw(rtfx=0.0, request_throughput=0.0), tp="8", pp="1") + self.assertEqual(m["client.request_throughput"], 0.0) + self.assertIsNotNone(m["client.request_throughput"]) + self.assertEqual(m["client.rtfx"], 0.0) + + def test_numeric_scalars_stay_numeric(self): + m = _metrics(_raw(), tp="8", pp="1") + for nk in ("client.total_token_throughput", "client.mean_ttft_ms", "client.p99_itl_ms"): + with self.subTest(key=nk): + self.assertIsInstance(m[nk], (int, float)) + + def test_only_per_gpu_throughput_changes_with_pp(self): + # AC7: varying pp changes per_gpu_throughput and NOTHING else. + m1 = _metrics(_raw(), tp="8", isl=_ISL, pp="1") + m2 = _metrics(_raw(), tp="8", isl=_ISL, pp="2") + self.assertEqual(set(m1), set(m2)) + for key in m1: + if key == "client.per_gpu_throughput": + continue + with self.subTest(key=key): + self.assertEqual(m1[key], m2[key]) + self.assertNotEqual(m1["client.per_gpu_throughput"], m2["client.per_gpu_throughput"]) + + +# =========================================================================== +# client.failed fallback derivation (vllm_parsing.py:70-78). +# When failed is missing/None but completed and num_prompts are present: +# failed = max(0, int(num_prompts) - int(completed)), guarded -> None on bad input. +# =========================================================================== +class TestFailedFallbackDerivation(unittest.TestCase): + def test_failed_absent_and_completed_le_num_prompts_derives(self): + raw = _raw() + del raw["failed"] + raw["num_prompts"] = 3200 + raw["completed"] = 1791 + m = _metrics(raw, tp="8", pp="1") + self.assertEqual(m["client.failed"], 3200 - 1791) + + def test_failed_absent_and_completed_gt_num_prompts_clamped_to_zero(self): + # max(0, ...) must clamp -- never emit a negative failed count. + raw = _raw() + del raw["failed"] + raw["num_prompts"] = 100 + raw["completed"] = 150 + m = _metrics(raw, tp="8", pp="1") + self.assertEqual(m["client.failed"], 0) + + def test_failed_absent_and_none_valued_still_derives(self): + # failed present-but-None is treated as missing -> fallback fires. + raw = _raw(failed=None) + raw["num_prompts"] = 3200 + raw["completed"] = 1791 + m = _metrics(raw, tp="8", pp="1") + self.assertEqual(m["client.failed"], 3200 - 1791) + + def test_failed_absent_nonnumeric_inputs_not_injected(self): + # Guarded try/except -> failed stays None and the key is NOT injected. + cases = [ + {"num_prompts": "auto", "completed": 1791}, + {"num_prompts": 3200, "completed": "auto"}, + {"num_prompts": None, "completed": 1791}, + ] + for overrides in cases: + with self.subTest(**overrides): + raw = _raw() + del raw["failed"] + raw.update(overrides) + m = _metrics(raw, tp="8", pp="1") + self.assertNotIn("client.failed", m) + + def test_failed_present_fallback_not_invoked(self): + # An explicit failed value wins; the fallback must not overwrite it even + # when num_prompts-completed would compute a different number. + raw = _raw(failed=5) + raw["num_prompts"] = 3200 + raw["completed"] = 1791 # would derive 1409, must be ignored + m = _metrics(raw, tp="8", pp="1") + self.assertEqual(m["client.failed"], 5) + + +# =========================================================================== +# Module constants -- pin the spec's non-functional "no change" requirements +# and the record-only (ungated) status of per_gpu_throughput. +# =========================================================================== +class TestModuleConstants(unittest.TestCase): + def test_per_gpu_throughput_is_record_only_not_gated(self): + # Spec: per_gpu_throughput is NOT in GATED_METRICS -> the fix cannot + # flip any pass/fail gate. + self.assertNotIn("per_gpu_throughput", vllm_parsing.GATED_METRICS) + + def test_per_gpu_throughput_registered_in_client_metrics(self): + units = dict(vllm_parsing.CLIENT_METRICS) + self.assertIn("per_gpu_throughput", units) + self.assertEqual(units["per_gpu_throughput"], "tok/s") + + def test_client_metrics_short_names_are_unique(self): + # CLIENT_METRIC_UNITS is `dict(CLIENT_METRICS)`, which silently collapses + # a duplicate short name to its last entry -- assert there are none. + short_names = [short for short, _unit in vllm_parsing.CLIENT_METRICS] + self.assertEqual(len(short_names), len(set(short_names))) + + def test_client_metric_units_matches_client_metrics(self): + self.assertEqual(vllm_parsing.CLIENT_METRIC_UNITS["total_token_throughput"], "tok/s") + self.assertEqual(vllm_parsing.CLIENT_METRIC_UNITS["mean_ttft_ms"], "ms") + + def test_gated_metrics_subset_of_client_metrics(self): + client_short = {short for short, _unit in vllm_parsing.CLIENT_METRICS} + self.assertEqual(vllm_parsing.GATED_METRICS - client_short, set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_vllm_report_preset.py b/cvs/lib/inference/unittests/test_vllm_report_preset.py new file mode 100644 index 000000000..4326e8517 --- /dev/null +++ b/cvs/lib/inference/unittests/test_vllm_report_preset.py @@ -0,0 +1,102 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +import unittest + +from cvs.lib.inference.utils.vllm_parsing import ( + CLIENT_METRICS, + GATED_METRICS, + METRIC_TIER_ORDER, + METRIC_TIERS, + VLLM_RESULTS_COLUMNS, + tier_metric_specs, +) +from cvs.lib.report.presets.vllm import VLLM_REPORT_CONFIG + + +class TestVllmReportPreset(unittest.TestCase): + def test_results_columns_fixed_positional_prefix(self): + fixed = VLLM_RESULTS_COLUMNS[:7] + self.assertEqual( + fixed, + ( + ("Model", None), + ("GPU", None), + ("ISL", None), + ("OSL", None), + ("Policy", None), + ("Conc", None), + ("Host", None), + ), + ) + + def test_metric_tiers_subset_of_tier_order(self): + self.assertTrue(set(METRIC_TIERS) <= set(METRIC_TIER_ORDER)) + + def test_gated_metrics_partitioned_exactly_once(self): + tiered = [m for names in METRIC_TIERS.values() for m in names] + # No duplicates across tiers. + self.assertEqual(len(tiered), len(set(tiered))) + # Every gated metric lands in exactly one non-record tier. + self.assertEqual(set(tiered), set(GATED_METRICS)) + + def test_gated_metrics_subset_of_client_metrics(self): + client_short = {short for short, _unit in CLIENT_METRICS} + missing = GATED_METRICS - client_short + self.assertEqual(missing, set(), f"GATED_METRICS not in CLIENT_METRICS: {missing}") + + def test_tier_metric_specs_throughput(self): + cell = { + "client.output_throughput": {"kind": "min_tok_s", "value": 1}, + "client.mean_ttft_ms": {"kind": "max_ms", "value": 2}, + } + specs = tier_metric_specs(cell, "throughput") + self.assertIn("client.output_throughput", specs) + self.assertNotIn("client.mean_ttft_ms", specs) + + def test_tier_metric_specs_record_includes_non_tiered(self): + cell = { + "client.num_prompts": {"kind": "within", "value": 100}, + "client.output_throughput": {"kind": "min_tok_s", "value": 1}, + } + specs = tier_metric_specs(cell, "record") + self.assertIn("client.num_prompts", specs) + self.assertNotIn("client.output_throughput", specs) + + def test_preset_config_identity(self): + self.assertEqual(VLLM_REPORT_CONFIG.suite_id, "vllm") + self.assertEqual(VLLM_REPORT_CONFIG.inference_test_substring, "test_vllm_inference") + self.assertEqual(VLLM_REPORT_CONFIG.row_card_test_names, ("test_metric", "test_gpu_metric", "test_prom_metric")) + + def test_preset_lifecycle_labels_match_what_suite_records(self): + # Guard against drift: the vLLM suite (cvs/tests/inference/vllm/vllm.py) + # records exactly these session-level stages via lifecycle.record(...). + suite_recorded = { + "container_launch", + "topology_discovery", + "model_fetch", + "server_ready", + "teardown", + } + self.assertTrue(set(VLLM_REPORT_CONFIG.session_lifecycle_labels) <= suite_recorded) + self.assertTrue(set(VLLM_REPORT_CONFIG.cell_lifecycle_labels) <= suite_recorded) + + def test_auto_register_resolves_vllm_stem(self): + from cvs.lib.report.auto_register import try_auto_register_inference_suite_report + from cvs.lib.report.registry import get_suite_report_config + + class _FakeConfig: + pass + + cfg = _FakeConfig() + cfg._suite_name = "vllm" + cfg._suite_report_config = None + registered = try_auto_register_inference_suite_report(cfg) + self.assertTrue(registered) + self.assertIs(get_suite_report_config(cfg), VLLM_REPORT_CONFIG) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/unittests/test_vllm_server_metrics.py b/cvs/lib/inference/unittests/test_vllm_server_metrics.py new file mode 100644 index 000000000..18dd29d31 --- /dev/null +++ b/cvs/lib/inference/unittests/test_vllm_server_metrics.py @@ -0,0 +1,356 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.inference.utils.vllm_server_metrics. + +Black-box tests authored from the behavioral spec only (impl-blind). The +module contains pure parsers for vLLM's engine-side Prometheus `/metrics` +exposition-format text: no I/O, no hardware, pure text/dict transformations. + +Contract under test: + parse_prometheus_text(raw) -> {metric_name: {"buckets": {le: count}, + "sum": float, "count": float}} for histograms, {metric_name: float} + for bare gauges. Degrades to {} on empty/unparseable input; never raises. + diff_histogram(before, after) -> {le: after_count - before_count}, clamped + to >= 0. None if `after` has no buckets. + histogram_quantile(buckets, q) -> linear interpolation between bucket + boundaries; None on empty/zero-count buckets. + to_prom_metrics(before_text, after_text) -> the composed prom.* dict; + all-None (never partial, never a raise) if either scrape is + missing/unparseable. + +Framework: unittest.TestCase + self.subTest + unittest.mock (no pytest), +matching test_gpu.py's conventions. +''' + +import unittest +from unittest.mock import MagicMock + +from cvs.lib.inference.utils.vllm_server_metrics import ( + PROM_METRICS, + PROM_METRIC_UNITS, + diff_histogram, + histogram_quantile, + parse_prometheus_text, + to_prom_metrics, +) +from cvs.lib.inference.vllm_job import scrape_vllm_metrics + +# Shared bucket list (queue/prefill/decode/inference/e2e), seconds. +_BUCKETS = [ + 0.3, + 0.5, + 0.8, + 1.0, + 1.5, + 2.0, + 2.5, + 5.0, + 10.0, + 15.0, + 20.0, + 30.0, + 40.0, + 50.0, + 60.0, + 120.0, + 240.0, + 480.0, + 960.0, + 1920.0, + 7680.0, +] + + +def _histogram_text(name: str, cumulative_counts: dict, total_sum: float) -> str: + """Build real Prometheus exposition-format text for one histogram metric. + + cumulative_counts: {le_str: cumulative_count}, must include "+Inf". + """ + lines = [f"# HELP {name} test histogram", f"# TYPE {name} histogram"] + for le, count in cumulative_counts.items(): + lines.append(f'{name}_bucket{{le="{le}"}} {count}') + total = cumulative_counts["+Inf"] + lines.append(f"{name}_sum {total_sum}") + lines.append(f"{name}_count {total}") + return "\n".join(lines) + + +def _full_scrape_text(queue_counts, queue_sum, prefill_counts, prefill_sum) -> str: + parts = [ + _histogram_text("vllm:request_queue_time_seconds", queue_counts, queue_sum), + _histogram_text("vllm:request_prefill_time_seconds", prefill_counts, prefill_sum), + "# HELP vllm:num_requests_waiting test gauge", + "# TYPE vllm:num_requests_waiting gauge", + "vllm:num_requests_waiting 0", + ] + return "\n".join(parts) + + +def _cumulative(observations: list) -> dict: + """Bucket a list of raw second-values into cumulative le-counts using + the real shared bucket list, plus '+Inf'.""" + counts = {} + running = 0 + for b in _BUCKETS: + running += sum(1 for o in observations if o <= b) + counts[str(b)] = float(running) + counts["+Inf"] = float(len(observations)) + return counts + + +class TestParsePrometheusText(unittest.TestCase): + def test_empty_and_none_degrade_to_empty_dict(self): + for raw in (None, "", " ", "\n\n"): + with self.subTest(raw=repr(raw)): + self.assertEqual(parse_prometheus_text(raw), {}) + + def test_parses_histogram_buckets_sum_count(self): + text = _histogram_text( + "vllm:request_queue_time_seconds", + {"0.3": 2.0, "0.5": 3.0, "+Inf": 5.0}, + total_sum=1.75, + ) + out = parse_prometheus_text(text) + self.assertIn("vllm:request_queue_time_seconds", out) + hist = out["vllm:request_queue_time_seconds"] + self.assertEqual(hist["buckets"], {"0.3": 2.0, "0.5": 3.0, "+Inf": 5.0}) + self.assertEqual(hist["sum"], 1.75) + self.assertEqual(hist["count"], 5.0) + + def test_ignores_help_and_type_comment_lines(self): + text = "\n".join( + [ + "# HELP vllm:request_queue_time_seconds queue wait time", + "# TYPE vllm:request_queue_time_seconds histogram", + 'vllm:request_queue_time_seconds_bucket{le="0.3"} 1', + "vllm:request_queue_time_seconds_sum 0.2", + "vllm:request_queue_time_seconds_count 1", + ] + ) + out = parse_prometheus_text(text) + self.assertEqual(set(out.keys()), {"vllm:request_queue_time_seconds"}) + + def test_parses_bare_gauge_line(self): + text = "\n".join( + [ + "# TYPE vllm:num_requests_waiting gauge", + "vllm:num_requests_waiting 3", + ] + ) + out = parse_prometheus_text(text) + self.assertEqual(out["vllm:num_requests_waiting"], 3.0) + + def test_multiple_metrics_coexist(self): + text = _full_scrape_text(_cumulative([0.1, 0.2]), 0.3, _cumulative([0.4]), 0.4) + out = parse_prometheus_text(text) + self.assertIn("vllm:request_queue_time_seconds", out) + self.assertIn("vllm:request_prefill_time_seconds", out) + self.assertIn("vllm:num_requests_waiting", out) + + def test_never_raises_on_malformed_lines(self): + garbage_texts = [ + "not a valid prometheus line at all", + "vllm:request_queue_time_seconds_bucket{le=\"not_a_number_or_inf\"} abc", + "\x00\x01\x02 binary garbage", + "vllm:foo_sum not_a_float", + "vllm:foo_count", + ] + for raw in garbage_texts: + with self.subTest(raw=repr(raw)): + try: + parse_prometheus_text(raw) + except Exception as exc: # noqa: BLE001 + self.fail(f"parse_prometheus_text raised unexpectedly on {raw!r}: {exc!r}") + + def test_truncated_text_degrades_gracefully(self): + # A bucket line cut off mid-value; count line normal. + text = "vllm:request_queue_time_seconds_bucket{le=\"0.3\"} 1.\nvllm:request_queue_time_seconds_count 5" + try: + out = parse_prometheus_text(text) + except Exception as exc: # noqa: BLE001 + self.fail(f"parse_prometheus_text raised unexpectedly: {exc!r}") + # The malformed bucket line is simply skipped; count line still parses. + self.assertEqual(out.get("vllm:request_queue_time_seconds", {}).get("count"), 5.0) + + +class TestDiffHistogram(unittest.TestCase): + def test_simple_before_after_diff(self): + before = {"buckets": {"0.3": 2.0, "+Inf": 5.0}} + after = {"buckets": {"0.3": 4.0, "+Inf": 9.0}} + self.assertEqual(diff_histogram(before, after), {"0.3": 2.0, "+Inf": 4.0}) + + def test_missing_before_bucket_treated_as_zero(self): + before = {"buckets": {"+Inf": 5.0}} + after = {"buckets": {"0.3": 1.0, "+Inf": 6.0}} + self.assertEqual(diff_histogram(before, after), {"0.3": 1.0, "+Inf": 1.0}) + + def test_none_before_treated_as_all_zero(self): + after = {"buckets": {"0.3": 1.0, "+Inf": 3.0}} + self.assertEqual(diff_histogram(None, after), {"0.3": 1.0, "+Inf": 3.0}) + + def test_negative_diff_clamped_to_zero(self): + # Simulates a scrape taken across a server restart: after < before. + before = {"buckets": {"0.3": 10.0, "+Inf": 20.0}} + after = {"buckets": {"0.3": 1.0, "+Inf": 2.0}} + self.assertEqual(diff_histogram(before, after), {"0.3": 0.0, "+Inf": 0.0}) + + def test_none_after_returns_none(self): + before = {"buckets": {"0.3": 1.0, "+Inf": 1.0}} + self.assertIsNone(diff_histogram(before, None)) + + def test_empty_after_buckets_returns_none(self): + self.assertIsNone(diff_histogram({"buckets": {}}, {"buckets": {}})) + + +class TestHistogramQuantile(unittest.TestCase): + def test_zero_count_returns_none(self): + self.assertIsNone(histogram_quantile({"0.3": 0.0, "+Inf": 0.0}, 0.5)) + + def test_empty_or_none_returns_none(self): + self.assertIsNone(histogram_quantile({}, 0.5)) + self.assertIsNone(histogram_quantile(None, 0.5)) + + def test_all_mass_in_one_bucket(self): + # Every observation lands at or below 0.3s (the first bucket). + # Linear interpolation assumes uniform distribution between the + # implicit lower bound (0) and this bucket's boundary (0.3): + # target rank = 0.5*10 = 5; frac = (5-0)/(10-0) = 0.5; + # interpolated = 0 + 0.5*(0.3-0) = 0.15. + buckets = {"0.3": 10.0, "0.5": 10.0, "+Inf": 10.0} + self.assertAlmostEqual(histogram_quantile(buckets, 0.5), 0.15) + + def test_hand_computed_interpolation_p50(self): + # 0 <= x <= 0.3: 2 obs (cumulative 2); 0.3 < x <= 0.5: 8 obs (cumulative + # 10); target rank for p50 of 10 total = 5. Falls in the (0.3, 0.5] + # bucket: prev_bound=0.3 prev_count=2, bound=0.5 count=10. + # frac = (5-2)/(10-2) = 0.375; interpolated = 0.3 + 0.375*(0.5-0.3) = 0.375 + buckets = {"0.3": 2.0, "0.5": 10.0, "+Inf": 10.0} + self.assertAlmostEqual(histogram_quantile(buckets, 0.5), 0.375) + + def test_hand_computed_interpolation_p95(self): + # 20 total obs: cumulative 0.3->5, 0.5->18, 1.0->20. p95 target rank=19. + # Falls in (0.5, 1.0]: prev_bound=0.5 prev_count=18, bound=1.0 count=20. + # frac = (19-18)/(20-18) = 0.5; interpolated = 0.5 + 0.5*(1.0-0.5) = 0.75 + buckets = {"0.3": 5.0, "0.5": 18.0, "1.0": 20.0, "+Inf": 20.0} + self.assertAlmostEqual(histogram_quantile(buckets, 0.95), 0.75) + + def test_le_inf_only_bucket(self): + # Degenerate case: only the +Inf bucket present. + buckets = {"+Inf": 4.0} + self.assertEqual(histogram_quantile(buckets, 0.5), float("inf")) + + def test_target_in_inf_bucket_clamps_to_highest_finite_bound(self): + # 10 total obs, all but 1 land at or below 1.0s; the last only shows + # up in "+Inf" (an overloaded request exceeding every finite bucket). + # p95 target rank = 9.5, which only the "+Inf" bucket satisfies. + # PromQL cannot interpolate past the last finite boundary, so it + # clamps to it (1.0) instead of returning +Inf. + buckets = {"0.3": 5.0, "0.5": 8.0, "1.0": 9.0, "+Inf": 10.0} + self.assertEqual(histogram_quantile(buckets, 0.95), 1.0) + + def test_never_raises_on_malformed_le_values(self): + try: + out = histogram_quantile({"not_a_number": 1.0, "+Inf": 2.0}, 0.5) + except Exception as exc: # noqa: BLE001 + self.fail(f"histogram_quantile raised unexpectedly: {exc!r}") + else: + self.assertIsNone(out) + + +class TestToPromMetrics(unittest.TestCase): + def test_all_prom_metrics_keys_present_shape(self): + expected_keys = {f"prom.{short}" for short, _unit in PROM_METRICS} + out = to_prom_metrics(None, None) + self.assertEqual(set(out.keys()), expected_keys) + + def test_none_before_or_after_yields_all_none(self): + after_text = _full_scrape_text(_cumulative([0.1]), 0.1, _cumulative([0.2]), 0.2) + for before, after in ((None, after_text), (after_text, None), (None, None)): + with self.subTest(before=before, after=after): + out = to_prom_metrics(before, after) + for k in out: + self.assertIsNone(out[k]) + + def test_unparseable_text_yields_all_none_not_raise(self): + try: + out = to_prom_metrics("garbage before", "garbage after") + except Exception as exc: # noqa: BLE001 + self.fail(f"to_prom_metrics raised unexpectedly: {exc!r}") + for k in out: + self.assertIsNone(out[k]) + + def test_end_to_end_realistic_before_after_pair(self): + # "before" scrape: server already served 3 queue-wait observations + # from a prior cell (0.1, 0.2, 0.4s) -- the reused-server baseline. + before_text = _full_scrape_text(_cumulative([0.1, 0.2, 0.4]), 0.7, _cumulative([0.2, 0.3]), 0.5) + # "after" scrape: this cell added 2 more queue-wait obs (0.6, 0.9s) + # and 1 more prefill obs (1.2s) on top of the same server's counters. + after_text = _full_scrape_text( + _cumulative([0.1, 0.2, 0.4, 0.6, 0.9]), + 2.2, + _cumulative([0.2, 0.3, 1.2]), + 1.7, + ) + out = to_prom_metrics(before_text, after_text) + # This cell's isolated queue-wait observations are exactly [0.6, 0.9] + # (0.6 falls in bucket 0.8, 0.9 falls in bucket 1.0); p50 of 2 obs + # falls in/around the first of the two remaining buckets. + self.assertIsNotNone(out["prom.queue_time_p50_ms"]) + self.assertIsNotNone(out["prom.queue_time_p95_ms"]) + self.assertIsNotNone(out["prom.prefill_time_p50_ms"]) + self.assertIsNotNone(out["prom.prefill_time_p95_ms"]) + # Values are in ms (seconds * 1000), and in the right ballpark given + # only [0.6, 0.9] contributed post-diff (600-1000ms range). + self.assertGreater(out["prom.queue_time_p50_ms"], 500) + self.assertLess(out["prom.queue_time_p50_ms"], 1100) + + def test_prom_metric_units_cover_every_metric(self): + for short, unit in PROM_METRICS: + with self.subTest(short=short): + self.assertEqual(PROM_METRIC_UNITS[short], unit) + + +class TestScrapeVllmMetrics(unittest.TestCase): + """I/O-boundary test for scrape_vllm_metrics (lives in vllm_job.py). + + Mirrors TestCaptureGpuMetrics's assert_called_once_with style: mock orch, + pin the exact command string, verify degrade-on-failure never raises. + """ + + def test_happy_path_returns_raw_text(self): + orch = MagicMock() + orch.exec_on_head.return_value = {"node0": "vllm:num_requests_waiting 0\n"} + out = scrape_vllm_metrics(orch, "http://0.0.0.0", "8888") + self.assertEqual(out, "vllm:num_requests_waiting 0\n") + orch.exec_on_head.assert_called_once_with("curl -sf http://0.0.0.0:8888/metrics") + + def test_timeout_kwarg_passed_through_when_given(self): + orch = MagicMock() + orch.exec_on_head.return_value = {"node0": "vllm:num_requests_waiting 0\n"} + scrape_vllm_metrics(orch, "http://0.0.0.0", "8888", timeout_s=30) + orch.exec_on_head.assert_called_once_with("curl -sf http://0.0.0.0:8888/metrics", timeout=30) + + def test_curl_failure_exception_degrades_to_none(self): + orch = MagicMock() + orch.exec_on_head.side_effect = RuntimeError("connection refused") + try: + out = scrape_vllm_metrics(orch, "http://0.0.0.0", "8888") + except Exception as exc: # noqa: BLE001 + self.fail(f"scrape_vllm_metrics raised unexpectedly: {exc!r}") + self.assertIsNone(out) + + def test_empty_output_degrades_to_none(self): + orch = MagicMock() + orch.exec_on_head.return_value = {"node0": ""} + self.assertIsNone(scrape_vllm_metrics(orch, "http://0.0.0.0", "8888")) + + def test_no_hosts_in_output_degrades_to_none(self): + orch = MagicMock() + orch.exec_on_head.return_value = {} + self.assertIsNone(scrape_vllm_metrics(orch, "http://0.0.0.0", "8888")) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/inference/utils/AGENTS.md b/cvs/lib/inference/utils/AGENTS.md new file mode 100644 index 000000000..28aadada7 --- /dev/null +++ b/cvs/lib/inference/utils/AGENTS.md @@ -0,0 +1,330 @@ +# cvs/lib/inference/utils — inference-specific config and parsing + +**Boundary**: this is the serving/inference half of the config machinery. +The generic half (`BaseVariantConfig`, `substitute_config`, `evaluate_all`, `Paths`, `ContainerSpec`) +lives in `cvs/lib/utils/` — import from there, never duplicate it here. + +--- + +## Files + +### `inferencing_config_loader.py` + +#### Schema classes + +**`RoleServer`** (`_Forbid`): per-model server overrides. +- `serve_args: Dict[str, Any]` — extra `vllm serve` flags; scalar → `--flag value`, + `True` → bare `--flag`, list → flag repeated per element +- `env: Dict[str, str]` — env vars merged over orchestrator defaults +- Both default empty; fp8-kv cells set `--kv-cache-dtype` here to keep the generic driver model-agnostic + +**`Roles`** (`_Forbid`): wraps `RoleServer`. +- `server: RoleServer` — defaults to empty `RoleServer()` + +**`GoodputSlo`** (`_Forbid`): per-combo goodput gate, in milliseconds. +- `ttft_ms: float`, `tpot_ms: float`, `e2el_ms: float` +- **INPUT to the run** (passed to `vllm bench serve --goodput`), NOT a threshold to assert. + Lives in the sweep, not `threshold.json`. `_Forbid` ensures a typo'd SLO key fails load + rather than silently drop the SLO and run with the wrong gate on hardware. + +**`SeqCombo`** (`_Forbid`): one named sequence-length combination. +- `name: str` — the join key referenced by `Run.combo` +- `isl: str`, `osl: str` +- `goodput_slo: Optional[GoodputSlo]` — omit when no goodput gate is needed + +**`Run`** (`_Forbid`): one sweep cell — a named combo at a single concurrency. +- `combo: str` — references a `SeqCombo.name` +- `concurrency: int` +- Explicit `runs[]` replaces the old NxM cartesian (`sequence_combinations × concurrency_levels`); + you enumerate exactly the cells you want + +**`Sweep`** (`_Forbid`): the full sweep selector. +- `sequence_combinations: List[SeqCombo]` +- `runs: List[Run]` +- `@model_validator(mode="after")` delegates to `validate_sweep_selector` + +**`Params`** (`_Forbid`): `vllm bench serve` CLI flags; all fields are `str`. + +| Field | Default | Notes | +|---|---|---| +| `backend` | `"vllm"` | | +| `base_url` | `"http://0.0.0.0"` | | +| `port_no` | `"8888"` | | +| `dataset_name` | `"random"` | | +| `burstiness` | `"1.0"` | | +| `seed` | `"0"` | | +| `request_rate` | `"inf"` | | +| `random_range_ratio` | `"0.8"` | | +| `random_prefix_len` | `"0"` | | +| `tensor_parallelism` | `"1"` | used in `cell_key` and `per_gpu_throughput` | +| `tokenizer_mode` | `"auto"` | | +| `percentile_metrics` | `"ttft,tpot,itl,e2el"` | | +| `metric_percentiles` | `"50,90,95,99"` | | +| `num_prompts` | `"3200"` | overridden per-cell by `_num_prompts_for` | +| `client_poll_count` | `"20"` | see below | + +`client_poll_count` semantics: total client wait budget = +**(client_poll_count × 60 s) + 120 s initial wait**. +The poll loop exits as soon as the client finishes, so raising this never slows down fast cells. +Raise it for high-osl cells where large-output runs take longer to complete. (Regression: REG-20260609-001) + +**`VariantConfig(BaseVariantConfig)`**: the full typed config. +- Adds: `framework: Literal["vllm_single"]`, `gpu_arch: str`, `roles: Roles = Roles()`, + `params: Params`, `sweep: Sweep` +- Implements: `cell_key(isl, osl, concurrency)`, `expected_cells()` +- Has: `@model_validator(mode="after") _check_thresholds_cover_sweep` + +--- + +#### Public functions + +**`load_variant(config_path, cluster_dict) -> VariantConfig`** + +The function a suite's `variant_config` fixture calls. + +1. Delegates file read + 3-pass placeholder substitution to `substitute_config` +2. Attaches thresholds returned by `substitute_config` to the raw dict +3. Builds and returns a typed, validated `VariantConfig` + +Does not reimplement file reading or substitution — always calls `substitute_config`. +See `cvs/lib/utils/AGENTS.md` for the full `substitute_config` contract. + +**`validate_sweep_selector(combo_names, run_combo_refs)`** — PUBLIC ENTRY POINT + +Shared rule called by **both**: +- the typed `Sweep` validator at load time +- `pytest_generate_tests` at collection time (reads raw JSON before the loader runs) + +Checks: +- combo names are unique (duplicate → `ValueError`) +- every `run.combo` names a known `sequence_combination` (unknown → `ValueError`) + +Operates on plain `list[str]` so both call sites feed it without the full typed schema. +If you add a sweep check, add it here so both paths enforce it without drift. + +--- + +#### `_check_thresholds_cover_sweep` — two-axis coverage check + +`@model_validator(mode="after")` on `VariantConfig`. Fails at load time if the threshold file +does not match the sweep matrix. + +**Axis 1 — cell coverage** +- Every sweep cell produced by `expected_cells()` has an entry in `threshold.json` +- No threshold key names a non-existent cell (catches typos in threshold key names) + +**Axis 2 — gated-metric coverage** +- Every cell present in both sets has a spec for every `client.` key + (e.g. `client.total_token_throughput`, not `total_token_throughput`) +- Without this, a gated metric with no spec falls through `test_metric`'s `spec is None` + record-only branch and reports PASS with zero assertions even under `enforce_thresholds=true` +- Only checked for cells present in both expected and threshold sets (missing cells are already + reported by axis 1; no double-reporting) + +When `enforce_thresholds=false`: both failures become warnings, not errors. +The config loads as a record-only scaffold (metrics captured, nothing asserted). + +See `docs/cell-key-format.md` for the exact key format used in `threshold.json`. + +--- + +### `vllm_parsing.py` + +**`to_client_metrics(raw, *, tp, isl) -> dict`** + +Pure — no I/O, no orchestration. `raw` is the already-parsed JSON the load generator +writes to its `--result-dir` results artifact. Caller is responsible for fetching and +`json.load`-ing the artifact. + +- All stock scalars namespaced 1:1 as `client.` +- `client.goodput` = alias for stock's `request_goodput` (the name threshold files reference) +- Derived metrics (all guarded by `_safe_div`; degrade to `None` on missing/`None`/zero-divisor): + +| Metric | Formula | +|---|---| +| `client.per_gpu_throughput` | `total_token_throughput / tp` | +| `client.normalized_ttft_ms_per_tok` | `mean_ttft_ms / isl` | +| `client.decode_latency_ratio` | `p99_itl_ms / p50_itl_ms` | +| `client.decode_throughput_p50` | `1000.0 / median_tpot_ms` | +| `client.success_rate` | `completed / (completed + failed)` | + +See `docs/derived-metrics.md` for per-metric inputs, None conditions, and gated/record-only rationale. + +**`CLIENT_METRICS`** — ordered `list[(short_name, unit)]`. + +The display surface: one HTML row per metric per cell. Single definition shared by all vLLM +flavours — do not re-list per suite. + +**`CLIENT_METRIC_UNITS`** — `dict` form of `CLIENT_METRICS`. + +**`GATED_METRICS`** — the asserted subset of `CLIENT_METRICS`. + +Membership = "out of range means FAILURE". + +Closed-world default: a new metric added to `CLIENT_METRICS` is record-only until its name +is explicitly added to `GATED_METRICS`. The loader's coverage check then forces a spec for +that metric in every cell before the suite can run green. + +Currently gated: + +| Category | Members | +|---|---| +| Throughput | `total_token_throughput`, `output_throughput` | +| TTFT latency | `mean`, `median`, `p90`, `p95`, `p99` | +| TPOT latency | `mean`, `median`, `p90`, `p95`, `p99` | +| ITL latency | `mean`, `median`, `p95`, `p99` (no p90 producer) | +| E2EL latency | `mean`, `median`, `p90`, `p95`, `p99` | +| Run health | `success_rate` (floor), `failed` (ceiling) | + +Record-only by design: inputs (`num_prompts`), totals (`total_input_tokens`, +`total_output_tokens`), secondary throughputs (`per_gpu_throughput`, `request_throughput`, +`goodput`, `decode_throughput_p50`, `max_output_tokens_per_s`), diagnostic derivations +(`normalized_ttft_ms_per_tok`, `decode_latency_ratio`). + +--- + +## The sweep selector + +Named combos + explicit `runs[]` list replaces old NxM cartesian +(`sequence_combinations × concurrency_levels`). One `Run` = one `(combo, concurrency)` cell. +The sweep enumerates exactly the cells you want. + +`sequence_combinations` names each ISL/OSL shape once (with an optional goodput SLO); +`runs` references those names at specific concurrencies. This lets you include only the +cells that matter for a given model — no silent NxM explosion, no empty cells. + +--- + +## The serving-generic / vllm-specific seam + +`Params` is the only vllm-specific class. Everything else (`Sweep`, `SeqCombo`, `GoodputSlo`, +`Roles`, `cell_key`) is serving-generic and reusable when a second serving framework lands. + +**Second serving framework checklist:** +1. Subclass `Params` with your framework's CLI flags +2. Reuse `Sweep`/`SeqCombo`/`GoodputSlo`/`Roles` unchanged +3. Reuse `validate_sweep_selector` in your `pytest_generate_tests` +4. Write your own metric vocabulary (`_parsing.py`) +5. Define your own `GATED_METRICS` + +--- + +## Lifecycle-as-tests model + +Each stage of the test run is an **independent pytest test**, not fixture body code. +Each stage appears as a timed, independently pass/fail row in the HTML report. + +Standard lifecycle order (pinned in `pytest_collection_modifyitems`): + +| Rank | Test | Action | +|---|---|---| +| 0 | `test_launch_container` | `setup_containers()`; asserts container is running | +| 1 | `test_setup_sshd` | no-op skip for vllm; distributed runs use NCCL/gloo, not sshd | +| 2 | `test_discover_topology` | discovers IB HCA devices (distributed only; no-op on single-node) | +| 3 | `test_model_fetch` | ensures model bytes present; polls or downloads if remote | +| 4 | `test_openai_compatible_smoke` | brings up a short-lived server at a small fixed cell; probes GET/POST `/v1/*` via `VllmJob.probe_openai_endpoints()`; always stops its server | +| 5 | `test_vllm_inference` | benchmark loop per cell; stores results in `inf_res_dict` | +| 6 | `test_metric` | one test per metric per cell; reads `inf_res_dict`; asserts verdict | +| 7 | `test_print_results_table` | summary log; must run after all cells | +| 8 | `test_teardown` | `teardown_containers()`; sets `lifecycle.torn_down`; **never skips** | + +Rules: +- Every test except `test_launch_container`, `test_teardown`, and `test_print_results_table` + checks `lifecycle.failed` and skips if true. `test_launch_container` is the first stage + and is itself responsible for setting `lifecycle.failed`; it has no prior stage to guard + against. `test_teardown` must run even on failure. `test_print_results_table` guards only + on whether `inf_res_dict` is empty and logs whatever results were recorded. +- `test_openai_compatible_smoke` catches exceptions, sets `lifecycle.failed = True`, re-raises; + a `finally` always calls `job.stop_server()` so a smoke-server failure never leaves a stray + process for `test_vllm_inference`'s first cell to inherit +- `test_vllm_inference` catches exceptions, sets `lifecycle.failed = True`, re-raises +- `test_teardown` never skips — must run even on failure; sets `lifecycle.torn_down = True` + to suppress the `orch` fixture's leak-guard finalizer (prevents double teardown) + +`test_metric` verdict pattern: +- Reads value from `inf_res_dict`; attaches to `user_properties` for HTML rendering +- If `enforce_thresholds` and a spec exists for this cell+metric: calls `evaluate_all` + with **the full per-cell actuals dict** (not just the single metric value) so a + `min_ratio` spec can resolve its reference metric +- Otherwise: record-only PASS + +--- + +## conftest fixtures + +All fixtures are `scope="module"`. + +| Fixture | Owns | Key detail | +|---|---|---| +| `cluster_dict` | reads `--cluster_file` JSON; resolves placeholders | calls `resolve_cluster_config_placeholders` | +| `variant_config` | calls `load_variant(config_file, cluster_dict)` | the sole entry point to the typed schema | +| `lifecycle` | `_Lifecycle` instance | shared cross-test state: `failed`, `torn_down`, `report` | +| `orch` | builds `ContainerOrchestrator`; registers leak-guard finalizer | deep-merges variant container block onto cluster container block | +| `hf_token` | reads `variant_config.paths.hf_token_file` | skips if file absent | +| `inf_res_dict` | module-scoped `{}` keyed by `(model_id, gpu_arch, isl, osl, combo_name, concurrency)` | populated by `test_vllm_inference`; consumed by `test_metric` | + +`_deep_merge` helper: `OrchestratorConfig.from_configs` does a top-level `dict.update`, +so a bare variant container block wipes the cluster file's container settings. The conftest +deep-merges the variant block onto the cluster block so cluster-set scalar/dict keys survive +with the variant winning on conflicts. List keys (e.g. runtime args, volume mounts) are +replaced at the merge step and recombined additively downstream in `container.py`'s getters. + +Required pytest hooks (all in `conftest.py`): +- `pytest_collection_modifyitems` — pins lifecycle order; imported functions (e.g. + `test_print_results_table` from `_shared.py`) sort by source line, not insertion order, + so explicit pinning is mandatory +- `pytest_runtest_makereport` — attaches lifecycle timing rows to the HTML detail panel +- `pytest_html_results_table_header` / `pytest_html_results_table_row` — adds Value/Unit + columns; populated for `test_metric` rows, blank for lifecycle/inference rows + +--- + +## `pytest_generate_tests` mirror rule + +`pytest_generate_tests` runs at **collection time**, before any fixtures exist. +It reads the raw config JSON directly — it cannot use the `variant_config` fixture. + +To keep collection-time validation aligned with load-time validation: + +1. Call `validate_sweep_selector` on the raw combo names and run combo refs — mirrors the + typed `Sweep` validator so duplicate names and unknown refs fail collection, not silently drop +2. Validate each raw `goodput_slo` dict through `GoodputSlo(**combo["goodput_slo"])` — + mirrors the `_Forbid` model so a typo'd SLO key fails collection, not runs with the wrong gate + +**Rule**: if you add a check to `Sweep`, add it to `validate_sweep_selector` (or an equivalent +call in `pytest_generate_tests`) so both paths enforce it without drift. + +--- + +## Gotchas + +- **`cell_key` is the single source of truth** — the loader coverage check (`expected_cells`) + and the test verdict lookup (`test_metric`) both call it; change the format in one place and + everything keyed on it moves together. See `docs/cell-key-format.md` for the exact format. + +- **Both axes of `_check_thresholds_cover_sweep` must pass** — without axis 2 a gated metric + with no spec reports zero-assertion PASS even under `enforce_thresholds=true`; the silent + green is indistinguishable from a real pass. + +- **`pytest_generate_tests` reads raw JSON** — it runs before fixtures exist; mirror every + `Sweep` validator via `validate_sweep_selector` or the two paths drift. + +- **`GoodputSlo` is an INPUT, not a threshold** — typo'd key fails load (`_Forbid`); + lives in the sweep config, not `threshold.json`. + +- **`to_client_metrics` is deliberately I/O-free** — the fetch lives in the job class; + callers hand in an already-parsed dict. + +- **Derived metrics degrade to `None` via `_safe_div`, never crash** — `None` renders as + `-` in the HTML table; if the metric is gated, `evaluate_all` will report a loud violation. + +- **All gated-metric threshold keys must use the `client.` prefix** — the axis 2 check + resolves `client.` in the threshold dict, so a bare key (e.g. `total_token_throughput`) + is treated as absent and triggers a coverage failure even though the entry is present. + +- **`client.goodput` is an alias** for stock's `request_goodput` — the names differ; + threshold files must use `client.goodput`, not `client.request_goodput`. + +- **`client_poll_count` controls the total client wait budget**; too low and a long-osl + run times out before the client finishes. Raising it never slows down fast cells. + (Regression: REG-20260609-001) diff --git a/cvs/lib/inference/utils/__init__.py b/cvs/lib/inference/utils/__init__.py new file mode 100644 index 000000000..d3438a6e8 --- /dev/null +++ b/cvs/lib/inference/utils/__init__.py @@ -0,0 +1,4 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' diff --git a/cvs/lib/inference/utils/accuracy_config.py b/cvs/lib/inference/utils/accuracy_config.py new file mode 100644 index 000000000..5f22c5faf --- /dev/null +++ b/cvs/lib/inference/utils/accuracy_config.py @@ -0,0 +1,46 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Accuracy-evaluation config schema, shared across inference suites. + +`AccuracyTask`/`AccuracyConfig` define the `config.json`-side selection schema +for lm-eval-harness based accuracy tasks (see cvs/lib/inference/utils/AGENTS.md +for the broader accuracy-evaluation design). This module holds selection only +-- no threshold/gating values, which live in the sibling threshold.json file +and are joined against `AccuracyConfig.tasks` at runtime by a later unit. +''' + +from __future__ import annotations + +from typing import Any, Dict, List + +from pydantic import model_validator + +from cvs.lib.utils.config_loader import _Forbid + + +class AccuracyTask(_Forbid): + id: str + task: str + num_fewshot: int = 0 + metadata: Dict[str, Any] = {} + include_path: str = "" + num_concurrent: int = 8 + apply_chat_template: bool = False + gen_kwargs: Dict[str, Any] = {} + + +class AccuracyConfig(_Forbid): + tasks: List[AccuracyTask] = [] + + @model_validator(mode="after") + def _check_unique_task_ids(self): + from collections import Counter + + counts = Counter(t.id for t in self.tasks) + dupes = sorted(i for i, n in counts.items() if n > 1) + if dupes: + rendered = ", ".join(repr(d) for d in dupes) + raise ValueError(f"duplicate task id(s): {rendered}") + return self diff --git a/cvs/lib/inference/utils/cache_probe.py b/cvs/lib/inference/utils/cache_probe.py new file mode 100644 index 000000000..4c6457b14 --- /dev/null +++ b/cvs/lib/inference/utils/cache_probe.py @@ -0,0 +1,39 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Model-cache size probing helpers (no pytest dependency). +''' + +from __future__ import annotations + +import shlex + + +def du_bytes(orch, path): + """Bytes under ``path`` in the container. + + Returns 0 when absent or empty. Returns ``None`` when ``du`` cannot run so + callers do not treat infrastructure failure as "model not present". + """ + quoted = shlex.quote(path) + cmd = ( + f"if [ ! -e {quoted} ]; then echo __MISSING__; " + f"elif bytes=$(du -sb {quoted} 2>/dev/null | cut -f1) && [ -n \"$bytes\" ]; " + f"then echo \"$bytes\"; else echo __DU_ERROR__; fi" + ) + out = orch.exec(f"bash -c {shlex.quote(cmd)}") + total = 0 + saw_marker = False + for text in (out or {}).values(): + text = (text or "").strip() + if text == "__DU_ERROR__": + return None + if text == "__MISSING__": + saw_marker = True + continue + if text.isdigit(): + total += int(text) + if saw_marker and total == 0: + return 0 + return total diff --git a/cvs/lib/inference/utils/docs/atom-parsing.md b/cvs/lib/inference/utils/docs/atom-parsing.md new file mode 100644 index 000000000..6d5323e82 --- /dev/null +++ b/cvs/lib/inference/utils/docs/atom-parsing.md @@ -0,0 +1,37 @@ +# ATOM parsing + +`atom_parsing.py` owns the **W1 SLO contract** (`GATED_METRICS`) and +ATOM-specific derived metrics. It reuses `vllm_parsing.to_client_metrics` for the +stock `benchmark_serving` / `vllm bench serve` JSON scalars because ATOM and vLLM +bench artifacts share the same keys. + +## Drivers and artifacts + +| `params.driver` | Result artifact | Parser entry | +| --- | --- | --- | +| `atom` | `{result_stem}.json` from `benchmark_serving` | `to_client_metrics` | +| `vllm`, `vllm_atom` | vLLM bench `{result_stem}` (no `.json` suffix) | `to_client_metrics` | +| `sglang` | SGLang bench log / artifact (client poll on log today) | `to_client_metrics` when JSON present | + +Multinode **scaling** adds `scaling.efficiency_pct` when +`params.scaling_baseline_output_throughput` is set (multinode PP configs). + +## Why not `vllm_parsing` alone? + +- `vllm_single` keeps its own `GATED_METRICS` (legacy suite). +- W1 gates (`per_gpu_throughput`, `output_tput_per_gpu`, tail percentiles) live here. +- Driver choice affects orchestration, not the `client.*` namespace once metrics are parsed. + +## Derived metrics (display + gates) + +| Metric | Formula | +| --- | --- | +| `client.per_gpu_throughput` | `total_token_throughput / tp` (from shared parser) | +| `client.output_tput_per_gpu` | `output_throughput / tp` (added in this module) | +| `scaling.efficiency_pct` | `output_throughput / (baseline × nnodes) × 100` when baseline set | + +## Consumers + +- `atom_orch.AtomJob.parse_results` +- `atom_config_loader` threshold coverage (`gated_metrics=GATED_METRICS`) +- `cvs.tests.inference.atom.atom` — `test_cell_metrics` tiers (throughput, ttft, tpot, health, record) diff --git a/cvs/lib/inference/utils/docs/cell-key-format.md b/cvs/lib/inference/utils/docs/cell-key-format.md new file mode 100644 index 000000000..cb4f7f919 --- /dev/null +++ b/cvs/lib/inference/utils/docs/cell-key-format.md @@ -0,0 +1,116 @@ +# Cell key format + +The cell key is a string that uniquely identifies one sweep cell. It is the single source +of truth shared between: +- `VariantConfig.cell_key()` — generates the key +- `threshold.json` top-level keys — what the key must match +- The test's verdict lookup — reads threshold specs from `variant_config.thresholds` keyed by this string +- pytest parametrize ids — `pytest_generate_tests` builds ids as + `combo_name + "-conc" + concurrency + "-" + metric_short` + (e.g. `w1_isl=1000_osl=1000-conc16-mean_ttft_ms`); the cell key string never + appears in pytest parametrize ids + +--- + +## Format specification + +``` +ISL={isl},OSL={osl},TP={tensor_parallelism},CONC={concurrency} +``` + +**All four fields are required, in this exact order, with no spaces.** + +| Field | Source | Description | +|---|---|---| +| `ISL=` | `SeqCombo.isl` (string from config) | Input sequence length in tokens | +| `OSL=` | `SeqCombo.osl` (string from config) | Output sequence length in tokens | +| `TP=` | `Params.tensor_parallelism` (string from config) | Tensor parallelism degree; comes from `params`, not from the run | +| `CONC=` | `Run.concurrency` (integer from config) | Number of concurrent requests for this run | + +Values are interpolated verbatim from the config fields — no numeric normalization. +`isl` and `osl` are string-typed in the schema; `concurrency` is an integer but +Python's f-string renders it directly. What appears in the config is what appears in +the key. + +**Implementation** (from `VariantConfig.cell_key`): +```python +f"ISL={isl},OSL={osl},TP={self.params.tensor_parallelism},CONC={concurrency}" +``` + +--- + +## Where it is used + +**`threshold.json` top-level keys** — each key names one sweep cell. The coverage +check (`_check_thresholds_cover_sweep`) enforces a two-way match at load time: +every cell key produced by `expected_cells()` must appear as a threshold key, and +every threshold key must name a cell the sweep actually runs. Axis 2 of the same +check verifies every present cell has a spec for every `GATED_METRICS` member. +The spec keys the coverage check looks for are prefixed with `client.` +(e.g., `client.mean_ttft_ms`, `client.output_throughput`) — bare `GATED_METRICS` +names without this prefix will not satisfy the check. + +**`test_metric` verdict lookup** — the test calls +`variant_config.cell_key(isl, osl, concurrency)` to build the key it looks up in +the threshold specs (`variant_config.thresholds.get(cell)`). A format mismatch means +no spec is found, and the test falls through to the record-only branch — PASS with +zero assertions, silently, even under `enforce_thresholds=true`. + +Note: `inf_res_dict` is keyed by the tuple +`(model_id, gpu_arch, isl, osl, combo_name, concurrency)`, which `test_metric` +builds independently before `cell_key` is called. The cell_key string is not used +to index `inf_res_dict`. + +--- + +## Worked example — `w1_llama31_70b_fp8_config.json` + +The config declares `tensor_parallelism: "8"` in `params`, and the sweep has five +cells all at `concurrency: 16`: + +```json +"params": { "tensor_parallelism": "8" }, +"sweep": { + "sequence_combinations": [ + { "name": "w1_isl=1000_osl=1000", "isl": "1000", "osl": "1000" }, + { "name": "w1_isl=8000_osl=1000", "isl": "8000", "osl": "1000" }, + { "name": "w1_isl=1000_osl=8000", "isl": "1000", "osl": "8000" }, + { "name": "w1_isl=1000_osl=4000", "isl": "1000", "osl": "4000" }, + { "name": "w1_isl=5000_osl=1024", "isl": "5000", "osl": "1024" } + ], + "runs": [ + { "combo": "w1_isl=1000_osl=1000", "concurrency": 16 }, + { "combo": "w1_isl=8000_osl=1000", "concurrency": 16 }, + { "combo": "w1_isl=1000_osl=8000", "concurrency": 16 }, + { "combo": "w1_isl=1000_osl=4000", "concurrency": 16 }, + { "combo": "w1_isl=5000_osl=1024", "concurrency": 16 } + ] +} +``` + +`expected_cells()` returns — and `llama31_70b_fp8_threshold.json` must use as top-level keys: + +``` +ISL=1000,OSL=1000,TP=8,CONC=16 +ISL=8000,OSL=1000,TP=8,CONC=16 +ISL=1000,OSL=8000,TP=8,CONC=16 +ISL=1000,OSL=4000,TP=8,CONC=16 +ISL=5000,OSL=1024,TP=8,CONC=16 +``` + +--- + +## Common mistakes + +| Mistake | Effect | +|---|---| +| Adding a space: `ISL=1000, OSL=1000,...` | Key mismatch — no threshold found, no verdict | +| Changing field order: `TP=8,ISL=1000,...` | Key mismatch — no threshold found, no verdict | +| Using a different separator: `ISL=1000\|OSL=1000\|...` | Key mismatch — no threshold found, no verdict | +| Normalizing values: `ISL=1024` when config says `"1000"` | Key mismatch — values are verbatim from the config string | +| Typo in threshold.json key | Axis 1 of `_check_thresholds_cover_sweep` catches this at load time | + +Any key mismatch silently drops the cell — the test finds no spec and falls through to the +record-only branch, reporting PASS with zero assertions. The load-time coverage check +(`_check_thresholds_cover_sweep`) is what catches this before the test runs: when +`enforce_thresholds=true` it raises `ValueError`; when false it emits a warning. diff --git a/cvs/lib/inference/utils/docs/derived-metrics.md b/cvs/lib/inference/utils/docs/derived-metrics.md new file mode 100644 index 000000000..99596a270 --- /dev/null +++ b/cvs/lib/inference/utils/docs/derived-metrics.md @@ -0,0 +1,159 @@ +# Derived metrics + +`to_client_metrics` in `vllm_parsing.py` appends five derived metrics on top of +the stock `vllm bench serve` scalars. Every derived metric is guarded by `_safe_div`. + +--- + +## `_safe_div` contract + +```python +def _safe_div(num, den): + if num is None or den is None: + return None + try: + den = float(den) + if den == 0: + return None + return float(num) / den + except (TypeError, ValueError): + return None +``` + +Returns `None` when: + +- Either operand is `None` (missing key or explicit null in the artifact) +- The divisor is `0` +- Either operand cannot be converted to `float` (`TypeError` or `ValueError`) + +Never raises `TypeError` or `ValueError`; never returns a bogus `0`. Note: `OverflowError` is not caught — callers must ensure artifact values are within the representable float range. + +### What `None` means downstream + +| Context | Behaviour | +|---|---| +| HTML results table | Displayed as `-` | +| Record-only metric (not in `GATED_METRICS`) | Silently recorded; no assertion | +| Gated metric with a threshold spec | `evaluate_all` raises `ThresholdViolation` — the violation string names the metric and states the value is `None` (metric unavailable for this run) | + +A `None` gated metric is always a loud failure, never a silent skip. + +--- + +## Derived metrics + +### `client.per_gpu_throughput` + +**Formula**: `total_token_throughput / tp` + +**Inputs from artifact**: `total_token_throughput` (stock scalar) + +**Out-of-band input**: `tp` — tensor parallelism, passed by the caller from +`params.tensor_parallelism` + +**None when**: `total_token_throughput` is missing or `None`, `tp` is `0` or +unconvertible to `float`. + +**Gated or record-only**: record-only. Secondary throughput used for per-GPU +capacity analysis. `total_token_throughput` (the primary throughput) is gated +instead. + +--- + +### `client.normalized_ttft_ms_per_tok` + +**Formula**: `mean_ttft_ms / isl` + +**Inputs from artifact**: `mean_ttft_ms` (stock scalar) + +**Out-of-band input**: `isl` — input sequence length, passed by the caller from +the sweep `SeqCombo.isl` + +**None when**: `mean_ttft_ms` is missing or `None`, `isl` is `0` or +unconvertible. + +**Gated or record-only**: record-only. Diagnostic derivation — normalises TTFT +by input length to expose prefill efficiency across cells with different ISLs. +The per-quantile `*_ttft_ms` metrics are gated instead. + +--- + +### `client.decode_latency_ratio` + +**Formula**: `p99_itl_ms / p50_itl_ms` + +**Inputs from artifact**: `p99_itl_ms` (stock scalar), `p50_itl_ms` (looked up +via `raw.get("p50_itl_ms")`) + +**Important**: real `vllm bench serve` artifacts emit `median_itl_ms` for the +50th-percentile ITL, **not** `p50_itl_ms`. The source calls +`raw.get("p50_itl_ms")`, so this key is absent from every real artifact and the +denominator is always `None`. As a result, `client.decode_latency_ratio` is +**always `None` on real runs** — it is record-only by design and the `None` +value is silently recorded (no assertion, displayed as `-` in the results +table). + +**None when**: `p50_itl_ms` is absent from the artifact (always, with real +runs), or either input is `None`, or `p50_itl_ms` is `0`. + +**Gated or record-only**: record-only. Diagnostic derivation — intended to +measure tail latency inflation (how much worse p99 ITL is than the median). The +individual quantile ITL metrics (`p95_itl_ms`, `p99_itl_ms`, etc.) are gated +instead. + +--- + +### `client.decode_throughput_p50` + +**Formula**: `1000.0 / median_tpot_ms` + +**Inputs from artifact**: `median_tpot_ms` (stock scalar) + +**None when**: `median_tpot_ms` is missing, `None`, or `0`. + +**Gated or record-only**: record-only. Secondary throughput — converts median +time-per-output-token to tokens-per-second for human-readable throughput +comparisons. `median_tpot_ms` (the latency metric) is gated instead. + +--- + +### `client.success_rate` + +**Formula**: `completed / (completed + failed)` + +**Inputs from artifact**: `completed`, `failed` (both stock scalars) + +**Implementation note**: the denominator is computed as +`completed + failed` only when both are non-`None`; if either is `None`, the +total is set to `None` and `_safe_div` returns `None`. + +```python +total_req = None if completed is None or failed is None else completed + failed +m["client.success_rate"] = _safe_div(completed, total_req) +``` + +**None when**: either `completed` or `failed` is missing or `None`. + +**Gated or record-only**: **gated**. Run health: a floor on the fraction of +requests that completed without error. A success rate below threshold indicates +infrastructure failures, OOM, or timeouts — not a performance regression. + +--- + +## `client.goodput` alias + +```python +m["client.goodput"] = raw.get("request_goodput") +``` + +Stock `vllm bench serve` emits the key `request_goodput`. The results table and +`threshold.json` reference it as `client.goodput` (without the `request_` prefix) +for readability. The alias is set unconditionally alongside the stock +`client.request_goodput` entry that the 1:1 namespace copy also produces. + +If `--goodput` was not passed to `vllm bench serve`, stock emits +`request_goodput: null`, so `client.goodput` will be `None`. + +**Gated or record-only**: record-only. Goodput is only meaningful when a +`GoodputSlo` was configured in the sweep, and the value reflects the SLO input +rather than an independent throughput measurement. diff --git a/cvs/lib/inference/utils/inference_suite_lifecycle.py b/cvs/lib/inference/utils/inference_suite_lifecycle.py new file mode 100644 index 000000000..97f2ba4ea --- /dev/null +++ b/cvs/lib/inference/utils/inference_suite_lifecycle.py @@ -0,0 +1,280 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Reusable **lifecycle-as-tests** helpers for DTNI inference suites. + +``atom`` imports the stage tests from here today; other suites +(``vllm_single``, future IX parity frameworks) can reuse the same module instead +of copying launch / sshd / model-fetch / teardown blocks. + +**Suite module** — import stage tests so pytest collects them:: + + from cvs.lib.inference.utils.inference_suite_lifecycle import ( + test_accuracy_eval, + test_launch_container, + test_model_fetch, + test_setup_sshd, + test_teardown, + ) + +**conftest.py** — wire shared fixtures and HTML hooks:: + + from cvs.lib.inference.utils.inference_suite_lifecycle import ( + InferenceLifecycle, + attach_lifecycle_html_table, + html_metric_table_header, + html_metric_table_row, + sort_lifecycle_items, + ) + +Also provides ``sweep_cell_result_key``; see :mod:`cvs.lib.inference.utils.cache_probe` for ``du_bytes``. + +Optional HTML/JSON suite report: add ``cvs/lib/report/presets/.py`` (see +``cvs/lib/report/README.md``); root ``cvs/conftest.py`` auto-wires hooks when ``--html`` is set. +''' + +from __future__ import annotations + +import shlex +import time + +import pytest + +try: + import pytest_html +except ImportError: + pytest_html = None + +from cvs.lib import globals +from cvs.lib.inference.utils.cache_probe import du_bytes +from cvs.lib.inference.utils.lm_eval_job import run_accuracy_tasks +from cvs.lib.utils.verdict import evaluate_all + +log = globals.log + +FETCH_POLL_COUNT = 80 +FETCH_POLL_WAIT_S = 30 +FETCH_PRESENCE_RETRIES = 5 + + +class InferenceLifecycle: + """Cross-test state shared by lifecycle stage tests in one module scope.""" + + def __init__(self): + self.failed = False + self.torn_down = False + self.report = {} + + def record(self, nodeid, label, value, unit="s"): + self.report.setdefault(nodeid, []).append((label, value, unit)) + + +def sweep_cell_result_key(variant_config, seq_combo, isl, osl, concurrency): + """Canonical ``inf_res_dict`` key for one sweep cell.""" + return ( + variant_config.model.id, + variant_config.gpu_arch, + isl, + osl, + seq_combo.get("name", "default"), + concurrency, + ) + + +def test_launch_container(orch, variant_config, lifecycle, request): + """Stage 1: launch the container.""" + t = time.monotonic() + ok = orch.setup_containers() + lifecycle.record(request.node.nodeid, "container_launch", time.monotonic() - t) + if not ok: + lifecycle.failed = True + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + pytest.fail(f"setup_containers() returned False for {name}") + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + if not orch.verify_containers_running(name): + lifecycle.failed = True + pytest.fail(f"container {name} not running after setup_containers()") + + +def test_setup_sshd(orch, lifecycle, request): + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + from cvs.core.orchestrators.container import sshd_port_listen_ok, sshd_port_listen_probe_cmd + + t = time.monotonic() + ok = orch.setup_sshd() + lifecycle.record(request.node.nodeid, "sshd_setup", time.monotonic() - t) + if not ok: + lifecycle.failed = True + pytest.fail("setup_sshd() returned False") + if len(orch.hosts) > 1: + probe = orch.exec(sshd_port_listen_probe_cmd(getattr(orch, "ssh_port", 2224))) + if not any(sshd_port_listen_ok(v) for v in (probe or {}).values()): + lifecycle.failed = True + pytest.fail("sshd not listening on 2224 after setup_sshd()") + + +def test_model_fetch(orch, variant_config, lifecycle, request): + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + models_dir = variant_config.paths.models_dir + if not models_dir: + pytest.skip("paths.models_dir unset; cannot locate/verify the HF cache") + + remote = getattr(variant_config.model, "remote", 0) + t = time.monotonic() + orch.exec(f"mkdir -p {shlex.quote(models_dir)}") + + if not remote: + final = 0 + for it in range(FETCH_PRESENCE_RETRIES): + cur = du_bytes(orch, models_dir) + if cur is None: + lifecycle.failed = True + pytest.fail(f"could not measure model cache size under {models_dir} (du error)") + final = cur + log.info("[fetch presence %d] size=%.1fGB", it, final / 1e9) + if final > 0: + break + time.sleep(FETCH_POLL_WAIT_S) + else: + fetch = ( + f"HF_HUB_CACHE={shlex.quote(models_dir)} " + f"nohup hf download {shlex.quote(variant_config.model.id)} " + f"> /tmp/hf_fetch.log 2>&1 &" + ) + orch.exec("bash -c " + shlex.quote(fetch)) + prev = -1 + stable = 0 + final = du_bytes(orch, models_dir) + if final is None: + lifecycle.failed = True + pytest.fail(f"could not measure model cache size under {models_dir} (du error)") + for it in range(FETCH_POLL_COUNT): + cur = du_bytes(orch, models_dir) + if cur is None: + lifecycle.failed = True + pytest.fail(f"could not measure model cache size under {models_dir} (du error)") + final = cur + log.info("[fetch poll %d] size=%.1fGB", it, cur / 1e9) + if cur > 0 and cur == prev: + stable += 1 + if stable >= 2: + break + else: + stable = 0 + prev = cur + time.sleep(FETCH_POLL_WAIT_S) + + lifecycle.record(request.node.nodeid, "model_fetch", time.monotonic() - t) + lifecycle.record(request.node.nodeid, "model_size", final / 1e9, "GB") + if final <= 0: + lifecycle.failed = True + pytest.fail(f"no model bytes under {models_dir} after fetch") + + +def test_accuracy_eval(orch, variant_config, accuracy_task, lifecycle, request): + """Opt-in stage: run one lm-eval-harness accuracy task against the already-live server. + + One pytest test (= one HTML row) per accuracy task, parametrized by + `accuracy_task` (a task id from config.json's `accuracy.tasks`, an + AccuracyConfig). Each task is gated/reported independently: a failure or + threshold violation in one task's node does not skip or fail its sibling + tasks' nodes -- unlike the shared `lifecycle.failed` flag used by the rest + of the lifecycle, which is intentionally NOT set here. + + An absent `accuracy` block or empty `tasks: []` means this suite run has + no accuracy tasks configured; `pytest_generate_tests` parametrizes with an + empty list in that case, which pytest auto-skips as a single node -- same + convention as a perf metric with no threshold entry. Gating values live in + the sibling threshold.json's `accuracy` block, keyed by task id (see + cvs/lib/inference/utils/AGENTS.md for the full design). + """ + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + accuracy_config = getattr(variant_config, "accuracy", None) + tasks_by_id = {task.id: task for task in (accuracy_config.tasks if accuracy_config else [])} + task = tasks_by_id.get(accuracy_task) + if task is None: + pytest.skip(f"accuracy task {accuracy_task!r} not present in accuracy.tasks") + + params = variant_config.params + output_dir = f"{variant_config.paths.log_dir}/accuracy" + + t = time.monotonic() + try: + actuals_by_id = run_accuracy_tasks( + orch=orch, + tasks=[task], + base_url=f"{params.base_url}:{params.port_no}", + model_id=variant_config.model.id, + model_path=variant_config.model.id, + output_dir=output_dir, + ) + except RuntimeError as e: + lifecycle.record(request.node.nodeid, "accuracy_eval", time.monotonic() - t) + pytest.fail(str(e)) + lifecycle.record(request.node.nodeid, "accuracy_eval", time.monotonic() - t) + + actual = actuals_by_id.get(accuracy_task, {}) + for metric_key, value in actual.items(): + lifecycle.record(request.node.nodeid, f"{accuracy_task}.{metric_key}", value, "") + + if not variant_config.enforce_thresholds: + return + accuracy_thresholds = (variant_config.thresholds or {}).get("accuracy", {}) + evaluate_all(actual, accuracy_thresholds.get(accuracy_task, {})) + + +def test_teardown(orch, lifecycle, request): + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + t = time.monotonic() + orch.teardown_containers() + lifecycle.record(request.node.nodeid, "teardown", time.monotonic() - t) + if orch.verify_containers_running(name): + pytest.fail(f"container {name} still running after teardown_containers()") + lifecycle.torn_down = True + + +def sort_lifecycle_items(items, rank): + items.sort(key=lambda it: rank.get(it.originalname or it.name.split("[")[0], 99)) + + +def attach_lifecycle_html_table(item, report): + if report.when != "call": + return + lc = item.funcargs.get("lifecycle") + rows = getattr(lc, "report", {}).get(item.nodeid) if lc else None + if not rows: + return + if pytest_html is None: + return + body = "".join(f"{label}{value:.1f}{unit}" for label, value, unit in rows) + html = f"{body}
stagevalueunit
" + extras = getattr(report, "extras", []) + extras.append(pytest_html.extras.html(html)) + report.extras = extras + + +# def html_metric_table_header(cells): +# cells.insert(-1, "Value") +# cells.insert(-1, "Unit") +# +# +# def html_metric_table_row(report, cells): +# props = dict(report.user_properties) +# has = "metric_value" in props +# val = props.get("metric_value") +# unit = props.get("metric_unit", "") if has else "" +# if not has: +# shown = "" +# elif val is None: +# shown = "-" +# elif isinstance(val, float): +# shown = f"{val:.3f}" +# else: +# shown = str(val) +# cells.insert(-1, f"{shown}") +# cells.insert(-1, f"{unit}") diff --git a/cvs/lib/inference/utils/inference_suite_results_table.py b/cvs/lib/inference/utils/inference_suite_results_table.py new file mode 100644 index 000000000..c9d5ea5e7 --- /dev/null +++ b/cvs/lib/inference/utils/inference_suite_results_table.py @@ -0,0 +1,93 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Reusable end-of-session results table for DTNI inference suites. + +Pair with ``cvs.lib.report`` suite reports for an optional HTML/JSON dashboard +that uses the same column preset and ``inf_res_dict`` keys. + +Bind a column preset with ``make_print_results_table(columns)`` and export the +returned callable as ``test_print_results_table`` from the suite module (see +``cvs/tests/inference/atom/_shared.py``). Other suites can supply +their own ``(header_label, client.*_key)`` tuples without duplicating the +tabulate loop. +''' + +from __future__ import annotations + +from cvs.lib import globals + +log = globals.log + +# Column tuple: ``(header label, client.* metric key or None for fixed key fields)``. +# First seven columns are always Model, GPU, ISL, OSL, Policy, Conc, Host. + +ATOM_RESULTS_COLUMNS = ( + ("Model", None), + ("GPU", None), + ("ISL", None), + ("OSL", None), + ("Policy", None), + ("Conc", None), + ("Host", None), + ("Output tok/s", "client.output_throughput"), + ("Total tok/s", "client.total_token_throughput"), + ("Mean TTFT (ms)", "client.mean_ttft_ms"), + ("Mean TPOT (ms)", "client.mean_tpot_ms"), + ("P99 ITL (ms)", "client.p99_itl_ms"), + ("Scaling eff. (%)", "scaling.efficiency_pct"), +) + +# Optional preset for suites that want vLLM-style columns (not wired in vllm_single yet). +VLLM_SINGLE_RESULTS_COLUMNS = ( + ("Model", None), + ("GPU", None), + ("ISL", None), + ("OSL", None), + ("Policy", None), + ("Conc", None), + ("Host", None), + ("Req/s", "client.request_throughput"), + ("Total tok/s", "client.total_token_throughput"), + ("Mean TTFT (ms)", "client.mean_ttft_ms"), + ("P95 TTFT (ms)", "client.p95_ttft_ms"), + ("Mean TPOT (ms)", "client.mean_tpot_ms"), + ("P95 TPOT (ms)", "client.p95_tpot_ms"), + ("P99 ITL (ms)", "client.p99_itl_ms"), + ("Goodput (req/s)", "client.goodput"), +) + + +def _cell(metrics, key): + v = metrics.get(key) + return "-" if v is None else v + + +def print_results_table(inf_res_dict, columns): + from tabulate import tabulate + + if not inf_res_dict: + log.info("inf_res_dict empty, nothing to print") + return + headers = [label for label, _key in columns] + metric_keys = [key for _label, key in columns] + rows = [] + for key, host_dict in inf_res_dict.items(): + model, gpu, isl, osl, policy, conc = key + fixed = [model, gpu, isl, osl, policy, conc] + for host, metrics in host_dict.items(): + row = list(fixed) + row.append(host) + row.extend(_cell(metrics, mk) if mk else "-" for mk in metric_keys[7:]) + rows.append(row) + log.info("\n" + tabulate(rows, headers=headers, tablefmt="github")) + + +def make_print_results_table(columns): + """Return a ``test_print_results_table(inf_res_dict)`` callable for one column spec.""" + + def test_print_results_table(inf_res_dict): + print_results_table(inf_res_dict, columns) + + return test_print_results_table diff --git a/cvs/lib/inference/utils/inferencing_config_loader.py b/cvs/lib/inference/utils/inferencing_config_loader.py new file mode 100644 index 000000000..8be2d6a34 --- /dev/null +++ b/cvs/lib/inference/utils/inferencing_config_loader.py @@ -0,0 +1,224 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Inference-specific config schema for the vllm_single suite. + +The framework-agnostic machinery (paths/model/image/container schema, the +3-pass placeholder substitution, the `enforce_thresholds` gate, and the +`substitute_config` file-read helper) lives in `cvs.lib.utils.config_loader`. +This module holds the inference half: the sweep selector +(`SeqCombo`/`Run`/`Sweep`), the goodput SLO, the framework `Params`, the +`server` role, and `VariantConfig(BaseVariantConfig)` with the +ISL/OSL/TP/CONC `cell_key` and its sweep-coverage check. + +`Sweep`/`SeqCombo`/`GoodputSlo`/`Roles`/`cell_key` are inference-generic (any +serving framework sweeps sequence shapes at concurrencies); only `Params` (the +`vllm bench serve` flags) is framework-flavored and is the seam to subclass +when a second serving framework lands. +''' + +from __future__ import annotations + +import warnings +from collections import Counter +from typing import Any, Dict, List, Optional + +from pydantic import model_validator +from typing_extensions import Literal + +from cvs.lib.utils.config_loader import BaseVariantConfig, _Forbid, substitute_config +from cvs.lib.inference.utils.vllm_parsing import GATED_METRICS + + +# ---------- pydantic models (inference) ---------- + + +class RoleServer(_Forbid): + # The `vllm serve` command is built in Python (cvs.lib.inference.vllm_single), + # not cloned from a `.sh` script, so a run is self-contained. Per-model + # server quirks live here: extra `vllm serve` flags (serve_args) and env vars + # merged over the defaults the orchestrator sets. Both default empty; the + # fp8-kv cell sets its --kv-cache-dtype via serve_args, kept out of the + # generic driver so it stays model-agnostic. A {flag: value} map (flag + # without the leading --): a scalar renders `--flag value`, True renders a + # bare `--flag`, and a list renders the flag once per element. Cleaner than + # a flat [flag, value, flag, value] list and still covers vllm's bare and + # repeatable flags. + serve_args: Dict[str, Any] = {} + env: Dict[str, str] = {} + + +class Roles(_Forbid): + server: RoleServer = RoleServer() + + +class GoodputSlo(_Forbid): + # Per-cell SLOs for the goodput gate, in milliseconds. An INPUT to the run + # (passed to `vllm bench serve --goodput`), NOT a threshold to assert -- so + # it lives in the sweep, not threshold.json. Attached per seq_combo because + # e2el scales ~linearly with osl. _Forbid: a typo'd key fails load, not a + # silently-dropped SLO. + ttft_ms: float + tpot_ms: float + e2el_ms: float + + +class SeqCombo(_Forbid): + # `name` is the join key the `runs` selector references; required. + name: str + isl: str + osl: str + goodput_slo: Optional[GoodputSlo] = None + + +class Run(_Forbid): + # One sweep cell: a named combo run at a single concurrency. The explicit + # list of Runs replaces the old `sequence_combinations x concurrency_levels` + # cartesian -- you enumerate exactly the cells you want, no NxM explosion. + combo: str + concurrency: int + + +NON_SWEEP_THRESHOLD_KEYS = {"accuracy"} + + +def validate_thresholds_cover_sweep( + *, + expected_cells, + thresholds, + enforce_thresholds: bool, + gated_metrics=None, + gated_gpu_metrics=None, +) -> None: + """Shared sweep/threshold coverage check for inference variant configs.""" + expected = set(expected_cells) + present = set(thresholds.keys()) - NON_SWEEP_THRESHOLD_KEYS + missing = sorted(expected - present) + extra = sorted(present - expected) + problems = [] + if missing: + problems.append(f"sweep cells with no threshold entry: {missing}") + if extra: + problems.append(f"threshold keys matching no sweep cell (typo?): {extra}") + gated = gated_metrics if gated_metrics is not None else GATED_METRICS + gated_keys = [f"client.{m}" for m in sorted(gated)] + if gated_gpu_metrics: + gated_keys += [f"gpu.{m}" for m in sorted(gated_gpu_metrics)] + gated_gaps = {} + for cell in sorted(expected & present): + specs = thresholds.get(cell) or {} + absent = [k for k in gated_keys if k not in specs] + if absent: + gated_gaps[cell] = absent + if gated_gaps: + problems.append(f"cells missing gated-metric specs: {gated_gaps}") + if problems: + msg = "threshold.json does not match the sweep matrix; " + "; ".join(problems) + if enforce_thresholds: + raise ValueError(msg) + warnings.warn(f"{msg} (enforce_thresholds=false -> record-only)", stacklevel=3) + + +def validate_sweep_selector(combo_names, run_combo_refs): + """The sweep-selector rule: combo names unique, every run.combo names one. + + The single home for this check, shared by the typed `Sweep` validator (load + time) and `pytest_generate_tests` (collection time, which reads raw JSON + before the loader runs) so the two can never drift. Operates on plain lists + of strings so both call sites can feed it. + + Without it a duplicate name silently shadows a combo and a typo'd + `run.combo` is a silently-dropped cell -- either way the sweep runs a + different matrix than the config reads. + """ + counts = Counter(combo_names) + dupes = sorted(name for name, count in counts.items() if count > 1) + if dupes: + raise ValueError(f"duplicate sequence_combination names: {dupes}") + known = set(counts) + unknown = sorted({r for r in run_combo_refs if r not in known}) + if unknown: + raise ValueError(f"run.combo names no sequence_combination: {unknown} (known: {sorted(known)})") + + +class Sweep(_Forbid): + sequence_combinations: List[SeqCombo] + runs: List[Run] + + @model_validator(mode="after") + def _check_runs_reference_known_combos(self): + validate_sweep_selector( + [c.name for c in self.sequence_combinations], + [r.combo for r in self.runs], + ) + return self + + +class Params(_Forbid): + backend: str = "vllm" + base_url: str = "http://0.0.0.0" + port_no: str = "8888" + dataset_name: str = "random" + burstiness: str = "1.0" + seed: str = "0" + request_rate: str = "inf" + random_range_ratio: str = "0.8" + random_prefix_len: str = "0" + tensor_parallelism: str = "1" + tokenizer_mode: str = "auto" + percentile_metrics: str = "ttft,tpot,itl,e2el" + metric_percentiles: str = "50,90,95,99" + num_prompts: str = "3200" + # Completion-poll budget for the bench client = client_poll_count * 60s + # (plus a 120s initial wait). Large-output cells (high osl) need a bigger + # budget; the poll loop exits as soon as the client finishes, so raising + # this never slows down fast cells. See regressions REG-20260609-001. + client_poll_count: str = "20" + + +class VariantConfig(BaseVariantConfig): + framework: Literal["vllm_single"] + gpu_arch: str + roles: Roles = Roles() + params: Params + sweep: Sweep + + def cell_key(self, isl, osl, concurrency): + """The canonical threshold key for one sweep cell. + + Single source of truth shared by the loader's coverage check and the + test's verdict lookup -- so the two can never drift on whitespace, + ordering, or field names. + """ + return f"ISL={isl},OSL={osl},TP={self.params.tensor_parallelism},CONC={concurrency}" + + def expected_cells(self): + """Every (isl, osl, conc) cell the sweep's `runs` selector picks.""" + by_name = {c.name: c for c in self.sweep.sequence_combinations} + return [self.cell_key(by_name[r.combo].isl, by_name[r.combo].osl, r.concurrency) for r in self.sweep.runs] + + @model_validator(mode="after") + def _check_thresholds_cover_sweep(self): + """Fail at load time if any sweep cell lacks a threshold entry.""" + validate_thresholds_cover_sweep( + expected_cells=self.expected_cells(), + thresholds=self.thresholds, + enforce_thresholds=self.enforce_thresholds, + ) + return self + + +# ---------- public API (inference) ---------- + + +def load_variant(config_path, cluster_dict): + """Load and validate a vllm_single variant config + its sibling threshold file. + + Delegates the file read + placeholder substitution + threshold discovery to + the generic `substitute_config`, then attaches the thresholds and builds the + typed `VariantConfig`. + """ + raw, thresholds = substitute_config(config_path, cluster_dict) + raw["thresholds"] = thresholds + return VariantConfig(**raw) diff --git a/cvs/lib/inference/utils/lm_eval_job.py b/cvs/lib/inference/utils/lm_eval_job.py new file mode 100644 index 000000000..baef6ff9d --- /dev/null +++ b/cvs/lib/inference/utils/lm_eval_job.py @@ -0,0 +1,145 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +lm-eval-harness command construction and execution against an already-live +inference server (see cvs/lib/inference/utils/AGENTS.md for the broader +accuracy-evaluation design). Routes through `orch.exec_on_head` rather than a +raw `docker exec`, matching current suite conventions (mirrors +`VllmJob.run_client`'s head-only execution rationale). +''' + +from __future__ import annotations + +import json +import shlex +from dataclasses import dataclass +from typing import Any, Dict, List + +from cvs.lib.inference.utils.accuracy_config import AccuracyTask +from cvs.lib.inference.utils.lm_eval_parsing import project + +LM_EVAL_INSTALL_CHECK_CMD = ( + "python -c 'import lm_eval, math_verify' 2>/dev/null || pip install -q 'lm-eval[api,math]>=0.4.4'" +) +PER_TASK_TIMEOUT_S = 4 * 60 * 60 + + +@dataclass +class LmEvalCtx: + base_url: str + model_id: str + model_path: str + output_dir: str + + +def build_lm_eval_cmd(task: AccuracyTask, ctx: LmEvalCtx) -> str: + model_flag = "local-chat-completions" if task.apply_chat_template else "local-completions" + endpoint_path = "/v1/chat/completions" if task.apply_chat_template else "/v1/completions" + + model_args = ",".join( + [ + f"base_url={ctx.base_url}{endpoint_path}", + f"model={ctx.model_id}", + f"tokenizer={ctx.model_path}", + "tokenizer_backend=huggingface", + f"num_concurrent={task.num_concurrent}", + "max_retries=3", + "trust_remote_code=True", + ] + ) + + args = [ + "lm_eval", + "--model", + model_flag, + "--model_args", + model_args, + "--tasks", + task.task, + "--num_fewshot", + str(task.num_fewshot), + "--output_path", + f"{ctx.output_dir}/{task.id}", + "--log_samples", + ] + + if task.apply_chat_template: + args.append("--apply_chat_template") + + if task.metadata: + args += ["--metadata", json.dumps(task.metadata)] + + if task.include_path: + args += ["--include_path", task.include_path] + + if task.gen_kwargs: + gen_kwargs = ",".join(f"{k}={v}" for k, v in task.gen_kwargs.items()) + args += ["--gen_kwargs", gen_kwargs] + + lm_eval_cmd = " ".join(shlex.quote(str(a)) for a in args) + return f"source /tmp/server_env_script.sh && {LM_EVAL_INSTALL_CHECK_CMD} && {lm_eval_cmd}" + + +def run_accuracy_tasks( + *, + orch: Any, + tasks: List[AccuracyTask], + base_url: str, + model_id: str, + model_path: str, + output_dir: str, +) -> Dict[str, Dict[str, float]]: + ctx = LmEvalCtx(base_url=base_url, model_id=model_id, model_path=model_path, output_dir=output_dir) + results: Dict[str, Dict[str, float]] = {} + + for task in tasks: + cmd = build_lm_eval_cmd(task, ctx) + out = orch.exec_on_head(cmd, timeout=PER_TASK_TIMEOUT_S, detailed=True) + try: + (run_result,) = out.values() + except ValueError as e: + raise RuntimeError( + f"lm_eval task {task.id!r}: expected exactly one exec_on_head result, got {len(out)}: {e}" + ) from e + run_output = (run_result or {}).get("output") or "" + exit_code = (run_result or {}).get("exit_code", -1) + if exit_code != 0: + raise RuntimeError( + f"lm_eval task {task.id!r} exited with code {exit_code} " + f"-- treating as a run failure. Command output tail: {run_output[-2000:]!r}" + ) + + task_out_dir = f"{output_dir}/{task.id}" + find_cmd = f"find {shlex.quote(task_out_dir)} -name 'results*.json' -printf '%T@ %p\\n' | sort -rn" + find_out = orch.exec_on_head(find_cmd) + try: + (find_output,) = find_out.values() + except ValueError as e: + raise RuntimeError( + f"lm_eval task {task.id!r}: expected exactly one exec_on_head result for find, got {len(find_out)}: {e}" + ) from e + lines = (find_output or "").strip().splitlines() + result_path = lines[0].split(" ", 1)[1] if lines else "" + + if not result_path: + raise RuntimeError( + f"lm_eval task {task.id!r} produced no results*.json under {task_out_dir} " + f"-- treating as a run failure (install or execution error). " + f"Command output tail: {run_output[-2000:]!r}" + ) + + cat_out = orch.exec_on_head(f"cat {shlex.quote(result_path)}") + try: + (payload_text,) = cat_out.values() + except ValueError as e: + raise RuntimeError( + f"lm_eval task {task.id!r}: expected exactly one exec_on_head result for cat, got {len(cat_out)}: {e}" + ) from e + try: + payload = json.loads(payload_text) + except json.JSONDecodeError as e: + raise RuntimeError(f"lm_eval task {task.id!r} produced unparseable results at {result_path}: {e}") from e + results[task.id] = project(payload) + + return results diff --git a/cvs/lib/inference/utils/lm_eval_parsing.py b/cvs/lib/inference/utils/lm_eval_parsing.py new file mode 100644 index 000000000..98ed63731 --- /dev/null +++ b/cvs/lib/inference/utils/lm_eval_parsing.py @@ -0,0 +1,34 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Pure JSON -> {scalar: float} projector for lm-eval-harness `results.json` +payloads (see cvs/lib/inference/utils/AGENTS.md for the broader +accuracy-evaluation design). Auto-discovers every numeric metric rather than +requiring a per-task registry, so group tasks (e.g. RULER's per-seq-length +metrics) and custom tasks fall out of the same walk with no special-casing. +''' + +from __future__ import annotations + +import math +from typing import Any, Dict + + +def _is_real_number(value: Any) -> bool: + if isinstance(value, bool): + return False + if not isinstance(value, (int, float)): + return False + return not math.isnan(value) + + +def project(payload: Dict[str, Any]) -> Dict[str, float]: + out = {} + for lm_task_name, metrics in payload.get("results", {}).items(): + for metric_key, value in metrics.items(): + if metric_key == "alias" or not _is_real_number(value): + continue + key = f"{lm_task_name}.{metric_key}".replace(",", "__") + out[key] = float(value) + return out diff --git a/cvs/lib/inference/utils/vllm_benchmark_scripts/README.md b/cvs/lib/inference/utils/vllm_benchmark_scripts/README.md new file mode 100644 index 000000000..73d14ed45 --- /dev/null +++ b/cvs/lib/inference/utils/vllm_benchmark_scripts/README.md @@ -0,0 +1,17 @@ +# vLLM benchmark server scripts (shared) + +Shell entrypoints for **`vllm serve`** kept for legacy **InferenceBaseJob** flows. + +- **`vllm_serve_mi300x.sh`** — reference MI300-class server flags (``enforce-eager``, ``gpu-memory-utilization``, etc.). **ATOM** and **vllm_single** encode equivalent flags in ``roles.server.serve_args`` instead of running this script. + +**Client benchmarks** use ``vllm bench serve`` (stock results artifact). CVS no longer clones a third-party ``bench_serving`` git repo for ATOM. + +If **both** the script path and the bench CLI are unavailable, install bench-capable vLLM in the image (e.g. `pip install 'vllm[bench]'`) or bind-mount a matching `benchmarks/` tree from a vLLM checkout. + +Client invocations also pass ``--temperature 0`` so greedy sampling matches the legacy +``benchmark_serving.py`` default, and clamp ``--random-range-ratio`` when the configured +spread would let ``(ISL+OSL)*(1+r)`` exceed ``max_model_length`` (otherwise vLLM rejects +most random prompts). + +Python API: ``bundled_scripts_dir()``, ``bash_export_bench_script_from_vllm_install()``, +``clamped_bench_random_range_ratio_str()``, ``validated_bench_script_basename()``. diff --git a/cvs/lib/inference/utils/vllm_benchmark_scripts/__init__.py b/cvs/lib/inference/utils/vllm_benchmark_scripts/__init__.py new file mode 100644 index 000000000..0842be5c1 --- /dev/null +++ b/cvs/lib/inference/utils/vllm_benchmark_scripts/__init__.py @@ -0,0 +1,200 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Canonical **vLLM benchmark server** shell helpers retained for legacy +:class:`~cvs.lib.inference.base.InferenceBaseJob` flows. **ATOM** and +**vllm_single** (:class:`~cvs.lib.inference.vllm_single.VllmJob`) build +``vllm serve`` in Python via ``roles.server.serve_args``; they do not stage +these scripts at runtime. + +Client load generation uses the **vLLM install’s** ``benchmarks/ + + + + +
+
+
+

__TITLE__

+

__SUBTITLE__

+
+ — +
+ + + +
+

Filters

+
+ + + + + + + +
+
Loading…
+
+ +
+

Overview

+
+
+ + + + + + + + + +
+

Cells

+
+
+
+
+
+ +__EMBEDDED_JSON__ + + + + diff --git a/cvs/lib/report/viewer/scaffold.py b/cvs/lib/report/viewer/scaffold.py new file mode 100644 index 000000000..e39996c94 --- /dev/null +++ b/cvs/lib/report/viewer/scaffold.py @@ -0,0 +1,52 @@ +'''Interactive viewer: filterable table, Chart.js concurrency charts, heatmap, gate matrix.''' + +from __future__ import annotations + +import html +import json +from pathlib import Path +from typing import Any, Mapping, Optional + +from cvs.lib.report.artifacts import export_payload + +_TEMPLATE_PATH = Path(__file__).with_name("interactive.html") + + +def _embedded_json_script(payload: Mapping[str, Any]) -> str: + """Inline JSON so the viewer works when opened via file:// (fetch is blocked).""" + raw = json.dumps(export_payload(payload), separators=(",", ":"), default=str) + # Keep serialized JSON from closing the surrounding ' + + +def write_interactive_viewer( + out_html: Path, + *, + json_basename: str, + title: str, + subtitle: str = "Interactive sweep explorer (loads sibling JSON sidecar)", + tier_order: tuple[str, ...] = ("throughput", "record"), + embed_payload: Optional[Mapping[str, Any]] = None, +) -> Path: + """Write a static viewer HTML that loads report data embedded or via sibling JSON.""" + out_html = Path(out_html) + out_html.parent.mkdir(parents=True, exist_ok=True) + template = _TEMPLATE_PATH.read_text(encoding="utf-8") + if not template.strip(): + raise FileNotFoundError(f"Viewer template is empty or missing: {_TEMPLATE_PATH}") + tier_js = "[" + ",".join(json.dumps(t) for t in tier_order) + "]" + embedded = _embedded_json_script(embed_payload) if embed_payload is not None else "" + doc = ( + template.replace("__TITLE__", html.escape(title)) + .replace("__SUBTITLE__", html.escape(subtitle)) + .replace("__JSON_PATH__", json.dumps(json_basename)) + .replace("__TIER_ORDER__", tier_js) + .replace("__EMBEDDED_JSON__", embedded) + ) + out_html.write_text(doc, encoding="utf-8") + return out_html + + +def viewer_basename_for(report_basename: str) -> str: + return f"{report_basename}_viewer.html" diff --git a/cvs/lib/report_plugins.py b/cvs/lib/report_plugins.py index 95ec4ce18..b24cd50b1 100644 --- a/cvs/lib/report_plugins.py +++ b/cvs/lib/report_plugins.py @@ -6,6 +6,7 @@ ''' import datetime +import html import re import shutil import sys @@ -117,12 +118,12 @@ def write_test_log(self, report, test_name=None): log_content = [] for section_name, section_content in report.sections: - log_content.append(f"

{section_name}

{section_content}
") + log_content.append(f"

{html.escape(section_name)}

{html.escape(section_content)}
") if log_content: # Persist a standalone html log page per test. log_path.write_text( - f"

{report.nodeid}

{''.join(log_content)}", + f"

{html.escape(report.nodeid)}

{''.join(log_content)}", encoding="utf-8", ) log.info("Wrote external test log: %s", log_path) @@ -471,6 +472,44 @@ def generate_reports_section(self): return html + def generate_suite_reports(self, session): + """Write registered suite report HTML/JSON into the pytest bundle before zip.""" + if not self.is_enabled: + return + + from cvs.lib.report.inference import publish_inference_suite_report + from cvs.lib.report.registry import get_session_results, get_suite_report_config + from cvs.lib.report.types import InferenceReportConfig + + report_config = get_suite_report_config(session.config) + if report_config is None: + suite_name = getattr(session.config, "_suite_name", "unknown") + log.info( + "Skipping suite report generation: no preset registered for suite '%s'", + suite_name, + ) + return + + store = get_session_results() + + if not isinstance(report_config, InferenceReportConfig): + log.warning("Unknown suite report config type: %s", type(report_config).__name__) + return + + inf_res_dict = store.get("inf_res_dict") + if not inf_res_dict: + log.info("Skipping suite report generation: no results in session store") + return + + publish_inference_suite_report( + report_config, + variant_config=store.get("variant_config"), + inf_res_dict=inf_res_dict, + lifecycle_report=store.get("lifecycle_report") or {}, + report_manager=self, + pytest_config=session.config, + ) + @staticmethod def inject_style_overrides(prefix): """Inject CSS to hide show/hide details UI elements.""" diff --git a/cvs/lib/sglang_disagg_lib.py b/cvs/lib/sglang_disagg_lib.py deleted file mode 100644 index 5748a787d..000000000 --- a/cvs/lib/sglang_disagg_lib.py +++ /dev/null @@ -1,1318 +0,0 @@ -''' -Copyright 2026 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import os -import re -import time - -from cvs.lib import globals -from cvs.lib.utils_lib import * -from cvs.lib.verify_lib import * - - -log = globals.log - -inference_err_dict = { - 'NCCL ERROR': 'NCCL ERROR|NCCL timeout|local work queue catastrophic error', - 'GPU HW ERROR': 'HW Exception by GPU|GPU Hang|Uncorrectable error|GPU Reset', - 'AssertionError': 'AssertionError|ValueError:|During handling of the above exception|triggered the following exception|RuntimeError|Python error: Aborted', - 'rocm Err': 'FAILED_PRECONDITION: No visible GPU devices|failed call to hipInit: HIP_ERROR_NoDevice|librocm reported version is: NOT_FOUND', - 'python err': 'ModuleNotFoundError: No module named|Fatal Python error:', - 'resource': 'RESOURCE_EXHAUSTED: Out of memory|failed: RESOURCE_EXHAUSTED|urllib.error.URLError|ConnectionRefusedError,HSA_STATUS_ERROR_OUT_OF_RESOURCES', - 'app_err': 'Service Unavailable|No decode workers available|No prefill workers available|Please check if decode servers are configured and healthy|Please check if prefill servers are configured and healthy|Cannot access gated repo|You must have access to it and be authenticated', -} - -err_counters_pattern = 'err|retransmit|drop|discard|naks|invalid|oflow|out_of_buffer|reset|fail' - - -def textwrap_for_yml(msg_string): - return '\n'.join([m.lstrip() for m in msg_string.split('\n')]) - - -class SglangDisaggPD: - def __init__( - self, - model_name, - inference_config_dict, - benchmark_params_dict, - hf_token, - p_phdl=None, - d_phdl=None, - r_phdl=None, - b_phdl=None, - gpu_type='mi300', - user_name=None, - priv_key_file=None, - ): - """ - Initialize a Disaggregated Prefill/Decode (PD) inference controller - for SGLang. - - This class encapsulates: - - Cluster topology (prefill, decode, proxy, benchmark nodes) - - SSH-based remote execution (via Pssh handlers) - - Inference configuration (networking, containers, env vars) - - Benchmark configuration (load, concurrency, prompt sizes) - - Args: - model_name (str): HuggingFace or local model identifier - inference_config_dict (dict): Cluster and runtime configuration - benchmark_params_dict (dict): Benchmark workload parameters - hf_token (str): HuggingFace access token - p_phdl, d_phdl, r_phdl, b_phdl: Optional pre-created SSH handlers - gpu_type (str): GPU type (e.g., mi300, mi325) - user_name (str): SSH username for remote nodes - priv_key_file (str): SSH private key file - """ - - # ------------------------------------------------------------------ - # Basic identity and authentication parameters - # ------------------------------------------------------------------ - self.user_name = user_name - self.priv_key_file = priv_key_file - self.model_name = model_name - self.hf_token = hf_token - self.gpu_type = gpu_type - - # ------------------------------------------------------------------ - # Store inference and benchmark configuration dictionaries - # These are typically loaded from a JSON/YAML configuration file - # ------------------------------------------------------------------ - self.inf_dict = inference_config_dict - self.bp_dict = benchmark_params_dict - - self.model_name = model_name - self.hf_token = hf_token - self.gpu_type = gpu_type - - # ------------------------------------------------------------------ - # Extract cluster topology for disaggregated inference - # - # Prefill nodes : Handle prompt ingestion + KV cache creation - # Decode nodes : Handle token generation - # Proxy node : Routes requests between prefill/decode - # Benchmark node : Generates inference load - # ------------------------------------------------------------------ - self.prefill_node_list = self.inf_dict['prefill_node_list'] - self.decode_node_list = self.inf_dict['decode_node_list'] - self.prefill_nnodes = len(self.prefill_node_list) - self.decode_nnodes = len(self.decode_node_list) - - self.proxy_node = list(self.inf_dict['proxy_router_node']) - self.benchmark_serv_node = list(self.inf_dict['benchmark_serv_node']) - - # ------------------------------------------------------------------ - # SSH handlers for each node group - # - # p_phdl : Prefill nodes - # d_phdl : Decode nodes - # r_phdl : Proxy/router node - # b_phdl : Benchmark client node - # ------------------------------------------------------------------ - self.p_phdl = p_phdl - self.d_phdl = d_phdl - self.r_phdl = r_phdl - self.b_phdl = b_phdl - - if self.p_phdl is None: - self.p_phdl = Pssh(log, self.prefill_node_list, user=self.user_name, pkey=self.priv_key_file) - - if self.d_phdl is None: - self.d_phdl = Pssh(log, self.decode_node_list, user=self.user_name, pkey=self.priv_key_file) - - if self.r_phdl is None: - self.r_phdl = Pssh(log, self.proxy_node, user=self.user_name, pkey=self.priv_key_file) - - if self.b_phdl is None: - self.b_phdl = Pssh(log, self.benchmark_serv_node, user=self.user_name, pkey=self.priv_key_file) - - self.job_cmd = '' - self.job_cmd_list = [] - self.inference_results_dict = {} - log.info("%s", self.gpu_type) - - # ------------------------------------------------------------------ - # Extract commonly used inference parameters for convenience - # ------------------------------------------------------------------ - # Needed only in the case of distributed inference - placeholder for future - # Intialize cluster stats dicts .. - self.rdma_stats_dict_before = {} - self.ethtool_stats_dict_before = {} - self.rdma_stats_dict_after = {} - self.inference_start_time = p_phdl.exec('date +"%a %b %e %H:%M"') - self.inference_end_time = None - - # ------------------------------------------------------------------ - # Set default benchmark parameters if not provided - # These control request generation and performance measurement - # ------------------------------------------------------------------ - self.home_dir = os.path.expanduser("~") - self.inf_dict.setdefault('container_image', 'lmsysorg/sglang:dev') - self.inf_dict.setdefault('container_name', 'sglang_container') - self.inf_dict.setdefault('nic_type', 'ainic') - self.inf_dict.setdefault('nccl_ib_hca_list', 'rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7') - self.inf_dict.setdefault('nccl_ib_hca', 'rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7') - self.inf_dict.setdefault('nccl_socket_ifname', 'eno0') - self.inf_dict.setdefault('gloo_socket_ifname', 'eno0') - self.inf_dict.setdefault('nccl_ib_gid_index', '1') - self.inf_dict.setdefault('nccl_debug', 'ERROR') - self.inf_dict.setdefault('data_cache_dir', f'{self.home_dir}/cache') - self.inf_dict.setdefault('log_dir', f'{self.home_dir}/LOG_DIR') - self.inf_dict.setdefault('max_concurrent_requests', '-1') - self.inf_dict.setdefault('queue_size', '100') - self.inf_dict.setdefault('queue_timeout_secs', '60') - self.inf_dict.setdefault('max_retries', '5') - - log.info('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%') - log.info(f'inference_dict = {self.inf_dict}') - log.info('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%') - self.container_image = self.inf_dict['container_image'] - self.container_name = self.inf_dict['container_name'] - - self.nic_type = self.inf_dict['nic_type'] - self.nccl_ib_hca_list = self.inf_dict['nccl_ib_hca_list'] - self.nccl_ib_hca = self.inf_dict['nccl_ib_hca'] - self.nccl_socket_ifname = self.inf_dict['nccl_socket_ifname'] - self.gloo_socket_ifname = self.inf_dict['gloo_socket_ifname'] - self.nccl_ib_gid_index = self.inf_dict['nccl_ib_gid_index'] - self.nccl_debug = self.inf_dict['nccl_debug'] - self.data_cache_dir = self.inf_dict['data_cache_dir'] - self.log_dir = self.inf_dict['log_dir'] - - # set defaults for benchmark param dict if not passed via JSON file - self.bp_dict.setdefault('backend', 'sglang') - self.bp_dict.setdefault('dataset_name', 'sharegpt') - self.bp_dict.setdefault('max_concurrency', '64') - self.bp_dict.setdefault('model', 'openai/gpt-oss-120b') - self.bp_dict.setdefault('num_prompts', '1000') - self.bp_dict.setdefault('input_sequence_length', '8192') - self.bp_dict.setdefault('burstiness', '1.0') - self.bp_dict.setdefault('seed', '0') - self.bp_dict.setdefault('request_rate', 'inf') - self.bp_dict.setdefault('max_model_length', '9216') - self.bp_dict.setdefault('random_range_ration', '1.0') - self.bp_dict.setdefault('random_prefix_len', '0') - self.bp_dict.setdefault('tensor_parallelism', '8') - self.bp_dict.setdefault('port_no', '8000') - self.bp_dict.setdefault('tokenizer_mode', 'auto') - self.bp_dict.setdefault('percentile_metrics', 'ttft,tpot,itl,e2el') - self.bp_dict.setdefault('metric_percentiles', '99') - self.bp_dict.setdefault('inference_poll_iterations', '16') - - self.inference_poll_iterations = self.bp_dict['inference_poll_iterations'] - - log.info('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%') - log.info(f'benchmark_params_dict = {self.bp_dict}') - log.info('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%') - - def install_container_packages( - self, - ): - """ - Install required system networking utilities inside inference containers. - - Purpose: - -------- - This method prepares the container environment for distributed inference - by installing basic networking and diagnostic tools that are commonly - needed for: - - Connectivity validation between nodes - - Debugging network paths (ping, ip route, ifconfig) - - Verifying NIC and routing configuration - - Troubleshooting NCCL/Gloo/RDMA-related issues - - These tools are installed inside the running container on: - - Prefill nodes - - Decode nodes - - Proxy/router nodes - """ - - log.info('Run pre inference tasks') - # Install ip tools - cmd = f'docker exec {self.container_name} /bin/bash -c " \ - sudo apt -y update; \ - sudo apt install -y iputils-ping; \ - sudo apt install -y iproute2; \ - sudo apt install -y net-tools" ' - self.p_phdl.exec(cmd) - self.d_phdl.exec(cmd) - self.r_phdl.exec(cmd) - - def exec_nic_setup_scripts( - self, - ): - """ - Execute NIC-related setup steps inside the inference container. - - Behavior: - - Only runs for distributed inference. - - If NIC type appears to be Broadcom/Thor, applies a temporary workaround: - * Copies the bnxt RDMA library from the host-named file to the container?s expected path. - * Verifies that ibv_devinfo shows a bnxt_ HCA (to confirm RDMA is wired correctly). - - Forces NCCL GID index to 3 for Broadcom/Thor (common requirement). - - Assumptions: - - self.s_phdl.exec runs a shell command and returns a dict: {node: stdout}. - - sudo is non-interactive within the container. - - The bnxt library file paths exist in the container base image. - """ - - # This is a temporary hack needed for broadcom nics to work within containers .. - if re.search('broadcom|thor', self.nic_type, re.I): - # override the gid_index to 3 for broadcom - self.nccl_ib_gid_index = 3 - cmd = f'docker exec {self.container_name} /bin/bash -c "sudo \ - cp /usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so.host \ - /usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so; \ - sleep 2; ibv_devinfo; sleep 2;" ' - pout_dict = self.p_phdl.exec(cmd) - dout_dict = self.d_phdl.exec(cmd) - for node in pout_dict.keys(): - if not re.search('hca_id:\s+(bnxt_|rocep)', pout_dict[node], re.I): - log.info("%s", pout_dict[node]) - fail_test(f'Broadcom libbnxt rdma driver is not properly copied on node {node}') - for node in dout_dict.keys(): - if not re.search('hca_id:\s+(bnxt_|rocep)', dout_dict[node], re.I): - log.info("%s", dout_dict[node]) - fail_test(f'Broadcom libbnxt rdma driver is not properly copied on node {node}') - - def check_ibv_devices( - self, - ): - """ - Verify that InfiniBand / RDMA devices are visible inside the container - on all relevant nodes. - - Purpose: - -------- - This method ensures that RDMA-capable devices (e.g., InfiniBand HCAs) - are correctly exposed inside the container environment. This is a - critical prerequisite for: - - NCCL / RCCL over RDMA - - High-performance distributed inference - - Low-latency, high-bandwidth GPU communication - - The check is performed on: - - Prefill nodes - - Decode nodes - - Proxy and benchmark nodes typically do not require RDMA access. - """ - for hdl in [self.p_phdl, self.d_phdl]: - cmd = f'''docker exec {self.container_name} /bin/bash -c "ibv_devinfo" ''' - out_dict = hdl.exec(cmd) - for node in out_dict.keys(): - if re.search('No IB devices found', out_dict[node], re.I): - fail_test(f'IB devices not seen inside the container for node {node}') - - def setup_prefill_container_env( - self, - ): - # Env setup for Prefill Nodes .. - p_cmd = f'''docker exec {self.container_name} /bin/bash -c "echo ' - - export LD_LIBRARY_PATH=/opt/rocm/lib:$LD_LIBRARY_PATH - export NCCL_DEBUG={self.inf_dict['nccl_debug']} - export NCCL_IB_HCA={self.inf_dict['nccl_ib_hca']} - export NCCL_IB_GID_INDEX={self.inf_dict['nccl_ib_gid_index']} - export HSA_FORCE_FINE_GRAIN_PCIE=1 - export NCCL_SOCKET_IFNAME={self.inf_dict['nccl_socket_ifname']} - export GLOO_SOCKET_IFNAME={self.inf_dict['gloo_socket_ifname']} - export GLOO_TCP_IFNAME={self.inf_dict['gloo_socket_ifname']} - export HSA_FORCE_FINE_GRAIN_PCIE=1 - - export MASTER_PREFILL_ADDR={self.inf_dict['prefill_coordinator_addr']} - export MASTER_PREFILL_PORT={self.inf_dict['prefill_coordinator_port']} - - export MODEL={self.bp_dict['model']} - export TP={self.bp_dict['tensor_parallelism']} - export HF_TOKEN={self.hf_token} - ' > /tmp/prefill_env_script.sh" - ''' - time.sleep(3) - formatted_p_cmd = textwrap_for_yml(p_cmd) - self.p_phdl.exec(formatted_p_cmd) - cmd = f'''docker exec {self.container_name} /bin/bash -c " \ - chmod 755 /tmp/prefill_env_script.sh; /tmp/prefill_env_script.sh" ''' - self.p_phdl.exec(cmd) - - def setup_decode_container_env( - self, - ): - # Env setup for Decode Nodes .. - d_cmd = f'''docker exec {self.container_name} /bin/bash -c "echo ' - - export LD_LIBRARY_PATH=/opt/rocm/lib:$LD_LIBRARY_PATH - export NCCL_DEBUG={self.inf_dict['nccl_debug']} - export NCCL_IB_HCA={self.inf_dict['nccl_ib_hca']} - export NCCL_IB_GID_INDEX={self.inf_dict['nccl_ib_gid_index']} - export HSA_FORCE_FINE_GRAIN_PCIE=1 - export NCCL_SOCKET_IFNAME={self.inf_dict['nccl_socket_ifname']} - export GLOO_SOCKET_IFNAME={self.inf_dict['gloo_socket_ifname']} - export GLOO_TCP_IFNAME={self.inf_dict['gloo_socket_ifname']} - export HSA_FORCE_FINE_GRAIN_PCIE=1 - - export MASTER_DECODE_ADDR={self.inf_dict['decode_coordinator_addr']} - export MASTER_DECODE_PORT={self.inf_dict['decode_coordinator_port']} - - export MODEL={self.bp_dict['model']} - export TP={self.bp_dict['tensor_parallelism']} - export HF_TOKEN={self.hf_token} - ' > /tmp/decode_env_script.sh" - ''' - time.sleep(3) - formatted_d_cmd = textwrap_for_yml(d_cmd) - self.d_phdl.exec(formatted_d_cmd) - cmd = f'''docker exec {self.container_name} /bin/bash -c " \ - chmod 755 /tmp/decode_env_script.sh; /tmp/decode_env_script.sh" ''' - self.d_phdl.exec(cmd) - - def setup_proxy_router_container_env( - self, - ): - # Env setup for Proxy Router Node .. - r_cmd = f'''docker exec {self.container_name} /bin/bash -c "echo ' - - export LD_LIBRARY_PATH=/opt/rocm/lib:$LD_LIBRARY_PATH - export NCCL_DEBUG={self.inf_dict['nccl_debug']} - export NCCL_IB_HCA={self.inf_dict['nccl_ib_hca']} - export NCCL_IB_GID_INDEX={self.inf_dict['nccl_ib_gid_index']} - export HSA_FORCE_FINE_GRAIN_PCIE=1 - export NCCL_SOCKET_IFNAME={self.inf_dict['nccl_socket_ifname']} - export GLOO_SOCKET_IFNAME={self.inf_dict['gloo_socket_ifname']} - export GLOO_TCP_IFNAME={self.inf_dict['gloo_socket_ifname']} - export HSA_FORCE_FINE_GRAIN_PCIE=1 - - export HF_TOKEN={self.hf_token} - ' > /tmp/router_env_script.sh" - ''' - time.sleep(3) - formatted_r_cmd = textwrap_for_yml(r_cmd) - self.r_phdl.exec(formatted_r_cmd) - cmd = f'''docker exec {self.container_name} /bin/bash -c " \ - chmod 755 /tmp/router_env_script.sh; /tmp/router_env_script.sh" ''' - self.r_phdl.exec(cmd) - - def setup_benchmark_serv_container_env( - self, - ): - # Env setup for Benchserv node .. - b_cmd = f'''docker exec {self.container_name} /bin/bash -c "echo ' - - export LD_LIBRARY_PATH=/opt/rocm/lib:$LD_LIBRARY_PATH - export NCCL_DEBUG={self.inf_dict['nccl_debug']} - export NCCL_IB_HCA={self.inf_dict['nccl_ib_hca']} - export NCCL_IB_GID_INDEX={self.inf_dict['nccl_ib_gid_index']} - export HSA_FORCE_FINE_GRAIN_PCIE=1 - export NCCL_SOCKET_IFNAME={self.inf_dict['nccl_socket_ifname']} - export GLOO_SOCKET_IFNAME={self.inf_dict['gloo_socket_ifname']} - export GLOO_TCP_IFNAME={self.inf_dict['gloo_socket_ifname']} - export HSA_FORCE_FINE_GRAIN_PCIE=1 - export HF_TOKEN={self.hf_token} - ' > /tmp/benchmark_env_script.sh" - ''' - time.sleep(3) - formatted_b_cmd = textwrap_for_yml(b_cmd) - self.b_phdl.exec(formatted_b_cmd) - cmd = f'''docker exec {self.container_name} /bin/bash -c " \ - chmod 755 /tmp/benchmark_env_script.sh; /tmp/benchmark_env_script.sh" ''' - self.b_phdl.exec(cmd) - time.sleep(5) - - def run_test_rmsnorm(self, max_jobs=192): - """ - Run RMSNorm 2D operator tests inside the SGLang container across - relevant nodes and validate correctness. - - Purpose: - -------- - This method executes the AITER RMSNorm 2D operator test, which validates: - - Correctness of RMSNorm kernel implementation - - Stability under high parallel job execution - - GPU kernel behavior under concurrent workloads - - The test is executed on: - - Prefill nodes - - Decode nodes - - Proxy/router nodes - - Args: - max_jobs (int): Maximum number of concurrent jobs to launch within - the RMSNorm test to stress the kernel. - """ - log.info('#================ * * * =========================#') - log.info('Run rmsnorm2d') - log.info('#================ * * * =========================#') - # ------------------------------------------------------------------ - # Construct command to run RMSNorm test inside the container - # - # Details: - # - MAX_JOBS controls parallelism inside the test - # - Output is redirected to a per-container log file - # - Command is executed in the background to allow parallel execution - # ------------------------------------------------------------------ - cmd = f'''docker exec {self.container_name} /bin/bash -c "MAX_JOBS={max_jobs} \ - python /sgl-workspace/aiter/op_tests/test_rmsnorm2d.py > /tmp/rsmnorm_test.log 2>&1 &" ''' - for hdl in [self.p_phdl, self.d_phdl, self.r_phdl]: - out_dict = hdl.exec(cmd) - log.info('Wait 180 secs for tests to complete') - time.sleep(180) - for hdl in [self.p_phdl, self.d_phdl, self.r_phdl]: - cmd = f'''docker exec {self.container_name} /bin/bash -c "cat /tmp/rsmnorm_test.log" ''' - out_dict = hdl.exec(cmd) - for node in out_dict.keys(): - if re.search('fail', out_dict[node], re.I): - log.warning(f'Some failures observed in test rmsnorm on node {node}') - fail_test(f'Some failures observed in test rmsnorm on node {node}') - - # supported --dtype {auto,half,float16,bfloat16,float,float32} - # supported --kv-cache-dtype {auto,fp8_e5m2,fp8_e4m3,bf16,bfloat16,fp4_e2m1} - def launch_prefill_servers(self, dtype='auto', kv_cache_dtype='auto'): - """ - Generate and stage Prefill server launch scripts on all Prefill nodes - for SGLang disaggregated inference. - - Purpose: - -------- - This method prepares the launch script for SGLang Prefill servers. - In disaggregated PD (Prefill / Decode) mode: - - Prefill servers are responsible for processing input prompts - - They generate KV cache entries - - KV cache is later consumed by Decode servers - - This method: - - Creates one launch script per Prefill node - - Sets distributed environment variables (NNODES, NODE_RANK) - - Configures SGLang for Prefill-only execution - - Does NOT start the servers yet; it stages the script for later execution - - Args: - dtype (str): Model compute datatype (e.g., fp16, bf16, auto) - kv_cache_dtype (str): KV cache datatype (e.g., fp16, bf16, auto) - """ - log.info('#================ * * * =========================#') - log.info('Create Prefill launch script on Prefill nodes') - log.info('#================ * * * =========================#') - - cmd_list = [] - prefill_node_list = self.inf_dict['prefill_node_list'] - log.info('%%%% self.prefill_nnodes {}'.format(self.prefill_nnodes)) - dist_init_addr = f"{self.inf_dict['prefill_coordinator_addr']}:{self.inf_dict['prefill_coordinator_port']}" - for i in range(0, int(self.prefill_nnodes)): - cmd = f'''docker exec {self.container_name} /bin/bash -c "echo ' - export NNODES={self.prefill_nnodes} - export NODE_RANK={i} - export SGLANG_USE_AITER=1 - python3 -m sglang.launch_server --model {self.bp_dict['model']} \ - --disaggregation-mode prefill \ - --disaggregation-ib-device {self.inf_dict['nccl_ib_hca']} \ - --host {prefill_node_list[i]} \ - --port {self.inf_dict['prefill_serv_port']} \ - --dtype {dtype} \ - --kv-cache-dtype {kv_cache_dtype} \ - --trust-remote-code \ - --tp {self.bp_dict['tensor_parallelism']} \ - --nnodes {self.prefill_nnodes} \ - --node-rank {i} \ - --dist-init-addr {dist_init_addr} \ - --disable-radix-cache --disable-cuda-graph \ - --mem-fraction-static {self.bp_dict['memory_fraction']} \ - --attention-backend aiter \ - --log-level {self.inf_dict['log_level']}' > /tmp/prefill_launch_script.sh" ''' - formatted_cmd = textwrap_for_yml(cmd) - cmd_list.append(formatted_cmd) - log.info('%%%%%%%%%%%%%%%%%%%') - log.info("%s", cmd_list) - log.info('%%%%%%%%%%%%%%%%%%%') - self.p_phdl.exec_cmd_list(cmd_list) - log.info('#================ * * * =========================#') - log.info('Launching Prefill servers on Prefill nodes') - log.info('#================ * * * =========================#') - cmd_list = [] - for i in range(0, int(self.prefill_nnodes)): - cmd = f'''docker exec {self.container_name} /bin/bash -c " \ - chmod 755 /tmp/prefill_launch_script.sh; \ - mkdir -p {self.log_dir}/prefill_node{i}; \ - source /tmp/prefill_env_script.sh && \ - nohup /tmp/prefill_launch_script.sh > \ - {self.log_dir}/prefill_node{i}/prefill_server.log 2>&1 &" ''' - formatted_cmd = textwrap_for_yml(cmd) - cmd_list.append(formatted_cmd) - self.p_phdl.exec_cmd_list(cmd_list) - time.sleep(5) - - def launch_decode_servers(self, dtype='auto', kv_cache_dtype='auto'): - """ - Generate and deploy Decode server launch scripts on all Decode nodes - for SGLang disaggregated inference. - - Purpose: - -------- - In disaggregated PD (Prefill / Decode) inference: - - Decode servers are responsible for token generation - - They consume KV cache generated by Prefill servers - - They perform the latency- and throughput-critical decode loop - - This method: - - Creates one Decode launch script per Decode node - - Sets distributed environment variables (NNODES, NODE_RANK) - - Configures SGLang for Decode-only execution - - Deploys the scripts to Decode nodes for later execution - - Args: - dtype (str): Model compute datatype (e.g., fp16, bf16, auto) - kv_cache_dtype (str): KV cache datatype (e.g., fp16, bf16, auto) - """ - log.info('#================ * * * =========================#') - log.info('Create Decode launch script on Decode nodes') - log.info('#================ * * * =========================#') - cmd_list = [] - decode_node_list = self.inf_dict['decode_node_list'] - log.info('%%%% self.decode_nnodes {}'.format(self.decode_nnodes)) - dist_init_addr = f"{self.inf_dict['decode_coordinator_addr']}:{self.inf_dict['decode_coordinator_port']}" - for i in range(0, int(self.decode_nnodes)): - cmd = f'''docker exec {self.container_name} /bin/bash -c "echo ' - export NNODES={self.decode_nnodes} - export NODE_RANK={i} - export SGLANG_USE_AITER=1 - python3 -m sglang.launch_server --model {self.bp_dict['model']} \ - --disaggregation-mode decode \ - --disaggregation-ib-device {self.inf_dict['nccl_ib_hca']} \ - --host {decode_node_list[i]} \ - --port {self.inf_dict['decode_serv_port']} \ - --trust-remote-code \ - --dtype {dtype} \ - --kv-cache-dtype {kv_cache_dtype} \ - --tp {self.bp_dict['tensor_parallelism']} \ - --nnodes {self.decode_nnodes} \ - --node-rank {i} \ - --dist-init-addr {dist_init_addr} \ - --disable-radix-cache --disable-cuda-graph \ - --mem-fraction-static {self.bp_dict['memory_fraction']} \ - --attention-backend aiter \ - --log-level {self.inf_dict['log_level']}' > /tmp/decode_launch_script.sh" ''' - formatted_cmd = textwrap_for_yml(cmd) - cmd_list.append(formatted_cmd) - log.info('%%%%%%%%%%%%%%%%%%%') - log.info("%s", cmd_list) - log.info('%%%%%%%%%%%%%%%%%%%') - self.d_phdl.exec_cmd_list(cmd_list) - log.info('#================ * * * =========================#') - log.info('Launching Decode servers on Decode nodes') - log.info('#================ * * * =========================#') - cmd_list = [] - for i in range(0, int(self.decode_nnodes)): - cmd = f'''docker exec {self.container_name} /bin/bash -c " \ - chmod 755 /tmp/decode_launch_script.sh; \ - mkdir -p {self.log_dir}/decode_node{i}; \ - source /tmp/decode_env_script.sh && \ - nohup bash /tmp/decode_launch_script.sh > \ - {self.log_dir}/decode_node{i}/decode_server.log 2>&1 &" ''' - formatted_cmd = textwrap_for_yml(cmd) - cmd_list.append(formatted_cmd) - self.d_phdl.exec_cmd_list(cmd_list) - - def poll_and_check_server_ready( - self, - ): - """ - Wait for Prefill and Decode servers to initialize and verify that they - are fully ready to accept inference requests. - - Purpose: - -------- - After launching Prefill and Decode server scripts, the servers require - time to: - - Initialize Python runtime - - Load model weights - - Allocate GPU memory - - Initialize RDMA / NCCL / Gloo communication - - Bind to network ports - - This method enforces a startup delay and then actively polls each server - to confirm readiness before inference traffic is sent. - """ - log.info('Waiting 120 secs after launching decode script') - time.sleep(120) - # for node_no in range(0, self.prefill_nnodes): - # self.poll_for_server_ready(node_no, 'prefill') - # for node_no in range(0, self.decode_nnodes): - # self.poll_for_server_ready(node_no, 'decode') - self.poll_for_server_ready(0, 'prefill') - self.poll_for_server_ready(0, 'decode') - - def launch_proxy_router( - self, - ): - """ - Generate and launch the SGLang Proxy Router for disaggregated - Prefill/Decode (PD) inference. - - Purpose: - -------- - The Proxy Router is the control-plane and data-plane entry point for - inference traffic in a disaggregated PD deployment. - - Responsibilities: - - Accept incoming inference requests - - Route prefill requests to Prefill servers - - Route decode requests to Decode servers - - Coordinate Prefill ? Decode handoff - - This method: - - Builds routing configuration dynamically based on cluster topology - - Creates a launch script on the Proxy Router node - - Launches the router as a background service - """ - - # ------------------------------------------------------------------ - # Build Prefill endpoint arguments for the router - # - # Each Prefill server is specified as: - # --prefill http://: - # ------------------------------------------------------------------ - - prefill_str = ( - f"--prefill http://{self.inf_dict['prefill_coordinator_addr']}:{self.inf_dict['prefill_serv_port']} " - ) - # ------------------------------------------------------------------ - # Build Decode endpoint arguments for the router - # - # Each Decode server is specified as: - # --decode http://: - # ------------------------------------------------------------------ - - decode_str = f"--decode http://{self.inf_dict['decode_coordinator_addr']}:{self.inf_dict['decode_serv_port']} " - log.info('#================ * * * =========================#') - log.info('Create Proxy Router launch script on Proxy Router nodes') - log.info('#================ * * * =========================#') - - # ------------------------------------------------------------------ - # Create the Proxy Router launch script - # - # Key flags: - # --pd-disaggregation : Enable Prefill/Decode disaggregation - # --prefill / --decode: Upstream Prefill and Decode endpoints - # --host 0.0.0.0 : Listen on all interfaces - # --port : External router port - # --log-dir : Directory for router logs - # - # NOTE: - # The script is written to disk but not executed here. - # ------------------------------------------------------------------ - cmd = f'''docker exec {self.container_name} /bin/bash -c "echo ' - python3 -m sglang_router.launch_router \ - --pd-disaggregation \ - {prefill_str} \ - {decode_str} \ - --host 0.0.0.0 \ - --port {self.inf_dict['proxy_router_port']} \ - --log-dir {self.inf_dict['log_dir']} \ - ' > /tmp/proxy_router_launch_script.sh" - ''' - formatted_cmd = textwrap_for_yml(cmd) - self.r_phdl.exec(formatted_cmd) - log.info('#================ * * * =========================#') - log.info('Launch Proxy Router script on Proxy Router nodes') - log.info('#================ * * * =========================#') - cmd = f'''docker exec {self.container_name} /bin/bash -c " \ - chmod 755 /tmp/proxy_router_launch_script.sh; \ - mkdir -p {self.log_dir}/proxy_router_node; \ - source /tmp/router_env_script.sh && \ - nohup bash /tmp/proxy_router_launch_script.sh > \ - {self.log_dir}/proxy_router_node/proxy_router.log 2>&1 &" ''' - formatted_cmd = textwrap_for_yml(cmd) - self.r_phdl.exec(formatted_cmd) - log.info('Waiting 120 secs after launching proxy router script') - time.sleep(120) - - def run_gsm8k_benchmark_test(self, d_type='auto'): - """ - Run the GSM8K inference benchmark against the SGLang disaggregated - Prefill/Decode deployment and validate throughput. - - Purpose: - -------- - This method executes a real-world inference workload (GSM8K question - answering) to: - - Validate end-to-end correctness of the inference pipeline - - Measure sustained output token throughput - - Ensure performance meets expected SLA thresholds - - The benchmark traffic is sent to the Proxy Router, which: - - Routes requests to Prefill servers - - Coordinates Decode servers for token generation - """ - log.info('#================ * * * =========================#') - log.info('Create Benchmark script') - log.info('#================ * * * =========================#') - - i_dict = self.bp_dict['inference_tests']['gsm8k'] - # ------------------------------------------------------------------ - # Construct command to run GSM8K benchmark inside the container - # - # Key steps: - # - Create a directory to store benchmark logs - # - Navigate to the GSM8K benchmark directory - # - Source environment variables required for benchmark execution - # - Launch the benchmark using nohup to allow async execution - # - # Benchmark parameters: - # --num-questions : Total GSM8K questions to run - # --parallel : Maximum concurrent inference requests - # --host / --port : Proxy Router endpoint for inference - # ------------------------------------------------------------------ - cmd = f'''docker exec {self.container_name} /bin/bash -c " - mkdir -p {self.log_dir}/benchmark_node; \ - cd /sgl-workspace/sglang/benchmark/gsm8k; \ - source /tmp/benchmark_env_script.sh && \ - nohup python3 ./bench_sglang.py \ - --num-questions {i_dict['num_questions']} \ - --parallel {i_dict['max_concurrency']} \ - --host http://0.0.0.0 --port {self.inf_dict['proxy_router_serv_port']}" ''' - formatted_cmd = textwrap_for_yml(cmd) - out_dict = self.b_phdl.exec(formatted_cmd, timeout=800) - time.sleep(5) - for node in out_dict.keys(): - if not re.search('Output throughput', out_dict[node], re.I): - fail_test(f'Benchmark test did not complete properly on node {node}, no throughput pattern seen') - else: - match = re.search('Output throughput:\s+([0-9\.]+)\s+token', out_dict[node], re.I) - actual_tps = match.group(1) - if float(actual_tps) < float(i_dict['expected_results'][d_type]['tokens_per_sec']): - fail_test( - f"Test FAILED due to low performance, \ - expected tokens per sec = {i_dict['expected_results'][d_type]['tokens_per_sec']}, \ - actual tokens per sec = {actual_tps}" - ) - - def benchserv_test_random(self, d_type='auto'): - """ - Run SGLang serving benchmark using a synthetic random dataset and - validate inference performance and correctness. - - Purpose: - -------- - This benchmark exercises the inference serving stack using randomly - generated input/output sequences to: - - Stress-test request scheduling and batching - - Evaluate sustained throughput under synthetic load - - Validate end-to-end serving stability independent of real datasets - - The benchmark targets the Proxy Router endpoint, ensuring that - Prefill, Decode, and routing logic work together correctly. - - Args: - d_type (str): Data type identifier used to select expected - performance thresholds (e.g., fp16, bf16, auto). - """ - log.info('#================ * * * =========================#') - log.info('Benchmark Random Dataset') - log.info('#================ * * * =========================#') - i_dict = self.bp_dict['inference_tests']['bench_serv_random'] - # ------------------------------------------------------------------ - # Construct command to run sglang.bench_serving with random dataset - # - # Key parameters: - # --dataset-name random : Use synthetic random prompts - # --num-prompts : Total number of inference requests - # --random-input : Input token length per request - # --random-output : Output token length per request - # --random-range-ratio : Variability in input/output lengths - # --host / --port : Proxy Router endpoint - # - # Output is redirected to a log file for later inspection. - # ------------------------------------------------------------------ - cmd = f'''docker exec {self.container_name} /bin/bash -c " - mkdir -p {self.log_dir}/benchmark_node; \ - source /tmp/benchmark_env_script.sh && \ - python3 -m sglang.bench_serving --backend {i_dict['backend']} \ - --dataset-name random \ - --num-prompts {i_dict['num_prompts']} \ - --random-input {i_dict['input_length']} \ - --random-output {i_dict['output_length']} \ - --random-range-ratio {i_dict['random_range_ratio']} \ - --host 0.0.0.0 --port {self.inf_dict['proxy_router_serv_port']} \ - > {self.log_dir}/benchmark_node/benchmark_results.log 2>&1" ''' - formatted_cmd = textwrap_for_yml(cmd) - self.b_phdl.exec(formatted_cmd, timeout=500) - time.sleep(5) - self.poll_for_inference_completion(iterations=10, waittime_between_iters=60) - self.verify_inference_results('bench_serv', i_dict['expected_results'][d_type]) - - def poll_for_server_ready(self, node_no, sglang_function, no_of_iterations=16): - """ - Poll SGLang Prefill or Decode server logs to determine when the server - is ready to accept inference traffic. - - Readiness definition: - --------------------- - A server is considered "ready" when its log shows successful HTTP - requests (HTTP 200 OK), indicating that: - - The server process has started - - The model is loaded - - Network endpoints are listening - - Request handling is functional - - Assumptions: - ------------ - - Log directory is located on a shared filesystem (e.g., NFS) - - Logs are accessible from a designated head node - - Each server writes logs to a predictable per-node path - - Args: - node_no (int): Index of the Prefill or Decode node being checked - sglang_function (str): Server role ('prefill' or 'decode') - no_of_iterations (int): Maximum number of polling attempts before - declaring failure - """ - # ------------------------------------------------------------------ - # Prefill server readiness check - # ------------------------------------------------------------------ - if re.search('prefill', sglang_function): - for j in range(1, no_of_iterations): - log.info(f'Starting poll iteration {j}') - out_dict = self.p_phdl.exec( - f'grep -B 20 -A 20 "200 OK" {self.log_dir}/prefill_node{node_no}/prefill_server.log' - ) - target_pnode = self.prefill_node_list[node_no] - if re.search('GET|POST', out_dict[target_pnode], re.I): - log.info('Wait 60 secs to start serving traffic') - time.sleep(60) - # if re.search('fired up and ready to roll', out_dict[target_pnode], re.I ): - # print('Prefill server {node_no} ready to serve') - return - else: - log.info('Wait for 120 secs and continue polling') - time.sleep(120) - - log.warning(f'Prefill node {node_no} did not get to ready to serve 200 OK state in {j} iterations') - fail_test(f'Prefill node {node_no} did not get to ready to serve 200 OK state in {j} iterations') - # ------------------------------------------------------------------ - # Decode server readiness check - # ------------------------------------------------------------------ - elif re.search('decode', sglang_function): - for j in range(1, no_of_iterations): - log.info(f'Starting poll iteration {j}') - out_dict = self.d_phdl.exec( - f'grep -B 20 -A 20 "200 OK" {self.log_dir}/decode_node{node_no}/decode_server.log' - ) - target_dnode = self.decode_node_list[node_no] - if re.search('GET|POST', out_dict[target_dnode]): - log.info('Wait 60 secs to start serving traffic') - time.sleep(60) - # if re.search('fired up and ready to roll', out_dict[target_dnode], re.I ): - # print('Decode server {node_no} ready to serve') - return - else: - log.info('Wait for 120 secs and continue polling') - time.sleep(120) - log.warning(f'Decode node {node_no} did not get to ready to serve 200 OK state in {j} iterations') - fail_test(f'Decode node {node_no} did not get to ready to serve 200 OK state in {j} iterations') - - def get_inference_results_dict(self, out_dict): - """ - Parse inference benchmark output logs and extract key performance metrics - into a structured dictionary. - - Purpose: - -------- - This method processes raw text output generated by inference benchmarks - (e.g., sglang.bench_serving) and extracts important metrics such as: - - Request counts - - Token throughput - - Latency statistics (TTFT, TPOT) - - Benchmark duration - - The extracted metrics are stored per node in: - self.inference_results_dict - - Args: - out_dict (dict): - Dictionary keyed by node identifier, where each value is the - raw stdout/stderr text produced by the benchmark on that node. - """ - self.inference_results_dict = {} - log.info('Inside get_inference_results_dict') - log.info("%s", out_dict) - for node in out_dict.keys(): - self.inference_results_dict[node] = {} - if re.search('Successful requests:', out_dict[node], re.I): - match = re.search('Successful requests:\s+([0-9]+)', out_dict[node], re.I) - self.inference_results_dict[node]['successful_requests'] = match.group(1) - if re.search('Benchmark duration\s+\(s\):\s+([0-9]+)', out_dict[node], re.I): - match = re.search('Benchmark duration\s+\(s\):\s+([0-9]+)', out_dict[node], re.I) - self.inference_results_dict[node]['benchmark_duration'] = match.group(1) - if re.search('Total input tokens:', out_dict[node], re.I): - match = re.search('Total input tokens:\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['total_input_tokens'] = match.group(1) - if re.search('Total generated tokens:', out_dict[node], re.I): - match = re.search('Total generated tokens:\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['Total generated tokens:'] = match.group(1) - if re.search('Request throughput \(req/s\):', out_dict[node], re.I): - match = re.search('Request throughput \(req/s\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['request_throughput_per_sec'] = match.group(1) - if re.search('Output token throughput \(tok/s\):', out_dict[node], re.I): - match = re.search('Output token throughput \(tok/s\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['output_throughput_per_sec'] = match.group(1) - if re.search('Mean TTFT \(ms\):', out_dict[node], re.I): - match = re.search('Mean TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['mean_ttft_ms'] = match.group(1) - if re.search('Median TTFT (ms):', out_dict[node], re.I): - match = re.search('Median TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['median_ttft_ms'] = match.group(1) - if re.search('P99 TTFT (ms):', out_dict[node], re.I): - match = re.search('P99 TTFT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['p99_ttft_ms'] = match.group(1) - if re.search('Mean TPOT \(ms\)', out_dict[node], re.I): - match = re.search('Mean TPOT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['mean_tpot_ms'] = match.group(1) - if re.search('Median TPOT \(ms\):', out_dict[node], re.I): - match = re.search('Median TPOT \(ms\):\s+([0-9]+)', out_dict[node], re.I) - self.inference_results_dict[node]['median_tpot_ms'] = match.group(1) - if re.search('P99 TPOT (ms):', out_dict[node], re.I): - match = re.search('P99 TPOT \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['p99_tpot_ms'] = match.group(1) - if re.search('Mean ITL \(ms\):', out_dict[node], re.I): - match = re.search('Mean ITL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['mean_itl_ms'] = match.group(1) - if re.search('Median ITL \(ms\):', out_dict[node], re.I): - match = re.search('Median ITL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['median_itl_ms'] = match.group(1) - if re.search('P99 ITL \(ms\):', out_dict[node], re.I): - match = re.search('P99 ITL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['p99_itl_ms'] = match.group(1) - if re.search('Mean E2EL \(ms\):', out_dict[node], re.I): - match = re.search('Mean E2EL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['mean_e2el_ms'] = match.group(1) - if re.search('Median E2EL \(ms\):', out_dict[node], re.I): - match = re.search('Median E2EL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['median_e2el_ms'] = match.group(1) - if re.search('P99 E2EL \(ms\):', out_dict[node], re.I): - match = re.search('P99 E2EL \(ms\):\s+([0-9\.]+)', out_dict[node], re.I) - self.inference_results_dict[node]['p99_e2el_ms'] = match.group(1) - - log.info("%s", self.inference_results_dict) - return self.inference_results_dict - - def scan_for_inference_errors( - self, - ): - """ - Scan Prefill and Decode server logs for known inference error patterns - and fail the test if any are detected. - - Purpose: - -------- - This method performs a post-inference health check by scanning - server logs for known error signatures that indicate: - - Runtime failures - - Communication errors (RDMA/NCCL) - - Out-of-memory conditions - - Kernel or backend crashes - - Fatal exceptions during inference - - The method ensures that even if benchmarks complete, silent or - non-fatal errors do not go unnoticed. - """ - log.info('Scan for inference errors') - inference_pass = True - - # Build the list of commands to read each node's inference log file - cmd_list = [] - - # Scan all prefill nodes - for j in range(0, int(self.prefill_nnodes)): - cmd = f"sudo tail -500 {self.log_dir}/prefill_node{j}/prefill_server.log" - cmd_list.append(cmd) - out_dict = self.p_phdl.exec_cmd_list(cmd_list) - - # Check the log content against all known inference error patterns - for node in out_dict.keys(): - for err_key in inference_err_dict: - if re.search(f'{inference_err_dict[err_key]}', out_dict[node]): - fail_test(f'ERROR {inference_err_dict[err_key]} seen in inference logs ..') - log.error('Aborting inference log polling') - inference_pass = False - - # Scan all decode nodes - cmd_list = [] - for j in range(0, int(self.decode_nnodes)): - cmd = f"sudo tail -500 {self.log_dir}/decode_node{j}/decode_server.log" - cmd_list.append(cmd) - out_dict = self.d_phdl.exec_cmd_list(cmd_list) - - # Check the log content against all known inference error patterns - for node in out_dict.keys(): - for err_key in inference_err_dict: - if re.search(f'{inference_err_dict[err_key]}', out_dict[node]): - fail_test(f'ERROR {inference_err_dict[err_key]} seen in inference logs ..') - log.error('Aborting inference log polling') - inference_pass = False - - return inference_pass - - def poll_for_inference_completion( - self, iterations=10, waittime_between_iters=60, total_timeout=3600, require_all_nodes=True - ): - """ - Poll benchmark logs to detect inference completion and extract results. - - Purpose: - -------- - This method monitors inference progress by periodically inspecting - benchmark output logs. It determines when inference has completed, - detects early failures, and enforces a global timeout. - - Completion criteria: - -------------------- - Inference is considered complete when the benchmark output contains - the pattern 'Serving Benchmark Result'. - - Failure criteria: - ----------------- - Any known inference error detected in Prefill or Decode logs - immediately aborts the process. - - Args: - iterations (int): - Maximum number of polling iterations. - waittime_between_iters (int): - Time (seconds) to wait between polling attempts. - total_timeout (int or None): - Maximum wall-clock time (seconds) allowed for inference. - require_all_nodes (bool): - If True, all nodes must report completion. - If False, completion by any node is sufficient. - """ - # Initial wait to give inference time to start logging - time.sleep(60) - - # Track wall-clock timeout if specified - start_time = time.time() - - def timed_out() -> bool: - return total_timeout is not None and (time.time() - start_time) >= float(total_timeout) - - completed_pattern = re.compile('Serving Benchmark Result', re.I) - # ------------------------------------------------------------------ - # Poll loop: periodically inspect benchmark logs for completion - # ------------------------------------------------------------------ - for itr in range(1, iterations + 1): - log.info(f'Starting iteration {itr}') - - # -------------------------------------------------------------- - # Early exit if any inference errors are detected - # - # This scans Prefill and Decode logs for known failure patterns - # (e.g., OOM, RDMA failures, backend crashes). - # -------------------------------------------------------------- - # Early abort on inference errors - if not self.scan_for_inference_errors(): - msg = 'Failures seen in inference logs, Aborting!!!' - fail_test(msg) - return {"status": "error", "reason": msg} - - # -------------------------------------------------------------- - # Read the most recent benchmark output - # - # Tail only the last 1000 lines to reduce I/O and parsing cost. - # -------------------------------------------------------------- - cmd = f"sudo tail -1000 {self.log_dir}/benchmark_node/benchmark_results.log" - - out_dict = self.b_phdl.exec(cmd) - - # Determine completion across nodes - node_completion = {} - for node, output in out_dict.items(): - node_completion[node] = bool(completed_pattern.search(output)) - - # -------------------------------------------------------------- - # Determine overall completion based on policy - # - # - require_all_nodes=True ? all nodes must complete - # - require_all_nodes=False ? any node completing is sufficient - # -------------------------------------------------------------- - if require_all_nodes: - all_complete = all(node_completion.values()) if node_completion else False - else: - all_complete = any(node_completion.values()) if node_completion else False - - # -------------------------------------------------------------- - # If inference is still running, wait and retry - # -------------------------------------------------------------- - if not all_complete: - if timed_out(): - msg = f"Timeout while waiting for inference completion after ~{int(time.time() - start_time)}s" - log.warning("%s", msg) - return {"status": "timeout", "reason": msg} - log.info('Inference still in progress') - # Short progress wait before the longer inter-iteration sleep - time.sleep(30) - time.sleep(int(waittime_between_iters)) - continue - - # -------------------------------------------------------------- - # Inference completed successfully - # - # Parse benchmark results and return structured output. - # -------------------------------------------------------------- - self.get_inference_results_dict(out_dict) - log.info('Completed Inference, returning !!!') - return {"status": "success", "results": self.inference_results_dict} - - # If we reached here, it means poll for inference completion failed - - # If we exhaust the iteration cap without completing, treat as timeout (or in_progress if no wall-clock limit) - if timed_out(): - msg = f"Timeout after maximum iterations ({self.inference_poll_iterations}) and ~{int(time.time() - start_time)}s" - log.warning("%s", msg) - return {"status": "timeout", "reason": msg} - else: - # If no wall-clock timeout was set and we hit the iteration cap, report in-progress - msg = f"Reached iteration cap ({self.inference_poll_iterations}) without completion; still in progress" - log.warning("%s", msg) - return {"status": "stuck_in_progress", "reason": msg} - - def verify_inference_results(self, test_name, expected_result_dict): - """ - Validate inference benchmark results against expected performance - thresholds and check for system-level errors. - - Purpose: - -------- - This method verifies that: - - Inference completed successfully on all nodes - - Performance metrics meet or exceed expected baselines - - Latency metrics stay below defined thresholds - - No kernel-level (dmesg) errors occurred during inference - - It acts as the final gate for inference validation. - """ - log.info('Verify Inference Completion Msg') - log.info('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%') - log.info("%s", self.inference_results_dict) - log.info('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%') - # ------------------------------------------------------------------ - # Validate metrics on a per-node basis - # ------------------------------------------------------------------ - for node in self.inference_results_dict.keys(): - log.info('%%%% node {}'.format(node)) - for metric_name in expected_result_dict.keys(): - log.info('%%% metric_name {}'.format(metric_name)) - if metric_name in self.inference_results_dict[node].keys(): - # latency metric, so actual should be lower than expected .. - log.info('%% metric found in inference results ^^^') - # ------------------------------------------------------ - # Latency metrics (e.g., TTFT, TPOT) - # - # For latency, lower values are better. - # Fail if actual latency exceeds expected threshold. - # ------------------------------------------------------ - if re.search('ms', metric_name, re.I): - log.info("%s", self.inference_results_dict[node][metric_name]) - log.info("%s", expected_result_dict[metric_name]) - if float(self.inference_results_dict[node][metric_name]) > float( - expected_result_dict[metric_name] - ): - fail_test( - f"FAIL - The metric {metric_name} actual value higher than expected \ - Actual = {self.inference_results_dict[node][metric_name]}, \ - Expected = {expected_result_dict[metric_name]}" - ) - # ------------------------------------------------------ - # Throughput and count metrics - # - # For throughput, higher values are better. - # Fail if actual throughput is lower than expected. - # ------------------------------------------------------ - else: - if float(self.inference_results_dict[node][metric_name]) < float( - expected_result_dict[metric_name] - ): - fail_test( - f"FAIL - The metric {metric_name} actual value lower than expected \ - Actual = {self.inference_results_dict[node][metric_name]}, \ - Expected = {expected_result_dict[metric_name]}" - ) - - # ------------------------------------------------------------------ - # Perform kernel-level (dmesg) error checks - # - # This ensures no silent hardware or driver errors occurred during - # inference (e.g., GPU resets, RDMA failures, IOMMU errors). - # ------------------------------------------------------------------ - self.inference_end_time = self.p_phdl.exec('date +"%a %b %e %H:%M"') - time.sleep(2) - verify_dmesg_for_errors(self.p_phdl, self.inference_start_time, self.inference_end_time) - verify_dmesg_for_errors(self.d_phdl, self.inference_start_time, self.inference_end_time) - verify_dmesg_for_errors(self.r_phdl, self.inference_start_time, self.inference_end_time) - verify_dmesg_for_errors(self.b_phdl, self.inference_start_time, self.inference_end_time) - log.info("%s", self.inference_results_dict) - - def sglang_disagg_gpu_counts(self, mem_threshold_mb=5000): - """ - After model load, count occupied GPUs per prefill/decode node via amd-smi. - """ - tp = int(self.bp_dict["tensor_parallelism"]) - - def _count_per_node(phdl): - per_node = {} - for node, payload in phdl.exec("sudo amd-smi metric --json").items(): - count = 0 - try: - entries = json.loads(payload.strip()) - except (json.JSONDecodeError, AttributeError): - log.warning("Failed to parse amd-smi JSON on node %s", node) - per_node[node] = 0 - continue - if isinstance(entries, dict) and "gpu_data" in entries: - entries = entries["gpu_data"] - if not isinstance(entries, list): - per_node[node] = 0 - continue - for g in entries: - used_mb = g.get("mem_usage", {}).get("used_vram", {}).get("value", 0) - if used_mb > mem_threshold_mb: - count += 1 - per_node[node] = count - return per_node - - prefill_per_node = _count_per_node(self.p_phdl) - decode_per_node = _count_per_node(self.d_phdl) - occupied_prefill = sum(prefill_per_node.values()) - occupied_decode = sum(decode_per_node.values()) - - result = { - "configured_tp": tp, - "prefill_per_node": prefill_per_node, - "decode_per_node": decode_per_node, - "prefill_occupied_gpus": occupied_prefill, - "decode_occupied_gpus": occupied_decode, - "total_occupied_gpus": occupied_prefill + occupied_decode, - } - - lines = [ - "", - f"Configured TP: {tp}", - "", - "Prefill:", - ] - for node, count in prefill_per_node.items(): - lines.append(f" {node}: {count} occupied GPUs") - lines.append(f" Total: {occupied_prefill} occupied GPUs") - lines.append("") - lines.append("Decode:") - for node, count in decode_per_node.items(): - lines.append(f" {node}: {count} occupied GPUs") - lines.append(f" Total: {occupied_decode} occupied GPUs") - lines.append("") - lines.append("Total hardware GPUs consumed:") - lines.append(f" {occupied_prefill + occupied_decode}") - - log.info("\n".join(lines)) - return result diff --git a/cvs/lib/training/jaxmaxtext/__init__.py b/cvs/lib/training/jaxmaxtext/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cvs/lib/training/jaxmaxtext/jaxmaxtext_training_lib.py b/cvs/lib/training/jaxmaxtext/jaxmaxtext_training_lib.py new file mode 100644 index 000000000..31bc72abd --- /dev/null +++ b/cvs/lib/training/jaxmaxtext/jaxmaxtext_training_lib.py @@ -0,0 +1,569 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Standalone JAX MaxText training job driven by a ContainerOrchestrator. + +This class talks only to `orch.exec`, which already routes into the running +container, and to a typed `TrainingVariantConfig` (see +`cvs.lib.training.jaxmaxtext.utils.training_config_loader`). + +All container interaction goes through `orch.exec()`. No direct Pssh or +docker_lib. The training command, env script, and MaxText YAML config are +built in Python and written into the container by the driver — no external +.sh scripts from the MAD repo. + +Both single-node and distributed training use this same class; the config's +`training.distributed` field drives the branching. +''' + +from __future__ import annotations + +import re +import shlex +import time + +from cvs.lib import globals +from cvs.lib.training.jaxmaxtext.utils.maxtext_parsing import ( + parse_training_log, + extract_step_metrics, + extract_eval_metrics, +) + +log = globals.log + +# Bound lazily to cvs.lib.verify_lib.verify_dmesg_for_errors on first use so this +# module stays importable without the broader utils stack that verify_lib pulls +# in (rocm_plib, node_scraper, pytest, ...). Tests patch this symbol directly. +_verify_dmesg_for_errors = None + +# Host-side timestamp used to bound the dmesg scan to this training window. +# Format matches what verify_dmesg_for_errors() expects (dmesg -T style). +_DMESG_TIME_CMD = 'date +"%a %b %e %H:%M"' + +# Default training-log error signatures (name -> regex). Used as the fallback +# when a config does not define `training.error_patterns`; a config's patterns +# fully replace this set. Kept here so the suite still detects common failures +# out of the box. +_TRAINING_ERR_PATTERNS = { + 'NCCL ERROR': r'NCCL ERROR|NCCL timeout|local work queue catastrophic error', + 'GPU HW ERROR': r'HW Exception by GPU|GPU Hang|Uncorrectable error|GPU Reset', + 'AssertionError': r'AssertionError|ValueError:|JaxStackTrace|During handling of the above exception|triggered the following exception', + 'rocm Err': r'FAILED_PRECONDITION: No visible GPU devices|failed call to hipInit: HIP_ERROR_NoDevice|librocm reported version is: NOT_FOUND', + 'python err': r'ModuleNotFoundError: No module named|Fatal Python error:', + 'tensorflow': r'tensorflow.CoordinationServiceError|tensorflow.BarrierError|CoordinationServiceError', + 'resource': r'RESOURCE_EXHAUSTED: Out of memory|failed: RESOURCE_EXHAUSTED', + 'segfault': r'Segmentation fault|SIGSEGV|signal 11|core dumped', +} + +_NAN_INF_RE = re.compile(r'(TFLOP/s/device|Tokens/s/device):\s*(NaN|Inf|-Inf)', re.I) + + +def _sanitize(name): + """Filesystem/run-name-safe token from a sweep name (non-alnum -> '_').""" + return re.sub(r'[^A-Za-z0-9]+', '_', str(name)).strip('_') or "default" + + +class MaxTextTrainingJob: + """JAX MaxText training job driven by an injected ContainerOrchestrator. + + All container/SSH plumbing belongs to `orch`. This class composes the + env script, MaxText YAML config, launches training in the background + inside the container, polls until complete, and parses the resulting log. + + The `orch` instance is expected to already have `setup_containers()` + called against it (by the test fixture); lifecycle is explicitly NOT + owned here. + """ + + def __init__(self, orch, variant, hf_token, sweep=None): + self.orch = orch + self.variant = variant + self.hf_token = hf_token + self.training = variant.training + + # Per-sweep run: merge the sweep's maxtext overrides onto the base config, + # and namespace the output dir by the sweep so parallel sweeps' logs never + # clobber each other (is_complete/parse_results read this per-sweep dir). + self.sweep = sweep + self.sweep_tag = _sanitize(sweep.name) if sweep is not None else None + merged = dict(self.training.maxtext_config) + if sweep is not None and getattr(sweep, "maxtext_overrides", None): + merged.update(sweep.maxtext_overrides) + self.maxtext_config = merged + + self.log_dir = variant.paths.log_dir + self.out_dir = f"{self.log_dir}/jaxmaxtext/{self.sweep_tag}" if self.sweep_tag else f"{self.log_dir}/jaxmaxtext" + self.num_nodes = len(orch.hosts) + # GPUs-per-node is config-driven -- do not assume a uniform 8-GPU topology. + # It feeds num_gpus -> tokens_per_sec_total -> scaling efficiency, so an + # implicit constant would silently skew a gated-adjacent metric. + self.gpus_per_node = int(getattr(self.training, "gpus_per_node", 8) or 8) + self.num_gpus = self.num_nodes * self.gpus_per_node + + # Training-log error signatures scanned during polling. Sourced from the + # config (`training.error_patterns`) so users can add/remove signatures + # without code changes; falls back to the built-in defaults when the + # config omits them. + self.error_patterns = dict(getattr(self.training, "error_patterns", None) or {}) or dict(_TRAINING_ERR_PATTERNS) + + self.step_metrics = [] + self.eval_metrics = [] + self.summary_metrics = {} + + # Host-side timestamp ({node: str}) captured when training launches, so + # scan_dmesg_for_errors() can slice the kernel log to this run's window. + self.training_start_time = None + + self._poll_wait_s = 60 + self._poll_count = int(self.training.steps * 10) + self._initial_wait_s = 60 + + self._scratch_dir = None # resolved lazily to /tmp//jax + self._train_script = None # resolved lazily to the first existing candidate + + def _get_scratch_dir(self): + """User-namespaced in-container scratch base (``/tmp//jax``). + + Namespacing by the container user avoids /tmp ownership collisions on + shared GPU nodes: a scratch dir left behind by one user would otherwise + block a different user's run with a permission error. Resolved once + (via ``id -un``) and cached. + """ + if self._scratch_dir: + return self._scratch_dir + user = "cvs" + try: + out = self.orch.exec("bash -c " + shlex.quote("id -un 2>/dev/null || true")) + raw = (out or {}).get(self.orch.hosts[0], "") + text = raw if isinstance(raw, str) else (raw or {}).get("output", "") + text = (text or "").strip() + if text: + user = text.splitlines()[-1].strip() or "cvs" + except Exception: # noqa: BLE001 - fall back to a safe default + pass + self._scratch_dir = f"/tmp/{user}/jax" + return self._scratch_dir + + def _resolve_train_script(self): + """Return the first configured train-script path that exists in the container. + + MaxText moved the train entrypoint across versions (v26.3 and earlier: + ``.../src/MaxText/train.py``; v26.4+: ``.../src/maxtext/trainers/pre_train/ + train.py``). The config lists candidates in ``train_script_paths`` and we + pick whichever the running image ships, so the same config works across + versions. Resolved once and cached. + """ + if self._train_script: + return self._train_script + candidates = list(getattr(self.training, "train_script_paths", None) or []) + single = getattr(self.training, "train_script", None) + if single and single not in candidates: + candidates.append(single) + if not candidates: + raise RuntimeError("no train_script_paths (or train_script) configured") + + probe = "".join(f"if [ -f {shlex.quote(p)} ]; then echo {shlex.quote(p)}; exit 0; fi; " for p in candidates) + out = self.orch.exec("bash -c " + shlex.quote(probe)) + raw = (out or {}).get(self.orch.hosts[0], "") + text = raw if isinstance(raw, str) else (raw or {}).get("output", "") + resolved = (text or "").strip().splitlines()[0].strip() if (text or "").strip() else "" + if not resolved: + raise RuntimeError(f"none of the configured train_script_paths exist in the container: {candidates}") + log.info("resolved train_script: %s", resolved) + self._train_script = resolved + return resolved + + # ---------- setup ---------- + + def setup_training_env(self): + """Write env script and MaxText YAML config into the container.""" + self.orch.exec(f"mkdir -p {shlex.quote(self._get_scratch_dir())}") + self.orch.exec(f"mkdir -p {shlex.quote(self.out_dir)}") + for i in range(self.num_nodes): + self.orch.exec(f"mkdir -p {shlex.quote(self.out_dir)}/out-node{i}") + + self._write_env_script() + self._write_maxtext_yaml() + + def _build_xla_flags_str(self): + parts = [] + for k, v in self.training.xla_flags.items(): + parts.append(f"--{k}={v}") + return " ".join(parts) + + def _write_env_script(self): + """Write the env script sourced before training launch.""" + lines = [] + + lines.append(f"export HF_TOKEN={shlex.quote(self.hf_token)}") + lines.append(f"export HF_HOME={shlex.quote(self.variant.paths.models_dir)}") + lines.append("export LD_LIBRARY_PATH=/opt/rocm/lib:$LD_LIBRARY_PATH") + + for k, v in self.training.env_vars.items(): + lines.append(f"export {k}={shlex.quote(str(v))}") + + xla_flags = self._build_xla_flags_str() + if xla_flags: + lines.append(f'export XLA_FLAGS="{xla_flags}"') + + if self.training.distributed: + nccl = self.training.nccl + if nccl.ib_hca: + lines.append(f"export NCCL_IB_HCA={shlex.quote(nccl.ib_hca)}") + if nccl.ib_hca_list: + lines.append(f"export NCCL_IB_HCA_LIST={shlex.quote(nccl.ib_hca_list)}") + if nccl.socket_ifname: + lines.append(f"export NCCL_SOCKET_IFNAME={shlex.quote(nccl.socket_ifname)}") + if nccl.gloo_socket_ifname: + lines.append(f"export GLOO_SOCKET_IFNAME={shlex.quote(nccl.gloo_socket_ifname)}") + else: + lines.append("export NCCL_IB_DISABLE=1") + lines.append("export NCCL_SHM_DISABLE=0") + lines.append("export NCCL_P2P_DISABLE=0") + + env_script = "\n".join(lines) + "\n" + env_path = f"{self._get_scratch_dir()}/maxtext_env.sh" + self.orch.exec("bash -c " + shlex.quote(f"printf '%s' {shlex.quote(env_script)} > {env_path}")) + + def _write_maxtext_yaml(self): + """Write the MaxText YAML config into the container.""" + mc = dict(self.maxtext_config) + + run_name = f"jaxmaxtext_{self.variant.model.id}" + if self.sweep_tag: + run_name = f"{run_name}_{self.sweep_tag}" + mc["run_name"] = run_name + mc["steps"] = self.training.steps + mc["enable_checkpointing"] = self.training.enable_checkpointing + mc["base_output_directory"] = self.out_dir + mc["tokenizer_path"] = self.training.tokenizer.tokenizer_path + + yml_lines = [] + for k, v in mc.items(): + if isinstance(v, list): + yml_lines.append(f'{k}: {v}') + elif isinstance(v, bool): + yml_lines.append(f"{k}: {'true' if v else 'false'}") + else: + yml_lines.append(f"{k}: {v}") + + yml_content = "\n".join(yml_lines) + "\n" + yml_path = f"{self._get_scratch_dir()}/maxtext_config.yml" + self.orch.exec("bash -c " + shlex.quote(f"cat > {yml_path} <<'YMLEOF'\n{yml_content}YMLEOF")) + + # ---------- RDMA / NIC setup ---------- + + def setup_rdma_lib(self): + """Copy host RDMA library into container (Broadcom/Thor2 NIC workaround).""" + rdma = self.training.rdma_lib + if not rdma.container_mount_file or not rdma.container_dest_file: + log.info("rdma_lib paths not configured, skipping") + return + cmd = f"sudo cp {shlex.quote(rdma.container_mount_file)} {shlex.quote(rdma.container_dest_file)}" + out = self.orch.exec(cmd) + for host, output in (out or {}).items(): + log.info("[rdma_lib %s] %s", host, (output or "")[:200]) + + verify = self.orch.exec("ibv_devinfo 2>/dev/null | head -20") + for host, output in (verify or {}).items(): + if not re.search(r'hca_id:\s+(bnxt_|rocep|rdma)', output or "", re.I): + raise RuntimeError(f"RDMA library not properly configured on {host}: {(output or '')[:300]}") + + # ---------- tokenizer ---------- + + def setup_tokenizer(self): + """Download HuggingFace tokenizer into the models dir.""" + tok = self.training.tokenizer + models_dir = self.variant.paths.models_dir + self.orch.exec(f"mkdir -p {shlex.quote(models_dir)}") + + hf_model = tok.hf_model_id + if not hf_model: + log.info("tokenizer.hf_model_id not set, skipping download") + return + + # Export the credentials inline rather than sourcing /tmp/jax/maxtext_env.sh: + # the tokenizer stage runs before setup_training_env() writes that env + # script, so sourcing it here fails with "No such file or directory". + dl_cmd = ( + f"export HF_TOKEN={shlex.quote(self.hf_token)} && " + f"export HF_HOME={shlex.quote(self.variant.paths.models_dir)} && " + f"huggingface-cli download {shlex.quote(hf_model)} --local-dir {shlex.quote(tok.tokenizer_path)}" + ) + log.info("downloading tokenizer: %s -> %s", hf_model, tok.tokenizer_path) + self.orch.exec("bash -c " + shlex.quote(dl_cmd)) + + # ---------- training launch ---------- + + def build_training_cmd(self): + """Build the per-node training launcher scripts and write them into the + container. + + Each node gets its own script with a distinct JAX_PROCESS_INDEX/NODE_RANK. + The scripts are written across nodes in a single parallel + ``orch.exec_cmd_list`` call where ``cmd_list[i]`` runs on ``hosts[i]`` -- + so rank i's launcher only ever lands on host i. + """ + scratch = self._get_scratch_dir() + train_script = self._resolve_train_script() + write_cmds = [] + for i in range(self.num_nodes): + launcher_lines = [ + "#!/bin/bash", + f"source {scratch}/maxtext_env.sh", + ] + + if self.training.distributed: + jax_dist = self.training.jax_distributed + # "auto" (or empty) -> use the first cluster node (node_dict order, + # i.e. orch.hosts[0]) as the JAX coordinator; an explicit IP in the + # config overrides it. + coordinator_ip = (getattr(jax_dist, "coordinator_ip", "") or "").strip() + if not coordinator_ip or coordinator_ip.lower() == "auto": + coordinator_ip = self.orch.hosts[0] + launcher_lines.extend( + [ + f"export JAX_COORDINATOR_IP={shlex.quote(coordinator_ip)}", + f"export JAX_COORDINATOR_PORT={shlex.quote(jax_dist.coordinator_port)}", + f"export NNODES={self.num_nodes}", + f"export NODE_RANK={i}", + f"export JAX_PROCESS_INDEX={i}", + f"export JAX_DISTRIBUTED_INITIALIZATION_TIMEOUT_SECONDS={jax_dist.initialization_timeout_seconds}", + f"export JAX_DISTRIBUTED_HEARTBEAT_TIMEOUT_SECONDS={jax_dist.heartbeat_timeout_seconds}", + ] + ) + else: + launcher_lines.extend( + [ + "export JAX_COORDINATOR_IP=localhost", + "export JAX_COORDINATOR_PORT=12346", + "export NNODES=1", + "export NODE_RANK=0", + "export JAX_PROCESS_INDEX=0", + ] + ) + + launcher_lines.append("export PYTHONPATH=$PYTHONPATH:/workspace/maxtext/") + log_file = f"{self.out_dir}/out-node{i}/training.log" + launcher_lines.append( + f"cd /workspace/maxtext && python {shlex.quote(train_script)} " + f"{scratch}/maxtext_config.yml 2>&1 | tee {shlex.quote(log_file)}" + ) + + script_content = "\n".join(launcher_lines) + "\n" + script_path = f"{scratch}/training_launcher_node{i}.sh" + write_cmds.append( + "bash -c " + + shlex.quote(f"printf '%s' {shlex.quote(script_content)} > {script_path} && chmod +x {script_path}") + ) + + # cmd_list[i] -> hosts[i]: write each node's launcher only on its own host. + self.orch.exec_cmd_list(write_cmds) + + def start_training(self): + """Launch training in the background on every node in parallel. + + Uses ``orch.exec_cmd_list`` so ``cmd_list[i]`` runs on ``hosts[i]``: each + node runs only its own rank-i launcher, and all ranks start together so + JAX distributed init can rendezvous within the timeout. Fanning the same + command out to every host (plain ``exec``) would start every rank on + every node, so multiple processes would claim the same JAX_PROCESS_INDEX + and the coordinator aborts with a "different incarnation" error. + """ + log.info("starting training on %d node(s)", self.num_nodes) + + # Record the host-side start time so a later dmesg scan only looks at + # kernel messages emitted during this training run. + self.training_start_time = self._host_date() + + scratch = self._get_scratch_dir() + launch_cmds = [] + for i in range(self.num_nodes): + script_path = f"{scratch}/training_launcher_node{i}.sh" + redirect_log = f"{self.out_dir}/out-node{i}/training_redirect_logs" + inner = f"nohup bash {script_path} > {shlex.quote(redirect_log)} 2>&1 &" + launch_cmds.append("bash -c " + shlex.quote(inner)) + + self.orch.exec_cmd_list(launch_cmds) + + time.sleep(self._initial_wait_s) + + # ---------- polling ---------- + + def is_complete(self): + """Check if training has completed on all nodes. + + Greps each node's own training.log in a single parallel + ``orch.exec_cmd_list`` call (``cmd_list[i]`` runs on ``hosts[i]``). Uses + ``|| true`` rather than ``|| echo 0`` so a no-match yields a clean "0": + ``grep -c`` already prints "0" and exits 1 on no match, so ``|| echo 0`` + would emit "0\\n0" and defeat the equality check below. + """ + final_step = self.training.steps - 1 + pattern = f"completed step:\\s*{final_step}," + cmd_list = [ + f"grep -cE {shlex.quote(pattern)} " + f"{shlex.quote(f'{self.out_dir}/out-node{i}/training.log')} 2>/dev/null || true" + for i in range(self.num_nodes) + ] + out = self.orch.exec_cmd_list(cmd_list) + if not out or len(out) < self.num_nodes: + return False + for _host, result in out.items(): + text = result if isinstance(result, str) else (result or {}).get("output", "") + text = (text or "").strip() + if not text or text == "0": + return False + return True + + def _scan_for_errors(self): + """Scan each node's own training log for known error patterns. + + Reads all nodes' logs in one parallel ``orch.exec_cmd_list`` call + (``cmd_list[i]`` runs on ``hosts[i]``). Raises on the first match. + """ + cmd_list = [ + f"tail -2000 {shlex.quote(f'{self.out_dir}/out-node{i}/training.log')} 2>/dev/null" + for i in range(self.num_nodes) + ] + out = self.orch.exec_cmd_list(cmd_list) + node_of = {h: i for i, h in enumerate(self.orch.hosts)} + for host, text in (out or {}).items(): + text = text if isinstance(text, str) else (text or {}).get("output", "") + text = text or "" + i = node_of.get(host, "?") + if _NAN_INF_RE.search(text): + raise RuntimeError(f"NaN/Inf in training metrics on {host} (node {i}): {text[-500:]}") + for err_name, err_pattern in self.error_patterns.items(): + if not err_pattern: + continue + if re.search(err_pattern, text, re.I): + raise RuntimeError(f"Training error '{err_name}' on {host} (node {i}): {text[-500:]}") + + def poll_for_completion(self, timeout_s=None): + """Poll is_complete() with error scanning until training finishes or times out.""" + if timeout_s is None: + timeout_s = self._poll_count * self._poll_wait_s + + start = time.monotonic() + for it in range(self._poll_count): + elapsed = time.monotonic() - start + if elapsed >= timeout_s: + raise RuntimeError(f"training did not complete within {timeout_s}s (polled {it} times)") + + self._scan_for_errors() + + if self.is_complete(): + log.info("training complete (poll iter=%d, %.0fs elapsed)", it, elapsed) + return + + log.info( + "training in progress (poll iter=%d, %.0fs elapsed)", + it, + elapsed, + ) + time.sleep(self._poll_wait_s) + + raise RuntimeError(f"training did not complete after {self._poll_count} poll iterations") + + # ---------- results ---------- + + def parse_results(self): + """Parse per-step metrics from training log, compute aggregates. + + Reads the training log from node 0 (the coordinator), parses it via + the pure `parse_training_log`, and stores both per-step and aggregate + metrics on self. + """ + log_file = f"{self.out_dir}/out-node0/training.log" + # Read node 0's (coordinator) log. Only hosts[0] runs the cat; the other + # nodes get a no-op so cmd_list[i] still lines up with hosts[i]. + cmd_list = [f"cat {shlex.quote(log_file)}" if i == 0 else "true" for i in range(self.num_nodes)] + out = self.orch.exec_cmd_list(cmd_list) or {} + raw = out.get(self.orch.hosts[0], "") + log_text = raw if isinstance(raw, str) else (raw or {}).get("output", "") + log_text = log_text or "" + + if not log_text.strip(): + raise RuntimeError(f"empty/missing training log: {log_file}") + + self.step_metrics = extract_step_metrics(log_text) + self.eval_metrics = extract_eval_metrics(log_text) + self.summary_metrics = parse_training_log(log_text, self.num_gpus) + return dict(self.summary_metrics) + + # ---------- system checks ---------- + + def _host_date(self): + """Return {host: timestamp} from the cluster host OS (not the container). + + Uses the baremetal fan-out handle (``orch.all``) so the timestamp lines + up with ``dmesg -T`` on the same hosts. Best-effort: returns None if the + handle/exec is unavailable so callers can skip the dmesg scan cleanly. + """ + allh = getattr(self.orch, "all", None) + if allh is None or not hasattr(allh, "exec"): + return None + try: + return allh.exec(_DMESG_TIME_CMD) + except Exception as e: # noqa: BLE001 - infra probe, never fatal + log.warning("could not capture host time for dmesg scan: %s", e) + return None + + def scan_dmesg_for_errors(self): + """Scan host kernel logs (dmesg) on all nodes for GPU/HW/kernel faults. + + Ports the sglang flow to the training suite: over the [start, end] window + captured around the training loop, the shared ``verify_dmesg_for_errors`` + scanner flags HW/crash/driver/network signatures via ``fail_test`` (these + roll up into the suite's aggregated failure summary) and logs + perf-degradation signatures as warnings only. + + Best-effort by design: gated on ``training.verify_dmesg`` (default on) and + wrapped so an infra failure of the scan itself (no passwordless sudo, an + unexpected ``date`` format, a missing baremetal handle) is logged and + swallowed -- it must never mask or replace the actual training result. + Requires a captured start time (i.e. ``start_training`` ran). + """ + if not getattr(self.training, "verify_dmesg", True): + log.info("dmesg verification disabled (training.verify_dmesg=false)") + return + if not self.training_start_time: + log.warning("dmesg verification skipped: no training start time captured") + return + allh = getattr(self.orch, "all", None) + if allh is None or not hasattr(allh, "exec"): + log.warning("dmesg verification skipped: no baremetal host handle (orch.all)") + return + try: + verify = _verify_dmesg_for_errors + if verify is None: + from cvs.lib.verify_lib import verify_dmesg_for_errors as verify + end_time = self._host_date() + time.sleep(2) + verify(allh, self.training_start_time, end_time) + except Exception as e: # noqa: BLE001 - scan infra failure is non-fatal + log.warning("dmesg verification skipped (scan failed): %s", e) + + # ---------- cleanup ---------- + + def stop_training(self): + """Best-effort kill of lingering training processes on every node. + + Called when a sweep fails/times out so the next sweep does not launch on + top of orphaned ranks (important for persistent containers, where per-run + teardown does not reap them). + + Uses a bracketed first character in the pattern: a running rank's cmdline + contains ``maxtext_config.yml``/``training_launcher_node`` and matches, + but this ``pkill`` wrapper's own cmdline contains the literal + ``[m]axtext_config.yml`` / ``[t]raining_launcher_node`` which the regex + does not match -- so pkill never targets itself. + """ + log.info("stopping lingering training processes") + self.orch.exec( + "bash -c " + + shlex.quote("pkill -9 -f '[m]axtext_config.yml' || true; pkill -9 -f '[t]raining_launcher_node' || true") + ) + time.sleep(3) diff --git a/cvs/lib/training/jaxmaxtext/unittests/__init__.py b/cvs/lib/training/jaxmaxtext/unittests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cvs/lib/training/jaxmaxtext/unittests/test_jaxmaxtext_training_lib.py b/cvs/lib/training/jaxmaxtext/unittests/test_jaxmaxtext_training_lib.py new file mode 100644 index 000000000..357eca398 --- /dev/null +++ b/cvs/lib/training/jaxmaxtext/unittests/test_jaxmaxtext_training_lib.py @@ -0,0 +1,423 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs/lib/training/jaxmaxtext/jaxmaxtext_training_lib.py::MaxTextTrainingJob. + +The job talks to the outside world only through an injected orchestrator +(`orch.exec` / `orch.exec_cmd_list`), so every test builds a job with a +MagicMock orch and a lightweight SimpleNamespace variant -- no SSH, no +container, no real sleeps (mirrors test_megatron_training_lib.py). +''' + +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from cvs.lib.training.jaxmaxtext.jaxmaxtext_training_lib import MaxTextTrainingJob + + +def _training(**overrides): + t = SimpleNamespace( + steps=3, + distributed=True, + enable_checkpointing=False, + train_script="/workspace/maxtext/src/MaxText/train.py", + maxtext_config={ + "per_device_batch_size": 2, + "max_target_length": 8192, + "scan_layers": True, + "mlp_activations": ["silu", "linear"], + }, + nic_type="thor2", + env_vars={"NCCL_DEBUG": "ERROR"}, + xla_flags={"xla_gpu_autotune_level": "0", "xla_gpu_enable_triton_gemm": "False"}, + nccl=SimpleNamespace( + ib_hca="rdma0", + ib_hca_list="rdma0,rdma1", + socket_ifname="eno0", + gloo_socket_ifname="eno0", + ), + jax_distributed=SimpleNamespace( + coordinator_ip="auto", + coordinator_port="12346", + initialization_timeout_seconds="1800", + heartbeat_timeout_seconds="900", + ), + rdma_lib=SimpleNamespace(container_mount_file="", container_dest_file=""), + tokenizer=SimpleNamespace(hf_model_id="", tokenizer_path="/models/tok"), + ) + for k, v in overrides.items(): + setattr(t, k, v) + return t + + +def _make_job(hosts=None, **training_overrides): + hosts = hosts or ["h0"] + orch = MagicMock() + orch.hosts = list(hosts) + orch.exec = MagicMock(return_value={}) + orch.exec_cmd_list = MagicMock(return_value={}) + variant = SimpleNamespace( + training=_training(**training_overrides), + model=SimpleNamespace(id="llama3.3-70b"), + paths=SimpleNamespace(log_dir="/logs", models_dir="/models"), + ) + return MaxTextTrainingJob(orch, variant, hf_token="dummy"), orch + + +def _wire_container_exec(orch, user="tester", script="/workspace/maxtext/src/MaxText/train.py"): + """Answer the in-container probes the job runs before building launchers. + + ``build_training_cmd`` resolves the scratch dir (``id -un``) and the train + script (a ``[ -f ... ]`` probe) via ``orch.exec``; without wiring these the + default empty response would make ``_resolve_train_script`` raise. + """ + + def _side(cmd, *a, **k): + text = str(cmd) + if "id -un" in text: + return {h: user for h in orch.hosts} + if "train.py" in text: + return {h: script for h in orch.hosts} + return {} + + orch.exec.side_effect = _side + + +def _log(steps=3): + lines = [] + for i in range(steps): + lines.append( + f"I0804 08:14:00 1 metric_logger.py:196] completed step: {i}, seconds: 0.5, " + f"TFLOP/s/device: 200.0, Tokens/s/device: 25000.0, total_weights: 1, loss: {9.0 - i}" + ) + return "\n".join(lines) + "\n" + + +class ConstructorTests(unittest.TestCase): + def test_node_and_gpu_counts(self): + job, _ = _make_job(hosts=["h0", "h1"]) + self.assertEqual(job.num_nodes, 2) + self.assertEqual(job.num_gpus, 16) + self.assertEqual(job.out_dir, "/logs/jaxmaxtext") + + def test_build_xla_flags_str(self): + job, _ = _make_job() + s = job._build_xla_flags_str() + self.assertIn("--xla_gpu_autotune_level=0", s) + self.assertIn("--xla_gpu_enable_triton_gemm=False", s) + + def test_gpus_per_node_from_config(self): + # num_gpus derives from config gpus_per_node, not a hardcoded 8. + job, _ = _make_job(hosts=["h0", "h1"], gpus_per_node=4) + self.assertEqual(job.gpus_per_node, 4) + self.assertEqual(job.num_gpus, 8) + + def test_gpus_per_node_defaults_to_8(self): + job, _ = _make_job(hosts=["h0"]) # fake config has no gpus_per_node + self.assertEqual(job.gpus_per_node, 8) + self.assertEqual(job.num_gpus, 8) + + +class StopTrainingTests(unittest.TestCase): + @patch("cvs.lib.training.jaxmaxtext.jaxmaxtext_training_lib.time.sleep") + def test_uses_bracketed_self_safe_pattern(self, _sleep): + job, orch = _make_job(hosts=["h0"]) + job.stop_training() + cmd = orch.exec.call_args.args[0] + # Bracketed first char so the pkill wrapper's own cmdline is not matched. + self.assertIn("[m]axtext_config.yml", cmd) + self.assertIn("[t]raining_launcher_node", cmd) + + +class IsCompleteTests(unittest.TestCase): + def test_all_nodes_complete(self): + job, orch = _make_job(hosts=["h0", "h1"]) + orch.exec_cmd_list.return_value = {"h0": "1", "h1": "1"} + self.assertTrue(job.is_complete()) + + def test_one_node_incomplete(self): + job, orch = _make_job(hosts=["h0", "h1"]) + orch.exec_cmd_list.return_value = {"h0": "1", "h1": "0"} + self.assertFalse(job.is_complete()) + + def test_missing_host_output(self): + job, orch = _make_job(hosts=["h0", "h1"]) + orch.exec_cmd_list.return_value = {"h0": "1"} + self.assertFalse(job.is_complete()) + + def test_dict_shaped_result(self): + job, orch = _make_job(hosts=["h0"]) + orch.exec_cmd_list.return_value = {"h0": {"output": "1"}} + self.assertTrue(job.is_complete()) + + +class ScanForErrorsTests(unittest.TestCase): + def test_clean_log_no_raise(self): + job, orch = _make_job(hosts=["h0"]) + orch.exec_cmd_list.return_value = {"h0": _log()} + job._scan_for_errors() # should not raise + + def test_nccl_error_raises(self): + job, orch = _make_job(hosts=["h0"]) + orch.exec_cmd_list.return_value = {"h0": "some log\nNCCL ERROR: unhandled\n"} + with self.assertRaises(RuntimeError): + job._scan_for_errors() + + def test_nan_metric_raises(self): + job, orch = _make_job(hosts=["h0"]) + orch.exec_cmd_list.return_value = {"h0": "completed step: 1, TFLOP/s/device: NaN\n"} + with self.assertRaises(RuntimeError): + job._scan_for_errors() + + def test_config_error_patterns_replace_defaults(self): + # A config-provided error_patterns set fully REPLACES the built-in defaults. + job, orch = _make_job(hosts=["h0"], error_patterns={"custom": "MY_CUSTOM_ERR"}) + # The default NCCL signature is no longer active -> no raise. + orch.exec_cmd_list.return_value = {"h0": "some log\nNCCL ERROR: unhandled\n"} + job._scan_for_errors() + # The custom signature IS active -> raises. + orch.exec_cmd_list.return_value = {"h0": "boom MY_CUSTOM_ERR here\n"} + with self.assertRaises(RuntimeError): + job._scan_for_errors() + + def test_default_error_patterns_used_when_config_empty(self): + # No config error_patterns -> built-in defaults apply. + job, orch = _make_job(hosts=["h0"]) + orch.exec_cmd_list.return_value = {"h0": "RESOURCE_EXHAUSTED: Out of memory\n"} + with self.assertRaises(RuntimeError): + job._scan_for_errors() + + def test_default_segfault_pattern_raises(self): + # segfault is part of the built-in default signatures. + job, orch = _make_job(hosts=["h0"]) + orch.exec_cmd_list.return_value = {"h0": "worker: Segmentation fault (core dumped)\n"} + with self.assertRaises(RuntimeError): + job._scan_for_errors() + + +class ParseResultsTests(unittest.TestCase): + def test_parses_from_node0_log(self): + job, orch = _make_job(hosts=["h0", "h1"]) + orch.exec_cmd_list.return_value = {"h0": _log(steps=3), "h1": ""} + summary = job.parse_results() + self.assertEqual(len(job.step_metrics), 3) + self.assertIn("training.final_loss", summary) + self.assertAlmostEqual(summary["training.final_loss"], 7.0) + + def test_empty_log_raises(self): + job, orch = _make_job(hosts=["h0"]) + orch.exec_cmd_list.return_value = {"h0": " "} + with self.assertRaises(RuntimeError): + job.parse_results() + + +class SetupRdmaLibTests(unittest.TestCase): + def test_skip_when_paths_unset(self): + job, orch = _make_job() # rdma_lib defaults are empty strings + job.setup_rdma_lib() + orch.exec.assert_not_called() + + def test_raises_when_devinfo_mismatch(self): + job, orch = _make_job(rdma_lib=SimpleNamespace(container_mount_file="/src.so", container_dest_file="/dst.so")) + orch.exec.return_value = {"h0": "no matching hca here"} + with self.assertRaises(RuntimeError): + job.setup_rdma_lib() + + def test_ok_when_devinfo_matches(self): + job, orch = _make_job(rdma_lib=SimpleNamespace(container_mount_file="/src.so", container_dest_file="/dst.so")) + orch.exec.return_value = {"h0": "hca_id: bnxt_re0\n"} + job.setup_rdma_lib() # should not raise + + +class SetupTokenizerTests(unittest.TestCase): + def test_skips_download_when_no_model_id(self): + job, orch = _make_job() # hf_model_id="" by default + job.setup_tokenizer() + # Only the mkdir exec fires; no huggingface-cli download command. + joined = " ".join(str(c.args[0]) for c in orch.exec.call_args_list) + self.assertNotIn("huggingface-cli", joined) + + def test_downloads_when_model_id_set(self): + job, orch = _make_job(tokenizer=SimpleNamespace(hf_model_id="org/model", tokenizer_path="/models/tok")) + job.setup_tokenizer() + joined = " ".join(str(c.args[0]) for c in orch.exec.call_args_list) + self.assertIn("huggingface-cli download", joined) + self.assertIn("org/model", joined) + + +class BuildTrainingCmdTests(unittest.TestCase): + def test_distributed_per_rank_indices(self): + job, orch = _make_job(hosts=["h0", "h1"]) + _wire_container_exec(orch) + job.build_training_cmd() + cmds = orch.exec_cmd_list.call_args.args[0] + self.assertEqual(len(cmds), 2) + self.assertIn("JAX_PROCESS_INDEX=0", cmds[0]) + self.assertIn("NODE_RANK=0", cmds[0]) + self.assertIn("JAX_PROCESS_INDEX=1", cmds[1]) + self.assertIn("NODE_RANK=1", cmds[1]) + # coordinator IP is host 0 + self.assertIn("JAX_COORDINATOR_IP=h0", cmds[0]) + # resolved train script and user-namespaced scratch dir are wired in + self.assertIn("/workspace/maxtext/src/MaxText/train.py", cmds[0]) + self.assertIn("/tmp/tester/jax/maxtext_env.sh", cmds[0]) + self.assertIn("/tmp/tester/jax/maxtext_config.yml", cmds[0]) + + def test_single_node_localhost_coordinator(self): + job, orch = _make_job(hosts=["h0"], distributed=False) + _wire_container_exec(orch) + job.build_training_cmd() + cmds = orch.exec_cmd_list.call_args.args[0] + self.assertEqual(len(cmds), 1) + self.assertIn("JAX_COORDINATOR_IP=localhost", cmds[0]) + self.assertIn("JAX_PROCESS_INDEX=0", cmds[0]) + + def test_auto_coordinator_uses_first_host(self): + # coordinator_ip "auto" -> first cluster node (orch.hosts[0]). + job, orch = _make_job(hosts=["10.0.0.5", "10.0.0.6"]) + _wire_container_exec(orch) + job.build_training_cmd() + cmds = orch.exec_cmd_list.call_args.args[0] + self.assertIn("JAX_COORDINATOR_IP=10.0.0.5", cmds[0]) + + def test_explicit_coordinator_ip_overrides_auto(self): + # A concrete coordinator_ip in the config wins over the first host. + job, orch = _make_job( + hosts=["10.0.0.5", "10.0.0.6"], + jax_distributed=SimpleNamespace( + coordinator_ip="10.9.9.9", + coordinator_port="12346", + initialization_timeout_seconds="1800", + heartbeat_timeout_seconds="900", + ), + ) + _wire_container_exec(orch) + job.build_training_cmd() + cmds = orch.exec_cmd_list.call_args.args[0] + self.assertIn("JAX_COORDINATOR_IP=10.9.9.9", cmds[0]) + + +class TrainScriptResolveTests(unittest.TestCase): + def test_returns_first_existing_probed_path(self): + job, orch = _make_job(hosts=["h0", "h1"]) + v264 = "/workspace/maxtext/src/maxtext/trainers/pre_train/train.py" + orch.exec.side_effect = lambda cmd, *a, **k: ({h: v264 for h in orch.hosts} if "train.py" in str(cmd) else {}) + self.assertEqual(job._resolve_train_script(), v264) + + def test_raises_when_no_candidate_exists(self): + job, orch = _make_job() + orch.exec.return_value = {"h0": ""} + with self.assertRaises(RuntimeError): + job._resolve_train_script() + + def test_result_is_cached(self): + job, orch = _make_job() + orch.exec.return_value = {"h0": "/workspace/maxtext/src/MaxText/train.py"} + first = job._resolve_train_script() + count_after_first = orch.exec.call_count + second = job._resolve_train_script() + self.assertEqual(first, second) + self.assertEqual(orch.exec.call_count, count_after_first) + + +class ScratchDirTests(unittest.TestCase): + def test_user_namespaced(self): + job, orch = _make_job() + orch.exec.return_value = {"h0": "alice"} + self.assertEqual(job._get_scratch_dir(), "/tmp/alice/jax") + + def test_falls_back_to_default_when_unresolved(self): + job, orch = _make_job() + orch.exec.return_value = {} + self.assertEqual(job._get_scratch_dir(), "/tmp/cvs/jax") + + def test_result_is_cached(self): + job, orch = _make_job() + orch.exec.return_value = {"h0": "bob"} + job._get_scratch_dir() + count_after_first = orch.exec.call_count + job._get_scratch_dir() + self.assertEqual(orch.exec.call_count, count_after_first) + + +class WriteMaxtextYamlTests(unittest.TestCase): + def test_yaml_content_has_run_name_steps_and_bools(self): + job, orch = _make_job() + job._write_maxtext_yaml() + written = " ".join(str(c.args[0]) for c in orch.exec.call_args_list) + self.assertIn("run_name: jaxmaxtext_llama3.3-70b", written) + self.assertIn("steps: 3", written) + # enable_checkpointing False -> rendered as lowercase yaml bool + self.assertIn("enable_checkpointing: false", written) + # scan_layers True -> lowercase bool + self.assertIn("scan_layers: true", written) + + +class StartTrainingTests(unittest.TestCase): + @patch("cvs.lib.training.jaxmaxtext.jaxmaxtext_training_lib.time.sleep") + def test_launches_per_node_backgrounded(self, _sleep): + job, orch = _make_job(hosts=["h0", "h1"]) + job.start_training() + cmds = orch.exec_cmd_list.call_args.args[0] + self.assertEqual(len(cmds), 2) + self.assertTrue(all("nohup bash" in c for c in cmds)) + _sleep.assert_called_once() + + @patch("cvs.lib.training.jaxmaxtext.jaxmaxtext_training_lib.time.sleep") + def test_captures_host_start_time(self, _sleep): + # start_training records the host-side start time (via orch.all) so the + # later dmesg scan can bound its window. + job, orch = _make_job(hosts=["h0", "h1"]) + orch.all = MagicMock() + orch.all.exec = MagicMock(return_value={"h0": "Mon Jan 2 03:04", "h1": "Mon Jan 2 03:04"}) + _wire_container_exec(orch) + job.start_training() + self.assertEqual(job.training_start_time, {"h0": "Mon Jan 2 03:04", "h1": "Mon Jan 2 03:04"}) + + +class ScanDmesgForErrorsTests(unittest.TestCase): + _LIB = "cvs.lib.training.jaxmaxtext.jaxmaxtext_training_lib" + + def _job_with_host(self, **overrides): + job, orch = _make_job(hosts=["h0", "h1"], **overrides) + orch.all = MagicMock() + orch.all.exec = MagicMock(return_value={"h0": "Mon Jan 2 03:04", "h1": "Mon Jan 2 03:04"}) + return job, orch + + @patch(f"{_LIB}.time.sleep") + @patch(f"{_LIB}._verify_dmesg_for_errors") + def test_scans_when_enabled_and_started(self, mock_verify, _sleep): + job, orch = self._job_with_host() + job.training_start_time = {"h0": "Mon Jan 2 03:00", "h1": "Mon Jan 2 03:00"} + job.scan_dmesg_for_errors() + mock_verify.assert_called_once() + args = mock_verify.call_args.args + self.assertIs(args[0], orch.all) # phdl = baremetal handle + self.assertEqual(args[1], job.training_start_time) # start of the window + + @patch(f"{_LIB}._verify_dmesg_for_errors") + def test_skipped_when_disabled(self, mock_verify): + job, _ = self._job_with_host(verify_dmesg=False) + job.training_start_time = {"h0": "t"} + job.scan_dmesg_for_errors() + mock_verify.assert_not_called() + + @patch(f"{_LIB}._verify_dmesg_for_errors") + def test_skipped_when_no_start_time(self, mock_verify): + job, _ = self._job_with_host() + job.training_start_time = None + job.scan_dmesg_for_errors() + mock_verify.assert_not_called() + + @patch(f"{_LIB}.time.sleep") + @patch(f"{_LIB}._verify_dmesg_for_errors", side_effect=RuntimeError("no passwordless sudo")) + def test_swallows_scan_failure(self, _mock_verify, _sleep): + job, _ = self._job_with_host() + job.training_start_time = {"h0": "t"} + job.scan_dmesg_for_errors() # infra failure must not propagate + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/training/jaxmaxtext/unittests/test_loss_curve.py b/cvs/lib/training/jaxmaxtext/unittests/test_loss_curve.py new file mode 100644 index 000000000..3e6abb539 --- /dev/null +++ b/cvs/lib/training/jaxmaxtext/unittests/test_loss_curve.py @@ -0,0 +1,42 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs/lib/training/jaxmaxtext/utils/loss_curve.py::render_loss_curve_png. +The renderer must never raise: it returns a path on success and None on empty +input or any failure (missing matplotlib, unwritable path). +''' + +import os +import tempfile +import unittest + +from cvs.lib.training.jaxmaxtext.utils.loss_curve import render_loss_curve_png + + +class RenderLossCurvePngTests(unittest.TestCase): + def test_empty_points_returns_none(self): + with tempfile.TemporaryDirectory() as d: + out = render_loss_curve_png([], os.path.join(d, "curve.png")) + self.assertIsNone(out) + + def test_renders_png_file(self): + points = [(0, 10.0), (10, 9.0), (20, 8.2), (30, 7.5)] + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "curve.png") + out = render_loss_curve_png(points, path, title="unit test curve") + # matplotlib is a declared dependency; when present we expect a file. + if out is not None: + self.assertEqual(out, path) + self.assertTrue(os.path.isfile(path)) + self.assertGreater(os.path.getsize(path), 0) + + def test_unwritable_path_returns_none(self): + # A path under a non-existent directory makes savefig fail; the helper + # must swallow it and return None rather than raising. + out = render_loss_curve_png([(0, 1.0), (1, 0.5)], "/nonexistent_dir_xyz/does/not/exist/curve.png") + self.assertIsNone(out) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/training/jaxmaxtext/unittests/test_maxtext_parsing.py b/cvs/lib/training/jaxmaxtext/unittests/test_maxtext_parsing.py new file mode 100644 index 000000000..7006f9b5e --- /dev/null +++ b/cvs/lib/training/jaxmaxtext/unittests/test_maxtext_parsing.py @@ -0,0 +1,195 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for the pure parsers in cvs/lib/training/jaxmaxtext/utils/maxtext_parsing.py: +step/eval extraction, aggregate metrics, convergence (row 33), validation loss +(row 34), and the loss-curve sampling + slope verdict (row 32). +''' + +import unittest + +from cvs.lib.training.jaxmaxtext.utils.maxtext_parsing import ( + compute_convergence, + evaluate_loss_decreasing, + extract_eval_metrics, + parse_training_log, + sample_loss_curve, +) + + +def _step_line(step, seconds, loss): + return ( + f"I0804 08:14:00.000000 1 metric_logger.py:196] completed step: {step}, " + f"seconds: {seconds}, TFLOP/s/device: 200.0, Tokens/s/device: 25000.0, " + f"total_weights: 393216, loss: {loss}, lm_loss: {loss}, perplexity: 10.0" + ) + + +class ExtractEvalMetricsTests(unittest.TestCase): + def test_no_eval_lines_returns_empty(self): + log = "\n".join(_step_line(i, 0.5, 9.0 - i) for i in range(3)) + self.assertEqual(extract_eval_metrics(log), []) + + def test_config_dump_lines_are_ignored(self): + # Config-dump lines mention eval + loss but are not eval results. + log = "\n".join( + [ + "I0804 08:13:33.767627 1 pyconfig.py:465] Config param target_eval_loss: 0.0", + "I0804 08:13:33.764653 1 pyconfig.py:465] Config param eval_interval: -1", + ] + ) + self.assertEqual(extract_eval_metrics(log), []) + + def test_parses_eval_loss_and_step(self): + log = "\n".join( + [ + _step_line(100, 0.5, 3.0), + "I0804 08:20:00.0 1 metric_logger.py:210] eval metrics after step: 100, eval_loss: 2.5", + ] + ) + evals = extract_eval_metrics(log) + self.assertEqual(len(evals), 1) + self.assertEqual(evals[0]["step"], 100) + self.assertAlmostEqual(evals[0]["eval_loss"], 2.5) + + def test_parses_bare_loss_on_eval_line(self): + log = "eval summary after step: 50, loss: 4.2" + evals = extract_eval_metrics(log) + self.assertEqual(len(evals), 1) + self.assertAlmostEqual(evals[0]["eval_loss"], 4.2) + + +class ComputeConvergenceTests(unittest.TestCase): + def setUp(self): + # loss falls 10, 8, 6, 4, 2 over 5 steps of 1.0s each. + self.steps = [{"step": i, "seconds": 1.0, "loss": 10.0 - 2 * i} for i in range(5)] + self.evals = [ + {"step": 2, "eval_loss": 6.5}, + {"step": 4, "eval_loss": 3.5}, + ] + + def test_disabled_when_target_non_positive(self): + self.assertEqual(compute_convergence(self.steps, self.evals, "auto", 0.0), (None, None)) + self.assertEqual(compute_convergence(self.steps, self.evals, "train_loss", -1.0), (None, None)) + + def test_train_loss_target(self): + # First step with loss <= 5.0 is step 3 (loss 4.0); cumulative time = 4.0s. + steps_to_target, time_to_target = compute_convergence(self.steps, self.evals, "train_loss", 5.0) + self.assertEqual(steps_to_target, 3) + self.assertAlmostEqual(time_to_target, 4.0) + + def test_eval_loss_target(self): + # First eval point with eval_loss <= 4.0 is step 4; cumulative time = 5.0s. + steps_to_target, time_to_target = compute_convergence(self.steps, self.evals, "eval_loss", 4.0) + self.assertEqual(steps_to_target, 4) + self.assertAlmostEqual(time_to_target, 5.0) + + def test_auto_prefers_eval_when_present(self): + steps_to_target, _ = compute_convergence(self.steps, self.evals, "auto", 4.0) + self.assertEqual(steps_to_target, 4) # eval step, not the train-loss step + + def test_auto_falls_back_to_train_loss_without_eval(self): + steps_to_target, _ = compute_convergence(self.steps, [], "auto", 5.0) + self.assertEqual(steps_to_target, 3) + + def test_target_never_reached(self): + self.assertEqual(compute_convergence(self.steps, self.evals, "train_loss", 0.5), (None, None)) + + def test_never_raises_on_empty(self): + self.assertEqual(compute_convergence([], [], "auto", 1.0), (None, None)) + + +class ParseTrainingLogEvalLossTests(unittest.TestCase): + def test_eval_loss_none_without_eval(self): + log = "\n".join(_step_line(i, 0.5, 9.0 - i) for i in range(3)) + res = parse_training_log(log, num_gpus=8) + self.assertIn("training.eval_loss", res) + self.assertIsNone(res["training.eval_loss"]) + + def test_eval_loss_reports_last_eval_point(self): + log = "\n".join( + [ + _step_line(0, 0.5, 9.0), + "eval metrics after step: 0, eval_loss: 8.0", + _step_line(1, 0.5, 8.5), + "eval metrics after step: 1, eval_loss: 7.0", + ] + ) + res = parse_training_log(log, num_gpus=8) + self.assertAlmostEqual(res["training.eval_loss"], 7.0) + + def test_empty_log_has_eval_loss_key(self): + res = parse_training_log("", num_gpus=8) + self.assertIn("training.eval_loss", res) + self.assertIsNone(res["training.eval_loss"]) + + +class SampleLossCurveTests(unittest.TestCase): + def _steps(self, n): + return [{"step": i, "seconds": 1.0, "loss": 10.0 - i * 0.1} for i in range(n)] + + def test_samples_every_n_plus_first_and_last(self): + pts = sample_loss_curve(self._steps(25), sample_every=10, milestone_steps=[]) + steps = [s for s, _ in pts] + # multiples of 10 (0,10,20) plus first (0) and last (24) + self.assertEqual(steps, [0, 10, 20, 24]) + + def test_includes_milestones(self): + steps_data = [{"step": i, "seconds": 1.0, "loss": 5.0} for i in (0, 3, 7, 12, 50)] + pts = sample_loss_curve(steps_data, sample_every=1000, milestone_steps=[7, 12]) + steps = [s for s, _ in pts] + # first(0), last(50) always; milestones 7 and 12 included; 3 excluded + self.assertEqual(steps, [0, 7, 12, 50]) + + def test_deduped_and_ordered(self): + pts = sample_loss_curve(self._steps(11), sample_every=5, milestone_steps=[0, 10]) + steps = [s for s, _ in pts] + self.assertEqual(steps, sorted(set(steps))) + self.assertEqual(steps, [0, 5, 10]) + + def test_ignores_steps_without_loss(self): + data = [{"step": 0, "seconds": 1.0}, {"step": 1, "seconds": 1.0, "loss": 3.0}] + pts = sample_loss_curve(data, sample_every=1, milestone_steps=[]) + self.assertEqual(pts, [(1, 3.0)]) + + def test_empty_input(self): + self.assertEqual(sample_loss_curve([], 10, [100]), []) + + +class EvaluateLossDecreasingTests(unittest.TestCase): + def test_decreasing(self): + pts = [(i, 10.0 - i) for i in range(6)] + decreasing, slope, _detail = evaluate_loss_decreasing(pts, max_slope=0.0) + self.assertTrue(decreasing) + self.assertAlmostEqual(slope, -1.0) + + def test_increasing(self): + pts = [(i, 1.0 + i) for i in range(6)] + decreasing, slope, _detail = evaluate_loss_decreasing(pts, max_slope=0.0) + self.assertFalse(decreasing) + self.assertGreater(slope, 0.0) + + def test_flat_is_not_decreasing_at_zero_tolerance(self): + pts = [(i, 5.0) for i in range(6)] + decreasing, slope, _detail = evaluate_loss_decreasing(pts, max_slope=0.0) + self.assertFalse(decreasing) + self.assertAlmostEqual(slope, 0.0) + + def test_too_few_points_returns_none(self): + self.assertIsNone(evaluate_loss_decreasing([(0, 5.0)], 0.0)) + self.assertIsNone(evaluate_loss_decreasing([], 0.0)) + + def test_degenerate_x_spread_returns_none(self): + # all steps identical -> zero denominator -> None (no crash) + self.assertIsNone(evaluate_loss_decreasing([(3, 5.0), (3, 4.0)], 0.0)) + + def test_noisy_but_downward(self): + pts = [(0, 10.0), (10, 9.5), (20, 9.8), (30, 8.0), (40, 7.9), (50, 6.5)] + decreasing, slope, _detail = evaluate_loss_decreasing(pts, max_slope=0.0) + self.assertTrue(decreasing) + self.assertLess(slope, 0.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/training/jaxmaxtext/unittests/test_training_config_loader.py b/cvs/lib/training/jaxmaxtext/unittests/test_training_config_loader.py new file mode 100644 index 000000000..9c0eba34b --- /dev/null +++ b/cvs/lib/training/jaxmaxtext/unittests/test_training_config_loader.py @@ -0,0 +1,125 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs/lib/training/jaxmaxtext/utils/training_config_loader.py: schema +defaults for the metric add-ons (scaling_baseline / convergence / loss_curve), +the expected_cells (sweep-name) contract, the threshold-coverage validator, and +a round-trip load of a real jaxmaxtext config file. +''' + +import unittest +import warnings +from pathlib import Path + +from cvs.lib.training.jaxmaxtext.utils.training_config_loader import ( + Convergence, + LossCurve, + ScalingBaseline, + load_training_variant, + validate_thresholds_cover_training, +) + +# Repo package root (the inner `cvs/` dir that holds `input/`): the test lives at +# cvs/lib/training/jaxmaxtext/unittests/, so parents[4] is that package root. +_PKG_ROOT = Path(__file__).resolve().parents[4] +_SINGLE_CONFIG = _PKG_ROOT / "input/config_file/training/jaxmaxtext/mi300x_jaxmaxtext_llama-3.3-70b_single.json" + + +class SchemaDefaultsTests(unittest.TestCase): + def test_scaling_baseline_defaults(self): + sb = ScalingBaseline() + self.assertEqual(sb.tokens_per_sec_total, 0.0) + self.assertEqual(sb.num_nodes, 1) + + def test_convergence_defaults(self): + c = Convergence() + self.assertEqual(c.target_metric, "auto") + self.assertEqual(c.target_value, 0.0) + + def test_loss_curve_defaults(self): + lc = LossCurve() + self.assertEqual(lc.sample_every, 10) + self.assertEqual(lc.milestone_steps, [100, 500, 1000, 5000]) + self.assertEqual(lc.max_slope, 0.0) + self.assertTrue(lc.enforce) + + +class ValidateThresholdsCoverTrainingTests(unittest.TestCase): + _GATED = { + "training.tflops_per_sec_per_gpu": {"kind": "min", "value": 1}, + "training.tokens_per_sec_per_gpu": {"kind": "min", "value": 1}, + "training.final_loss": {"kind": "max", "value": 15}, + "training.loss_decreased": {"kind": "min", "value": 1}, + } + + def test_missing_cell_raises_when_enforced(self): + with self.assertRaises(ValueError): + validate_thresholds_cover_training( + expected_cells=["CELL_A"], + thresholds={}, + enforce_thresholds=True, + ) + + def test_missing_cell_warns_when_not_enforced(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + validate_thresholds_cover_training( + expected_cells=["CELL_A"], + thresholds={}, + enforce_thresholds=False, + ) + self.assertTrue(any("does not match" in str(w.message) for w in caught)) + + def test_gated_metric_gap_raises_when_enforced(self): + # Cell present but missing the gated-metric specs -> coverage failure. + with self.assertRaises(ValueError): + validate_thresholds_cover_training( + expected_cells=["CELL_A"], + thresholds={"CELL_A": {}}, + enforce_thresholds=True, + ) + + def test_full_coverage_passes(self): + # No exception, no warning when every cell + gated metric is covered. + with warnings.catch_warnings(): + warnings.simplefilter("error") + validate_thresholds_cover_training( + expected_cells=["CELL_A"], + thresholds={"CELL_A": dict(self._GATED)}, + enforce_thresholds=True, + ) + + +class RealConfigRoundTripTests(unittest.TestCase): + def setUp(self): + if not _SINGLE_CONFIG.is_file(): + self.skipTest(f"config fixture missing: {_SINGLE_CONFIG}") + # Empty cluster dict -> {user-id} resolves to the local OS user. + self.cfg = load_training_variant(str(_SINGLE_CONFIG), {}) + + def test_metric_addon_blocks_present(self): + t = self.cfg.training + self.assertIsInstance(t.scaling_baseline, ScalingBaseline) + self.assertIsInstance(t.convergence, Convergence) + self.assertIsInstance(t.loss_curve, LossCurve) + + def test_expected_cells_are_declared_sweep_names(self): + # expected_cells() returns the declared sweep names verbatim -- the same + # keys used in the threshold file and looked up at runtime by metric(). + expected = self.cfg.expected_cells() + declared = [s.name for s in self.cfg.training.sweeps] + self.assertEqual(expected, declared) + # And every expected cell has a matching threshold entry (coverage). + for cell in expected: + self.assertIn(cell, self.cfg.thresholds) + + def test_eval_defaults_disabled(self): + # The config plumbs eval flags but leaves them disabled by default. + mc = self.cfg.training.maxtext_config + self.assertEqual(mc.get("eval_interval"), -1) + self.assertEqual(mc.get("eval_steps"), -1) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/training/jaxmaxtext/utils/__init__.py b/cvs/lib/training/jaxmaxtext/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cvs/lib/training/jaxmaxtext/utils/loss_curve.py b/cvs/lib/training/jaxmaxtext/utils/loss_curve.py new file mode 100644 index 000000000..4f6dd746a --- /dev/null +++ b/cvs/lib/training/jaxmaxtext/utils/loss_curve.py @@ -0,0 +1,67 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Loss-curve PNG rendering for the JAX MaxText suite (row 32). + +Kept separate from the pure log parser (`maxtext_parsing.py`) because it does +file I/O and lazily imports matplotlib. matplotlib is imported inside the +function with the headless ``Agg`` backend so that importing this module never +hard-requires the dependency, and a missing/broken matplotlib degrades to +``None`` rather than failing the run -- the loss-curve verdict is computed +independently of the plot. +''' + +from __future__ import annotations + +from cvs.lib import globals + +log = globals.log + + +def render_loss_curve_png(points, out_path, title=None): + """Render a training loss curve to a PNG file. + + Args: + points: ordered list of ``(step, loss)`` tuples (from + ``maxtext_parsing.sample_loss_curve``). + out_path: destination PNG path (str or Path). + title: optional plot title. + + Returns: + The ``out_path`` (as str) on success, or ``None`` if there is nothing to + plot or matplotlib is unavailable / rendering failed. Never raises. + """ + if not points: + log.info("loss curve: no points to plot, skipping PNG") + return None + + try: + import matplotlib + + matplotlib.use("Agg") # headless: no display needed on the CVS host + import matplotlib.pyplot as plt + except Exception as e: # noqa: BLE001 - plotting must never break the run + log.warning("loss curve: matplotlib unavailable, skipping PNG (%s)", e) + return None + + try: + steps = [p[0] for p in points] + losses = [p[1] for p in points] + + fig, ax = plt.subplots(figsize=(8, 4.5)) + ax.plot(steps, losses, marker="o", markersize=3, linewidth=1.5, color="#1f77b4") + ax.set_xlabel("step") + ax.set_ylabel("training loss") + ax.set_title(title or "Training Loss Curve") + ax.grid(True, linestyle="--", alpha=0.4) + fig.tight_layout() + + out_path = str(out_path) + fig.savefig(out_path, dpi=100) + plt.close(fig) + log.info("loss curve: wrote PNG %s (%d points)", out_path, len(points)) + return out_path + except Exception as e: # noqa: BLE001 + log.warning("loss curve: failed to render PNG (%s)", e) + return None diff --git a/cvs/lib/training/jaxmaxtext/utils/maxtext_parsing.py b/cvs/lib/training/jaxmaxtext/utils/maxtext_parsing.py new file mode 100644 index 000000000..362db3e29 --- /dev/null +++ b/cvs/lib/training/jaxmaxtext/utils/maxtext_parsing.py @@ -0,0 +1,391 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Pure parsers for MaxText training log output. + +MaxText logs per-step metrics in a comma-separated format: + completed step: 50, seconds: 1.234, TFLOP/s/device: 185.4, Tokens/s/device: 3456.7, total_weights: 7e9, loss: 6.543 + +During rampup phases, TFLOP/s/device and Tokens/s/device may be omitted. +Profiler steps log a different line ("completed profiler activation/deactivation step"). + +The parser extracts per-step metrics from the full training log, then computes +aggregate metrics (averages over last N steps, final loss, loss decrease check). +''' + +from __future__ import annotations + +import re + +# (short_name, unit) -- the display surface for training metrics. +# Values are looked up as "training." in the results dict. +TRAINING_METRICS = [ + ("tflops_per_sec_per_gpu", "TFLOP/s/GPU"), + ("tokens_per_sec_per_gpu", "tok/s/GPU"), + ("tokens_per_sec_total", "tok/s total"), + ("scaling_efficiency_pct", "%"), + ("step_time_seconds", "s/step"), + ("step_time_mean_ms", "ms/step"), + ("step_time_p50_ms", "ms/step"), + ("step_time_p95_ms", "ms/step"), + ("final_loss", "loss"), + ("loss_decreased", "bool"), + ("eval_loss", "loss"), + ("steps_to_target", "steps"), + ("time_to_target_seconds", "s"), +] +TRAINING_METRIC_UNITS = dict(TRAINING_METRICS) + +# The SLO contract: the subset of TRAINING_METRICS a calibrated run must assert. +# Membership = "out of range means FAILURE". Record-only by default: a NEW metric +# is record-only until its name is added here. Covers throughput (perf) plus the +# two correctness gates -- final_loss (sane loss ceiling) and loss_decreased +# (training actually reduced the loss) -- which every shipped threshold file +# already specs with max/min. +GATED_METRICS = { + "tflops_per_sec_per_gpu", + "tokens_per_sec_per_gpu", + "final_loss", + "loss_decreased", +} + +# Regex for a completed training step line. +# Example: "completed step: 50, seconds: 1.234, TFLOP/s/device: 185.4, Tokens/s/device: 3456.7, ..., loss: 6.543" +_STEP_RE = re.compile(r"completed step:\s*(\d+)") +_METRIC_RE = re.compile(r"(\S+?):\s*([\d.eE+\-]+)") + +# Eval (validation) lines. MaxText emits an eval summary when eval_interval > 0, +# but the exact wording is image-dependent and none of the current logs ran with +# eval enabled -- so we match defensively: a line that mentions "eval" and carries +# an eval-loss-like token. Both a step index and the loss are optional per line. +# NOTE: confirm this against a real eval-enabled run and tighten if needed. +_EVAL_LINE_RE = re.compile(r"\beval", re.I) +_EVAL_STEP_RE = re.compile(r"step:?\s*(\d+)", re.I) +_EVAL_LOSS_RE = re.compile(r"eval[_ ]?loss[:=]?\s*([\d.eE+\-]+)", re.I) +# Fallback: a bare "loss: X" on an eval line when the token is not prefixed with "eval". +_LOSS_RE = re.compile(r"\bloss[:=]?\s*([\d.eE+\-]+)", re.I) +# Config-dump lines ("Config param target_eval_loss: 0.0") mention eval + loss but +# are not eval results -- exclude them so they never register as eval points. +_CONFIG_LINE_RE = re.compile(r"config param|pyconfig", re.I) + + +def compute_scaling_efficiency( + tokens_per_sec_total, + num_nodes, + baseline_tokens_per_sec_total, + baseline_num_nodes=1, +): + """Scaling efficiency % for a training run. + + efficiency % = throughput_N / ((N / ref_N) * throughput_ref) * 100 + + where throughput_N is this run's total tokens/sec on `num_nodes` nodes and + throughput_ref is the reference (typically 1-node) total tokens/sec measured + on `baseline_num_nodes` nodes. 100% means perfectly linear scaling; lower + means communication/straggler overhead is eating into the added nodes. + + Returns None (record-only) when any input is missing or non-positive so an + uncalibrated baseline never produces a misleading number or a crash. + """ + if not tokens_per_sec_total or not baseline_tokens_per_sec_total: + return None + if not num_nodes or not baseline_num_nodes: + return None + ideal = (num_nodes / baseline_num_nodes) * baseline_tokens_per_sec_total + if ideal <= 0: + return None + return tokens_per_sec_total / ideal * 100.0 + + +def compute_convergence(step_metrics, eval_metrics, target_metric="auto", target_value=0.0): + """Steps and wall-clock to reach a target loss (row 33). + + target_metric: + - "eval_loss": converge on validation loss (eval_metrics points) + - "train_loss": converge on per-step training loss (step_metrics) + - "auto": use eval_metrics when present, else training loss + + A `target_value <= 0` disables the metric and returns (None, None) so an + uncalibrated target never gates or misleads. + + Returns (steps_to_target, time_to_target_seconds), where the time is the + cumulative sum of per-step `seconds` up to and including the target step. + This is training compute time (it includes the step-0 compile spike and + excludes eval/checkpoint overhead), not true wall-clock. Returns (None, None) + when disabled or the target is never reached. Never raises. + """ + if not target_value or target_value <= 0: + return (None, None) + + use_eval = target_metric == "eval_loss" or (target_metric == "auto" and bool(eval_metrics)) + + # Cumulative training seconds indexed by step, from the per-step lines. + cum = {} + running = 0.0 + for s in step_metrics or []: + sec = s.get("seconds") + if isinstance(sec, (int, float)): + running += sec + step = s.get("step") + if step is not None: + cum[step] = running + + target_step = None + if use_eval: + for e in eval_metrics or []: + loss = e.get("eval_loss") + if loss is not None and e.get("step") is not None and loss <= target_value: + target_step = e.get("step") + break + else: + for s in step_metrics or []: + loss = s.get("loss") + if loss is not None and s.get("step") is not None and loss <= target_value: + target_step = s.get("step") + break + + if target_step is None: + return (None, None) + + time_to_target = cum.get(target_step) + if time_to_target is None and cum: + # An eval step may not line up with a training-step key; take the + # cumulative time at the latest training step at or before the target. + prior = [t for st, t in cum.items() if st <= target_step] + time_to_target = max(prior) if prior else None + + return (target_step, time_to_target) + + +def sample_loss_curve(step_metrics, sample_every=10, milestone_steps=None): + """Downsample per-step training loss for the loss curve (row 32). + + Keeps a point when its step is a multiple of `sample_every`, is one of the + `milestone_steps` (e.g. 100/500/1k/5k), or is the first/last recorded step. + The first/last inclusion keeps short runs (fewer than `sample_every` steps) + from producing an empty curve. + + Returns an ordered, de-duplicated list of ``(step, loss)`` tuples. Only steps + that carry a numeric `loss` are considered. Never raises. + """ + milestones = set(milestone_steps or []) + every = sample_every if sample_every and sample_every > 0 else 1 + + loss_steps = [ + s for s in (step_metrics or []) if s.get("step") is not None and isinstance(s.get("loss"), (int, float)) + ] + if not loss_steps: + return [] + + first_step = loss_steps[0]["step"] + last_step = loss_steps[-1]["step"] + + picked = {} + for s in loss_steps: + step = s["step"] + if step % every == 0 or step in milestones or step in (first_step, last_step): + picked[step] = s["loss"] + + return [(step, picked[step]) for step in sorted(picked)] + + +def evaluate_loss_decreasing(points, max_slope=0.0): + """Decide whether a sampled loss curve trends downward (row 32). + + Fits a least-squares line to ``points`` (a list of ``(step, loss)``) and + treats the run as decreasing when the slope is below `max_slope` (default + 0.0, i.e. strictly negative). Uses a dependency-free closed form: + + slope = (n*Sxy - Sx*Sy) / (n*Sxx - Sx^2) + + Returns ``(decreasing: bool, slope: float, detail: str)`` or ``None`` when + there are fewer than 2 points (verdict not computable). Never raises; a + degenerate x-spread (all steps equal) also returns None. + """ + if not points or len(points) < 2: + return None + + n = len(points) + sx = sum(p[0] for p in points) + sy = sum(p[1] for p in points) + sxx = sum(p[0] * p[0] for p in points) + sxy = sum(p[0] * p[1] for p in points) + + denom = n * sxx - sx * sx + if denom == 0: + return None + + slope = (n * sxy - sx * sy) / denom + decreasing = slope < max_slope + detail = ( + f"loss slope {slope:.6g}/step over {n} points " + f"(first={points[0][1]:.4f}@{points[0][0]}, last={points[-1][1]:.4f}@{points[-1][0]}); " + f"{'decreasing' if decreasing else 'NOT decreasing'} (max_slope={max_slope})" + ) + return (decreasing, slope, detail) + + +def _percentile(values, q): + """Linear-interpolated percentile (q in [0, 100]) over a list of numbers. + + Returns None for an empty list. Matches numpy's default ('linear') + interpolation so p50 equals the median for even-length samples. + """ + if not values: + return None + xs = sorted(values) + if len(xs) == 1: + return xs[0] + rank = (q / 100.0) * (len(xs) - 1) + lo = int(rank) + hi = min(lo + 1, len(xs) - 1) + frac = rank - lo + return xs[lo] + (xs[hi] - xs[lo]) * frac + + +def _parse_step_line(line): + """Parse a single 'completed step: N, ...' line into a dict. + + Returns None for non-step lines (profiler steps, rampup, etc.). + """ + step_m = _STEP_RE.search(line) + if not step_m: + return None + if "profiler" in line.lower(): + return None + step = int(step_m.group(1)) + fields = {"step": step} + # Parse all key: value pairs from the comma-separated line. + # The regex grabs "key: numeric_value" pairs. + for m in _METRIC_RE.finditer(line): + key, val_str = m.group(1), m.group(2) + try: + val = float(val_str) + except ValueError: + continue + if key == "step": + continue + fields[key] = val + return fields + + +def extract_step_metrics(log_text): + """Extract per-step metric dicts from a MaxText training log. + + Returns a list of dicts, each with at least 'step' and optionally: + 'seconds', 'TFLOP/s/device', 'Tokens/s/device', 'loss', 'total_weights'. + """ + steps = [] + for line in log_text.splitlines(): + parsed = _parse_step_line(line) + if parsed is not None: + steps.append(parsed) + return steps + + +def extract_eval_metrics(log_text): + """Extract validation-loss points from a MaxText training log (row 34). + + Returns a list of ``{"step": int|None, "eval_loss": float}`` dicts, one per + eval summary line. Defensive by design: MaxText only emits eval output when + ``eval_interval > 0`` and the exact wording is image-dependent, so we accept + any non-config line that mentions "eval" and carries a loss token. Config + dumps (e.g. "Config param target_eval_loss: 0.0") are excluded. Returns + ``[]`` when eval was not enabled or the format is unrecognized. + + NOTE: validate the matched format against a real eval-enabled run and + tighten the regex if MaxText's eval line differs from what is assumed here. + """ + evals = [] + for line in log_text.splitlines(): + if not _EVAL_LINE_RE.search(line): + continue + if _CONFIG_LINE_RE.search(line): + continue + m = _EVAL_LOSS_RE.search(line) or _LOSS_RE.search(line) + if not m: + continue + try: + loss = float(m.group(1)) + except ValueError: + continue + step_m = _EVAL_STEP_RE.search(line) + step = int(step_m.group(1)) if step_m else None + evals.append({"step": step, "eval_loss": loss}) + return evals + + +def parse_training_log(log_text, num_gpus, avg_last_n=10): + """Parse MaxText training log into namespaced training.* metrics dict. + + Averages TFLOP/s/device and Tokens/s/device over the last `avg_last_n` + steps (matching the MAD benchmark parser behavior). Computes total + tokens/sec, final loss, and whether loss decreased from first to last step. + + Returns: {"training.": value, ...} + """ + steps = extract_step_metrics(log_text) + if not steps: + return { + "training.tflops_per_sec_per_gpu": None, + "training.tokens_per_sec_per_gpu": None, + "training.tokens_per_sec_total": None, + "training.step_time_seconds": None, + "training.step_time_mean_ms": None, + "training.step_time_p50_ms": None, + "training.step_time_p95_ms": None, + "training.final_loss": None, + "training.loss_decreased": None, + "training.eval_loss": None, + } + + # Filter to steps that have perf metrics (skip rampup steps without them). + perf_steps = [s for s in steps if "TFLOP/s/device" in s or "Tokens/s/device" in s] + tail = perf_steps[-avg_last_n:] if perf_steps else [] + + def _avg(key): + vals = [s[key] for s in tail if key in s] + return sum(vals) / len(vals) if vals else None + + tflops = _avg("TFLOP/s/device") + tokens_per_gpu = _avg("Tokens/s/device") + step_time = _avg("seconds") + + tokens_total = tokens_per_gpu * num_gpus if tokens_per_gpu is not None else None + + # Step-time distribution (ms) over steady-state steps. perf_steps already + # excludes rampup/profiler steps, whose compile-heavy outliers would inflate + # the tail and mask real jitter. Percentiles use the full steady-state + # window (not just `tail`) so p95 has enough samples to be meaningful. + step_seconds = [s["seconds"] for s in perf_steps if "seconds" in s] + step_time_mean_ms = (sum(step_seconds) / len(step_seconds) * 1000.0) if step_seconds else None + p50 = _percentile(step_seconds, 50) + p95 = _percentile(step_seconds, 95) + step_time_p50_ms = p50 * 1000.0 if p50 is not None else None + step_time_p95_ms = p95 * 1000.0 if p95 is not None else None + + # Loss metrics from all steps that have a loss value. + loss_steps = [s for s in steps if "loss" in s] + final_loss = loss_steps[-1]["loss"] if loss_steps else None + first_loss = loss_steps[0]["loss"] if loss_steps else None + loss_decreased = None + if first_loss is not None and final_loss is not None: + loss_decreased = 1 if final_loss < first_loss else 0 + + # Validation loss (row 34): last eval point, or None when eval was disabled. + eval_metrics = extract_eval_metrics(log_text) + eval_loss = eval_metrics[-1]["eval_loss"] if eval_metrics else None + + return { + "training.tflops_per_sec_per_gpu": tflops, + "training.tokens_per_sec_per_gpu": tokens_per_gpu, + "training.tokens_per_sec_total": tokens_total, + "training.step_time_seconds": step_time, + "training.step_time_mean_ms": step_time_mean_ms, + "training.step_time_p50_ms": step_time_p50_ms, + "training.step_time_p95_ms": step_time_p95_ms, + "training.final_loss": final_loss, + "training.loss_decreased": loss_decreased, + "training.eval_loss": eval_loss, + } diff --git a/cvs/lib/training/jaxmaxtext/utils/training_config_loader.py b/cvs/lib/training/jaxmaxtext/utils/training_config_loader.py new file mode 100644 index 000000000..dcb486b7f --- /dev/null +++ b/cvs/lib/training/jaxmaxtext/utils/training_config_loader.py @@ -0,0 +1,264 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Training-specific config schema for the jaxmaxtext suite. + +The framework-agnostic machinery (paths/model/container schema, the 3-pass +placeholder substitution, the `enforce_thresholds` gate, and the +`substitute_config` file-read helper) lives in `cvs.lib.utils.config_loader`. +This module holds the training half: the MaxText config, tokenizer, NCCL, +JAX distributed settings, RDMA lib, and `TrainingVariantConfig(BaseVariantConfig)`. + +A training suite does not sweep cells the way inference does (no NxM matrix of +ISL/OSL/concurrency). Instead each declared `sweep` is one full training run and +its `name` IS the threshold-file key (also the key `metric()` looks up at +runtime). `expected_cells()` therefore returns the declared sweep names, and the +coverage check validates the threshold file against those names directly. +''' + +from __future__ import annotations + +import warnings +from typing import Any, Dict, List, Literal + +from pydantic import field_validator + +from cvs.lib.utils.config_loader import BaseVariantConfig, _Allow, _Forbid, substitute_config +from cvs.lib.training.jaxmaxtext.utils.maxtext_parsing import GATED_METRICS + + +class Tokenizer(_Forbid): + hf_model_id: str + tokenizer_path: str + + +class NcclConfig(_Allow): + ib_hca_list: str = "" + ib_hca: str = "" + socket_ifname: str = "" + gloo_socket_ifname: str = "" + ib_tc: str = "41" + ib_sl: str = "0" + ib_gid_index: str = "3" + + @field_validator("ib_hca_list", "ib_hca", "socket_ifname", "gloo_socket_ifname") + @classmethod + def _reject_changeme(cls, v, info): + """Hard-exit when a cluster-specific RDMA/NIC field is left as ''. + + These device/interface names are cluster-specific and shipped as + '' placeholders (see the sibling _example_* values). Running a + distributed job with them unresolved would silently use the wrong + NIC/RDMA devices, so fail loudly at config load instead. + """ + if isinstance(v, str) and "" in v.lower(): + raise ValueError( + f"nccl.{info.field_name} is still ''. Set your cluster's RDMA/NIC " + "device/interface (see the sibling _example_* value) before running distributed training." + ) + return v + + +class JaxDistributed(_Forbid): + coordinator_ip: str = "auto" + coordinator_port: str = "12346" + initialization_timeout_seconds: str = "1800" + heartbeat_timeout_seconds: str = "900" + + +class RdmaLib(_Allow): + host_source_file: str = "" + container_mount_file: str = "" + container_dest_file: str = "" + + +class ScalingBaseline(_Allow): + """Reference (typically 1-node) throughput for scaling-efficiency %. + + `tokens_per_sec_total` is the TOTAL tokens/sec measured on a prior run of + `num_nodes` nodes (source it from a previous single-node run log). Scaling + efficiency % = throughput_N / ((N / num_nodes) * tokens_per_sec_total) * 100. + + Leave `tokens_per_sec_total` at 0.0 to disable the metric (it then reports + record-only as None instead of gating on an uncalibrated baseline). + """ + + tokens_per_sec_total: float = 0.0 + num_nodes: int = 1 + + +class Convergence(_Allow): + """Target for convergence / time-to-target-accuracy (row 33). + + `target_metric` selects the loss series to converge on: + - "eval_loss" : validation loss (requires eval enabled + parseable) + - "train_loss" : per-step training loss + - "auto" : eval loss when eval points exist, else training loss + + `target_value` is the loss threshold to reach; <= 0 disables the metric + (steps_to_target / time_to_target_seconds report record-only as None). + """ + + target_metric: Literal["auto", "train_loss", "eval_loss"] = "auto" + target_value: float = 0.0 + + +class LossCurve(_Allow): + """Loss-curve (row 32) sampling + pass/fail settings. + + `sample_every` and `milestone_steps` control which per-step losses are kept + for the plotted/asserted curve (keeps short runs non-empty). The verdict is + the least-squares slope of the sampled curve: the run passes when + `slope < max_slope` (default 0.0 = strictly decreasing). `enforce` gates the + test (fail on a non-decreasing curve); set False for record-only. + """ + + sample_every: int = 10 + milestone_steps: List[int] = [100, 500, 1000, 5000] + max_slope: float = 0.0 + enforce: bool = True + + +class Sweep(_Allow): + """One sweep entry = one full training run with per-run maxtext overrides. + + `name` is the canonical cell key (also the threshold-file key), e.g. + "NNODES=2,STEPS=30,PRECISION=BF16,BATCH=3,GBS=48,SEQLEN=8192". Only the + parameters that actually vary need a `maxtext_overrides` entry (for now just + precision, e.g. FP8 sets `quantization`); everything else falls back to the + base `maxtext_config`. + """ + + name: str + maxtext_overrides: Dict[str, Any] = {} + + +class TrainingConfig(_Allow): + distributed: bool = True + gpus_per_node: int = 8 # do not assume a uniform topology; override per cluster + # Scan host dmesg (all nodes) for GPU/HW/kernel faults over the training + # window. Set false on clusters without passwordless sudo for `dmesg`. + verify_dmesg: bool = True + steps: int = 30 + enable_checkpointing: bool = False + # MaxText moved the train entrypoint across versions; list candidates and the + # job picks whichever exists in the running container (first match wins). + # v26.4+: .../src/maxtext/trainers/pre_train/train.py + # v26.3 and earlier: .../src/MaxText/train.py + train_script_paths: List[str] = [ + "/workspace/maxtext/src/maxtext/trainers/pre_train/train.py", + "/workspace/maxtext/src/MaxText/train.py", + ] + # Deprecated single-path form; kept for backward compatibility and used as a + # final fallback candidate when train_script_paths is empty. + train_script: str = "/workspace/maxtext/src/MaxText/train.py" + maxtext_config: Dict[str, Any] = {} + tokenizer: Tokenizer + nic_type: str = "thor2" + rdma_lib: RdmaLib = RdmaLib() + env_vars: Dict[str, str] = {} + xla_flags: Dict[str, str] = {} + # {name: regex} error signatures scanned in the training log during polling. + # Empty -> the driver falls back to its built-in default set. Lets users + # add/remove signatures per config without touching code. + error_patterns: Dict[str, str] = {} + nccl: NcclConfig = NcclConfig() + jax_distributed: JaxDistributed = JaxDistributed() + scaling_baseline: ScalingBaseline = ScalingBaseline() + convergence: Convergence = Convergence() + loss_curve: LossCurve = LossCurve() + sweeps: List[Sweep] = [] + enabled_sweep_list: List[str] = [] + + +def validate_thresholds_cover_training( + *, + expected_cells, + thresholds, + enforce_thresholds: bool, + gated_metrics=None, +) -> None: + """Shared training threshold/cell coverage check.""" + expected = set(expected_cells) + # Skip "_"-prefixed metadata keys (e.g. "_comment") so they are not mistaken + # for a threshold cell that matches no training sweep. + present = {k for k in thresholds.keys() if not str(k).startswith("_")} + missing = sorted(expected - present) + extra = sorted(present - expected) + problems = [] + if missing: + problems.append(f"training cells with no threshold entry: {missing}") + if extra: + problems.append(f"threshold keys matching no training cell (typo?): {extra}") + gated = gated_metrics if gated_metrics is not None else GATED_METRICS + gated_keys = [f"training.{m}" for m in sorted(gated)] + gated_gaps = {} + for cell in sorted(expected & present): + specs = thresholds.get(cell) or {} + absent = [k for k in gated_keys if k not in specs] + if absent: + gated_gaps[cell] = absent + if gated_gaps: + problems.append(f"cells missing gated-metric specs: {gated_gaps}") + if problems: + msg = "threshold.json does not match the training config; " + "; ".join(problems) + if enforce_thresholds: + raise ValueError(msg) + warnings.warn(f"{msg} (enforce_thresholds=false -> record-only)", stacklevel=3) + + +class TrainingVariantConfig(BaseVariantConfig): + framework: Literal["jaxmaxtext"] + gpu_arch: str + training: TrainingConfig + + def expected_cells(self): + """Threshold cell keys this config expects: one per declared sweep. + + The sweep `name` IS the threshold-file key and the key `metric()` looks + up at runtime (see cvs/tests/training/jaxmaxtext/_common.py::metric), so + coverage is checked against the declared sweep names directly -- not a + synthesized key. `enabled_sweep_list` only selects which of these + actually run; the threshold file still carries an entry per declared + sweep. A config with no sweeps degrades to a single implicit "default" + cell. + """ + names = [s.name for s in self.training.sweeps] + return names or ["default"] + + def enabled_sweeps(self): + """Return the Sweep objects selected to run. + + `enabled_sweep_list` (if non-empty) selects a subset by name; otherwise + every declared sweep runs. A config with no `sweeps` degrades to a single + implicit sweep named "default" (its threshold cell, if any, is keyed + "default"), so the suite still runs unparametrized. + """ + sweeps = self.training.sweeps + if not sweeps: + return [Sweep(name="default")] + by_name = {s.name: s for s in sweeps} + names = self.training.enabled_sweep_list or [s.name for s in sweeps] + selected = [] + for n in names: + if n in by_name: + selected.append(by_name[n]) + else: + warnings.warn(f"enabled_sweep_list references unknown sweep '{n}'", stacklevel=2) + return selected + + +# ---------- public API (training) ---------- + + +def load_training_variant(config_path, cluster_dict): + """Load and validate a jaxmaxtext variant config + its sibling threshold file. + + Delegates the file read + placeholder substitution + threshold discovery to + the generic `substitute_config`, then attaches the thresholds and builds the + typed `TrainingVariantConfig`. + """ + raw, thresholds = substitute_config(config_path, cluster_dict) + raw["thresholds"] = thresholds + return TrainingVariantConfig(**raw) diff --git a/cvs/lib/training/megatron/__init__.py b/cvs/lib/training/megatron/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cvs/lib/training/megatron/megatron_lib.py b/cvs/lib/training/megatron/megatron_lib.py new file mode 100644 index 000000000..49b68bc79 --- /dev/null +++ b/cvs/lib/training/megatron/megatron_lib.py @@ -0,0 +1,889 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. +The year included in the foregoing notice is the year of creation of the work. +All code contained here is Property of Advanced Micro Devices, Inc. +''' + +import os +import re +import shlex +import time + +from cvs.lib import globals +from cvs.lib.utils_lib import * +from cvs.lib.verify_lib import * +from cvs.lib import linux_utils +from cvs.lib.training.megatron.utils.model_registry import TRAINING_SCRIPTS, PRECISION_FLAGS, BATCH_SIZE_FLAGS + +log = globals.log + + +training_err_dict = { + 'NCCL ERROR': 'NCCL ERROR|NCCL timeout|ncclRemoteError: A call failed possibly due to a network error|NCCL error:', + 'GPU HW ERROR': 'HW Exception by GPU|GPU Hang|Uncorrectable error|GPU Reset', + 'torch': 'torch.distributed.elastic.multiprocessing.errors', +} + +err_counters_pattern = 'err|retransmit|drop|discard|naks|invalid|oflow|out_of_buffer|reset|fail' + + +# Ordered fallback chains for parsing Megatron-LM training output. +# Each chain is tried in order; first non-empty match wins. Seeded with +# [new, old] so newer Megatron output (e.g. `throughput per GPU +# (TFLOP/s/GPU): N`) is preferred but the original format +# (`throughput per GPU: N`) still parses on older builds. +TRAINING_RESULT_PATTERNS = { + 'throughput_per_gpu': [r'throughput per GPU:\s+([0-9\.]+)'], + 'tokens_per_gpu': [r'tokens/GPU/s:\s+([0-9\.]+)'], + 'mem_usage': [r'mem usages:\s+([0-9\.]+)'], + 'elapsed_time_per_iteration': [r'elapsed time per iteration:\s+([0-9\.]+)'], +} + +# Per-iteration patterns for models whose training script does not append summary lines. +# These match the inline iteration format: `throughput per GPU (TFLOP/s/GPU): X`. +# Used as a fallback in get_training_results_dict when summary-line parsing returns empty. +TRAINING_ITERATION_PATTERNS = { + 'throughput_per_gpu': r'throughput per GPU\s*\([^)]*\)\s*:\s*([0-9.eE+\-]+)', + 'tokens_per_gpu': r'tokens/GPU/s\s*:\s*([0-9.eE+\-]+)', + 'mem_usage': r'mem usages:\s*([0-9.eE+\-]+)', + 'elapsed_time_per_iteration': r'elapsed time per iteration\s*\([^)]*\)\s*:\s*([0-9.eE+\-]+)', +} + + +def _parse_mean_from_iterations(log_text, pattern, skip_warmup=True): + """Parse per-iteration metric values from full log text and return their mean. + + Extracts all values matching `pattern` (one capture group) from `log_text`, + optionally skips the first match (iteration 1 warmup which is artificially + slow due to JIT compilation), and returns the mean as a string. + + Args: + log_text: Full training log text. + pattern: Regex with one capture group for the numeric metric value. + skip_warmup: If True and more than one value found, drop the first match. + + Returns: + Mean value as a string, or None if no values were found. + """ + matches = re.findall(pattern, log_text, re.I) + if not matches: + return None + values = [float(m) for m in matches] + if skip_warmup and len(values) > 1: + values = values[1:] + return str(sum(values) / len(values)) + + +TRAINING_PROGRESS_PATTERNS = [ + r'throughput per GPU(?:\s*\([^)]*\))?\s*:|tokens\/GPU\/s\s+[0-9]+', + r'throughput per GPU:|tokens\/GPU\/s\s+[0-9]+', +] + +TRAINING_NAN_PATTERNS = [ + r'throughput per GPU(?:\s*\([^)]*\))?\s*:\s+(?:NaN|Inf)', + r'throughput per GPU:\s+(?:NaN|Inf)', + r'tokens\/GPU\/s:\s+(?:NaN|Inf)', + r'mem usages:\s+(?:NaN|Inf)', +] + + +def _parse_training_results(output, full_log=None): + """Extract metric values from training-log text using ordered fallback chains. + + Primary: tries each pattern in TRAINING_RESULT_PATTERNS against `output` + (typically the last N lines containing shell-script-appended summary lines). + + Fallback: when a metric is still empty and `full_log` is provided, searches + the full log for per-iteration values using TRAINING_ITERATION_PATTERNS, + skips the warmup iteration, and stores the mean as a single-element list. + Used for models whose training script does not append summary lines. + + Args: + output (str): Tail of the training log (summary lines). + full_log (str|None): Full training log text for per-iteration fallback. + + Returns: + dict: {metric_name: list[str]} for every key in TRAINING_RESULT_PATTERNS. + """ + out = {} + for metric, patterns in TRAINING_RESULT_PATTERNS.items(): + out[metric] = [] + for pat in patterns: + matches = re.findall(pat, output, re.I) + if matches: + out[metric] = matches + break + if not out[metric] and full_log is not None: + pattern = TRAINING_ITERATION_PATTERNS.get(metric) + if pattern: + mean = _parse_mean_from_iterations(full_log, pattern, skip_warmup=True) + if mean: + out[metric] = [mean] + log.info('per-iteration fallback: %s = %s', metric, mean) + return out + + +def _is_training_complete(output, total_iters): + """Return True only when the final iteration line is present. + + Megatron emits `iteration /` on every step (both old and new + build formats), so the last iteration `/` marks true + completion. Checking for any per-iteration throughput line instead would + fire after step 1 and cut slow runs short.""" + n = int(total_iters) + return bool(re.search(rf'iteration\s+{n}\s*/\s*{n}\b', output)) + + +def _has_nan_inf_results(output): + """Return True if the training-log text shows a NaN/Inf result line + matching any pattern in TRAINING_NAN_PATTERNS.""" + return any(re.search(p, output, re.I) for p in TRAINING_NAN_PATTERNS) + + +# Library for building Megatron training jobs .. + + +def detect_rocm_path(orch, config_rocm_path): + """ + Detect the ROCm installation path inside the container, supporting both + old (/opt/rocm) and new (/opt/rocm/core-X.Y) layouts. + + Args: + orch: Orchestrator handle (ContainerOrchestrator). exec() runs inside + the container so the detected path reflects what the training + job actually sees at runtime. + config_rocm_path (str): Configured ROCm path from config file + (empty string or '' for auto-detect). + + Returns: + str: Detected ROCm path. + """ + if config_rocm_path and config_rocm_path != '': + log.info(f'Using configured ROCm path: {config_rocm_path}') + return config_rocm_path + + log.info('Auto-detecting ROCm path inside container...') + + # Try new ROCm layout first (/opt/rocm/core-X.Y) + out_dict = orch.exec('ls -d /opt/rocm/core-* 2>/dev/null | sort -V | tail -1') + for node, output in out_dict.items(): + if output and '/opt/rocm/core-' in output: + rocm_path = output.strip() + validate_dict = orch.exec( + f'test -d {rocm_path}/lib && ls {rocm_path}/lib/libamdhip64.so* 2>/dev/null | head -1' + ) + for _, lib_output in validate_dict.items(): + if lib_output.strip() and 'libamdhip64.so' in lib_output: + log.info(f'Detected ROCm path (new layout): {rocm_path}') + return rocm_path + + # Fall back to legacy /opt/rocm + out_dict = orch.exec('test -d /opt/rocm/lib && ls /opt/rocm/lib/libamdhip64.so* 2>/dev/null | head -1') + for node, output in out_dict.items(): + if output.strip() and 'libamdhip64.so' in output: + log.info('Detected ROCm path (legacy layout): /opt/rocm') + return '/opt/rocm' + + log.warning('Could not detect ROCm path, defaulting to /opt/rocm') + return '/opt/rocm' + + +class MegatronTrainingJob: + """ + Orchestrates a Megatron-LM Llama training job across one or more nodes. + + Responsibilities: + - Normalize training configuration and model parameters (with sensible defaults). + - Prepare per-node wrapper scripts and environment variables for distributed runs. + - Optionally collect pre/post network (RDMA/ethtool) stats for validation. + - Launch the job (single-node or distributed) inside a specified container. + - Poll logs for completion and errors; extract performance metrics from logs. + - Verify training results against expected thresholds and system health checks. + + Assumptions: + - phdl provides remote execution utilities across nodes: + - phdl.host_list (list of nodes) + - phdl.exec(cmd: str) -> Dict[node, str] or str, depending on implementation + - phdl.exec_cmd_list(cmd_list: List[str]) -> Dict[node, str] + - Docker container is pre-deployed and accessible on each node. + - Training scripts exist under {megatron_root}/examples/llama/ (default + `/workspace/Megatron-LM/`; configurable via the `megatron_root` and + `training_scripts` keys in the training config). + - External helpers referenced in the methods are available in scope: + - linux_utils.get_rdma_stats_dict, linux_utils.get_nic_ethtool_stats_dict + - json_to_dict, fail_test, verify_dmesg_for_errors, log, training_err_dict + - err_counters_pattern + """ + + def __init__( + self, + orch, + variant_config, + hf_token, + micro_batch_size, + global_batch_size, + precision=None, + distributed_training=False, + tune_model_params=False, + scripts_dir=None, + run_label=None, + ): + """ + Initialize job configuration from a MegatronVariantConfig + sweep-level params. + + Args: + orch: Orchestrator handle for container and host command execution. + variant_config: MegatronVariantConfig holding container, config, model_params, sweep. + hf_token: Hugging Face token passed to the job environment. + batch_size: Global batch size for this sweep cell (overrides model_params). + micro_batch_size: Micro batch size for this sweep cell (overrides model_params). + precision: Optional precision override for this sweep cell. When None, falls + back to model_params.precision (default: TE_FP8). + distributed_training: True for multi-node distributed runs. + tune_model_params: If True, adjust batch size based on cluster size. + scripts_dir: Optional override for the per-node wrapper scripts folder. + """ + + self.orch = orch + self.model_name = variant_config.model_params["model_name"] + self.hf_token = hf_token + self.tune_model_params = tune_model_params + + self.job_cmd = '' + self.job_cmd_list = [] + self.training_results_dict = {} + self.local_tokenizer_path = None + self.rdma_stats_dict_before = {} + self.ethtool_stats_dict_before = {} + self.rdma_stats_dict_after = {} + self.ethtool_stats_dict_after = {} + self.training_start_time = self.orch.all.exec('date') + self.training_end_time = None + + # Training config — copy to avoid mutating the variant_config dict + self.home_dir = os.path.expanduser("~") + tdict = dict(variant_config.config) + tdict.setdefault('training_iterations', 10) + tdict.setdefault('nnodes', '1') + tdict.setdefault('nic_type', 'thor2') + tdict.setdefault('hca_id_pattern', 'bnxt_|rocep') + tdict.setdefault('nccl_ib_hca_list', 'bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re6,bnxt_re7') + tdict.setdefault('nccl_ib_hca', 'bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re6,bnxt_re7') + tdict.setdefault('nccl_socket_ifname', 'ensf1np1') + tdict.setdefault('gloo_socket_ifname', 'ensf1np1') + tdict.setdefault('nccl_ib_gid_index', '3') + tdict.setdefault('nccl_debug', 'ERROR') + tdict.setdefault('data_cache_dir', f'{self.home_dir}/cache') + tdict.setdefault('log_dir', f'{self.home_dir}/LOGS') + tdict.setdefault('scripts_dir', f'{self.home_dir}/SCRIPTS') + tdict.setdefault('master_address', '127.0.0.1') + tdict.setdefault('verify_network_errors', 'False') + tdict.setdefault('rocm_dir', '') + tdict.setdefault('megatron_root', '/workspace/Megatron-LM') + tdict.setdefault('training_scripts', TRAINING_SCRIPTS) + + self.container_image = orch.container_config["image"] + self.distributed_training = distributed_training + self.iterations = int(tdict['training_iterations']) + self.nnodes = str(tdict['nnodes']) + if int(self.nnodes) != len(orch.hosts): + log.warning( + f"config nnodes={self.nnodes} does not match cluster host count={len(orch.hosts)}; " + f"using cluster host count" + ) + self.nnodes = str(len(orch.hosts)) + self.nic_type = tdict['nic_type'] + self.hca_id_pattern = tdict['hca_id_pattern'] + self.nccl_ib_hca_list = tdict['nccl_ib_hca_list'] + self.nccl_ib_hca = tdict['nccl_ib_hca'] + self.nccl_socket_ifname = tdict['nccl_socket_ifname'] + self.gloo_socket_ifname = tdict['gloo_socket_ifname'] + self.nccl_ib_gid_index = tdict['nccl_ib_gid_index'] + self.nccl_debug = tdict['nccl_debug'] + self.data_cache_dir = tdict['data_cache_dir'] + self.log_dir = tdict['log_dir'] + self.scripts_dir = scripts_dir if scripts_dir is not None else tdict['scripts_dir'] + self.master_address = tdict['master_address'] + self.verify_network_errors = tdict['verify_network_errors'] + self.rocm_path = detect_rocm_path(self.orch, tdict['rocm_dir']) + self.megatron_root = tdict['megatron_root'] + self.training_scripts = tdict['training_scripts'] + + # Model params — merge variant_config.model_params with sweep-level overrides + pdict = dict(variant_config.model_params) + pdict['micro_batch_size'] = micro_batch_size + pdict['global_batch_size'] = global_batch_size + if precision: + pdict['precision'] = precision + pdict.pop('model_name', None) + # Per-combo log dir so sweep combos don't overwrite each other's + # training.log. Label is the sweep combination name (run_label); falls + # back to a model_name/mbs/gbs/precision tag when none is provided. + raw_label = run_label or f"{self.model_name}_mbs{micro_batch_size}_gbs{global_batch_size}_{pdict['precision']}" + self.run_label = re.sub(r'[^A-Za-z0-9._-]', '_', str(raw_label)) + self.combo_log_dir = f'{self.log_dir}/megatron-logs/{self.run_label}' + + pdict.setdefault('tokenizer_model', 'meta-llama/Llama-3.1-70B') + pdict.setdefault('model_size', 70) + pdict.setdefault('sequence_length', '8192') + pdict.setdefault('micro_batch_size', '2') + pdict.setdefault('global_batch_size', '128') + pdict.setdefault('fsdp', '0') + pdict.setdefault('tensor_parallelism', '1') + pdict.setdefault('pipeline_parallelism', '1') + pdict.setdefault('recompute', '0') + pdict.setdefault('precision', 'TE_FP8') + + self.tokenizer_model = pdict['tokenizer_model'] + self.model_size = pdict['model_size'] + self.sequence_length = pdict['sequence_length'] + self.micro_batch_size = pdict['micro_batch_size'] + self.global_batch_size = pdict['global_batch_size'] + self.fsdp = pdict['fsdp'] + self.tensor_parallelism = pdict['tensor_parallelism'] + self.pipeline_parallelism = pdict['pipeline_parallelism'] + self.recompute = pdict['recompute'] + self.precision = pdict['precision'] + + # Resolve the training script for this tokenizer family. The mapping is + # config-driven (training_scripts), so adding a new family (e.g. 'llama-4') + # is a config edit, not a code edit. First-match-wins on dict insertion order. + # If no entry matches, self.training_script stays None — same fallthrough + # behavior as the previous if/elif chain (queued: raise instead). + self.training_script = None + for family, script_rel_path in self.training_scripts.items(): + if re.search(family, self.tokenizer_model, re.I): + self.training_script = f'{self.megatron_root}/{script_rel_path}' + break + + self.precision_env = 'TE_FP8=1' # default + self.mbs_env = 'MBS' + self.gbs_env = 'GBS' + for family, flags in PRECISION_FLAGS.items(): + if re.search(family, self.tokenizer_model, re.I): + self.precision_env = flags.get(self.precision, 'TE_FP8=1') + break + for family, flags in BATCH_SIZE_FLAGS.items(): + if re.search(family, self.tokenizer_model, re.I): + self.mbs_env = flags['mbs'] + self.gbs_env = flags['gbs'] + break + + if re.search('mixtral', self.tokenizer_model, re.I): + self.tp_env = 'TP_SIZE' + self.pp_env = 'PP_SIZE' + self.seq_env = 'SEQLEN' + else: + self.tp_env = 'TP' + self.pp_env = 'PP' + self.seq_env = 'SEQ_LENGTH' + + if re.search('mixtral|deepseek|qwen3', self.tokenizer_model, re.I): + self.iters_env = 'TRAIN_ITERS' + else: + self.iters_env = 'TOTAL_ITERS' + + # Remove and recreate the scripts dir on the bare host (volume-mounted path) + self.orch.all.exec(f'rm -rf {self.scripts_dir}') + self.orch.all.exec(f'mkdir -p {self.scripts_dir}') + self.orch.all.exec(f'sudo chmod 700 {self.scripts_dir}') + + # Let us override some of the params based on number of nodes and platform + # if override flag set .. + if self.tune_model_params: + # Assuming the training json configs were built with 4 nodes = 32 gpus + if int(self.global_batch_size) > 32: + if int(self.global_batch_size) % 32 == 0: + per_gpu_batch_size = int(self.global_batch_size) / 32 + self.global_batch_size = per_gpu_batch_size * int(self.nnodes) * 8 + + def _needs_local_tokenizer(self): + return bool(re.search(r'deepseek|mixtral', self.tokenizer_model, re.I)) + + def download_tokenizer_model(self): + """Download tokenizer.model locally for models that require a file path instead of an HF repo ID. + + No-op for llama/qwen (their training scripts accept the HF repo ID directly). + For deepseek/mixtral, downloads the tokenizer.model file into data_cache_dir + and stores the known local path in self.local_tokenizer_path. + """ + if not self._needs_local_tokenizer(): + return + + local_dir = f'{self.data_cache_dir}/{self.model_name}' + log.info('Downloading tokenizer.model for %s into %s', self.model_name, local_dir) + self.orch.exec( + f'export HF_TOKEN={shlex.quote(self.hf_token)}; ' + f'huggingface-cli download {self.tokenizer_model} ' + f'--include "tokenizer.model" ' + f'--local-dir {local_dir}' + ) + self.local_tokenizer_path = f'{local_dir}/tokenizer.model' + log.info('tokenizer.model path: %s', self.local_tokenizer_path) + + def stop_training_processes(self): + """Check GPU VRAM after a training combo and free memory if any processes remain. + + After normal training completion VRAM% is 0 and no KFD PIDs are present — + returns immediately in that case. If processes are still holding GPU memory + (crash or hang), extracts their PIDs from rocm-smi --showpids, kills them + with SIGKILL, then waits and verifies VRAM is clear before the next combo. + """ + log.info('Checking GPU memory state after training combo') + out_dict = self.orch.exec('rocm-smi --showpids 2>/dev/null') + + has_pids = False + for node, output in (out_dict or {}).items(): + if 'No KFD PIDs currently running' in (output or ''): + log.info('Node %s: VRAM already free, no GPU processes running', node) + else: + log.warning('Node %s: GPU processes still holding VRAM, will kill', node) + has_pids = True + + if not has_pids: + return + + # Extract PIDs (lines starting with a number) and SIGKILL on all nodes + self.orch.exec( + "rocm-smi --showpids 2>/dev/null " + "| awk '/^[0-9]+[[:space:]]/{print $1}' " + "| xargs -r kill -9 2>/dev/null || true; " + "sleep 10" + ) + + # Verify VRAM is now free + out_dict = self.orch.exec('rocm-smi --showpids 2>/dev/null') + for node, output in (out_dict or {}).items(): + if 'No KFD PIDs currently running' in (output or ''): + log.info('Node %s: VRAM successfully freed', node) + else: + log.warning('Node %s: GPU processes may still be running after kill attempt', node) + + def run_pretraining_tasks( + self, + ): + if self.distributed_training is True: + self.rdma_stats_dict_before = linux_utils.get_rdma_stats_dict(self.orch.all) + self.ethtool_stats_dict_before = linux_utils.get_nic_ethtool_stats_dict(self.orch.all) + + def exec_nic_setup_scripts( + self, + ): + """ + Prepare backend NICs inside containers before starting distributed training. + + This method applies in-container NIC setup steps when running distributed jobs. + It currently implements a Broadcom-specific workaround to ensure the RDMA + provider library (bnxt_re) is correctly available inside the container. + + Behavior: + - Only runs when distributed_training is True. + - If nic_type indicates Broadcom/Thor, it: + * Forces NCCL GID index to 3 (common Broadcom requirement). + * Copies the host-side libbnxt_re library into the container?s ibverbs path. + * Runs ibv_devinfo to verify the RDMA device enumerates as bnxt_. + * Fails the test if the expected device string is not detected. + + Assumptions: + - self.orch provides exec(...) to run commands inside containers on all nodes. + - Docker is installed and the container is already running on each node. + - The source and destination library paths are correct for the target image. + - fail_test(...) is available in scope to abort on setup failures. + + """ + # Run all your backend NIC related bringups for containers here .. + if self.distributed_training is True: + # This is a temporary hack needed for broadcom nics to work within containers .. + if re.search('broadcom|thor', self.nic_type, re.I): + # override the gid_index to 3 for broadcom + self.nccl_ib_gid_index = 3 + out_dict = self.orch.exec( + 'sudo cp /usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so.host ' + '/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so; ' + 'sleep 2;ibv_devinfo;sleep 2;' + ) + # Treat `hca_id_pattern` as a `|`-separated list of literal + # NIC-name prefixes. Each segment is `re.escape`d so users + # can't accidentally inject regex syntax (e.g. `mlx5+` is a + # literal 5-char prefix, not `mlx` + `5+` quantifier). + # For the default `bnxt_|rocep`, the emitted regex is + # byte-identical to the prior raw-interpolation behavior. + segments = [re.escape(s.strip()) for s in self.hca_id_pattern.split('|') if s.strip()] + if not segments: + fail_test( + f'hca_id_pattern parsed to zero non-empty segments, got: {self.hca_id_pattern!r}. ' + f'Expected a `|`-separated list of NIC-name prefixes, e.g. "bnxt_|rocep".' + ) + return False + hca_id_regex = rf'hca_id:\s+({"|".join(segments)})' + for node in out_dict.keys(): + if not re.search(hca_id_regex, out_dict[node], re.I): + log.info("%s", out_dict[node]) + fail_test(f'Broadcom libbnxt rdma driver is not properly copied on node {node}') + return False + return True + + def build_training_job_cmd( + self, + ): + # Construct the main megatron training command + # Compute the batch size and mini batch size based on the cluster size + # Add NIC and Socket details for distributed training .. + cmd = '' + + # cmd = f'docker exec {self.container_name} /bin/bash -c """' + + cmd = ( + cmd + + f'cd {self.megatron_root}; export MOCK_DATA=1; ' + + f'export IMAGE={self.container_image}; ' + + f'export HF_TOKEN="{self.hf_token}"; ' + + f'export DATA_CACHE_PATH={self.data_cache_dir}; ' + + f'export TOKENIZER_MODEL={self.local_tokenizer_path if self.local_tokenizer_path else self.tokenizer_model}; ' + + f'export LD_LIBRARY_PATH=/usr/local/lib/:{self.rocm_path}/lib:$LD_LIBRARY_PATH; ' + + f'export LOG_DIR={self.log_dir}; ' + + 'export EXP_NAME="megatron_training"; ' + + 'export TORCH_NCCL_ASYNC_ERROR_HANDLING=0; ' + ) + + if self.distributed_training is True: + # Add the backend network related environment variables .. + cmd = ( + cmd + + f'export NCCL_IB_HCA_LIST={self.nccl_ib_hca_list}; ' + + f'export NCCL_IB_HCA={self.nccl_ib_hca}; ' + + f'export NCCL_SOCKET_IFNAME={self.nccl_socket_ifname}; ' + + f'export GLOO_SOCKET_IFNAME={self.gloo_socket_ifname}; ' + + f'export NCCL_DEBUG={self.nccl_debug}; ' + + f'export NCCL_IB_GID_INDEX={self.nccl_ib_gid_index}; ' + ) + + if self.distributed_training is True: + # Build base cmd; NODE_RANK={i} is injected per-host in start_training_job + cmd = ( + cmd + + f'RECOMPUTE={self.recompute} ' + + f'{self.seq_env}={self.sequence_length} ' + + f'{self.mbs_env}={self.micro_batch_size} {self.gbs_env}={self.global_batch_size} ' + + f'{self.tp_env}={self.tensor_parallelism} ' + + f'{self.pp_env}={self.pipeline_parallelism} FSDP={self.fsdp} ' + + f'MODEL_SIZE={self.model_size} {self.iters_env}={self.iterations} ' + + self.precision_env + + ' ' + + f'MASTER_ADDR={self.master_address} NNODES={self.nnodes} ' + ) + + for i in range(len(self.orch.hosts)): + full_cmd = cmd + f'NODE_RANK={i} nohup bash {self.training_script} &' + script_cmd = f'umask 077; echo {shlex.quote(full_cmd)} > {self.scripts_dir}/distributed_wrapper_script_{i}.sh && chmod 700 {self.scripts_dir}/distributed_wrapper_script_{i}.sh' + self.job_cmd_list.append(script_cmd) + + else: + # Single node training case, run same cmd on all nodes. + cmd = ( + cmd + + f'RECOMPUTE={self.recompute} ' + + f'{self.seq_env}={self.sequence_length} ' + + f'{self.mbs_env}={self.micro_batch_size} {self.gbs_env}={self.global_batch_size} ' + + f'{self.tp_env}={self.tensor_parallelism} ' + + f'{self.pp_env}={self.pipeline_parallelism} FSDP={self.fsdp} ' + + f'MODEL_SIZE={self.model_size} {self.iters_env}={self.iterations} ' + ) + cmd = cmd + self.precision_env + ' ' + self.job_cmd = cmd + f'nohup bash {self.training_script} &' + + def start_training_job(self, timeout=500): + """ + Launch the Megatron-LM training job (distributed or single-node). + + Behavior: + - Prints debug information about the prepared commands. + - Distributed mode: + * Runs NIC setup workarounds (if any). + * Creates per-node distributed wrapper scripts across nodes via phdl.exec_cmd_list. + * Executes those scripts inside each node's container using docker exec. + - Single-node mode: + * Writes a single wrapper script locally. + * Executes the script inside the container. + * Ensures the training log file is writable. + - Sleeps for a short period to allow processes to initialize before polling. + + Args: + timeout (int): Reserved for future use (e.g., health checks). Not currently used. + + Assumptions: + - self.phdl provides: + * exec(cmd: str) -> per-node or local command execution + * exec_cmd_list(cmd_list: List[str]) -> parallel per-node execution + - Docker is installed and container self.container_name is available on each node. + - self.job_cmd_list (distributed) or self.job_cmd (single-node) has been populated + by build_training_job_cmd() prior to invocation. + """ + log.info('start training job') + log.info('%s', self.job_cmd_list) + log.info("%s", self.job_cmd) + n = len(self.orch.hosts) + + # Create per-node log dirs inside container on all hosts in parallel + self.orch.exec_cmd_list([f'mkdir -p {self.combo_log_dir}/out-node{i}' for i in range(n)]) + + # Patch TRAIN_LOG path in training script on all hosts in parallel (inside container) + self.orch.exec_cmd_list( + [ + f'sed -i "/^TRAIN_LOG=/c\\TRAIN_LOG={self.combo_log_dir}/out-node{i}/training.log" {self.training_script}' + for i in range(n) + ] + ) + + if self.distributed_training: + # Run any required NIC setup steps inside containers (e.g., Broadcom workaround) + if not self.exec_nic_setup_scripts(): + return + + self.orch.all.exec_cmd_list(self.job_cmd_list) + # Write per-node wrapper scripts on bare host in parallel (scripts_dir is a volume mount) + + # self.orch.all.exec_cmd_list([ + # f'echo {shlex.quote(self.job_cmd + f"NODE_RANK={i} nohup bash {self.training_script} &")} > ' + # f'{self.scripts_dir}/distributed_wrapper_script_{i}.sh ' + # f'&& chmod 777 {self.scripts_dir}/distributed_wrapper_script_{i}.sh' + # for i in range(n) + # ]) + + # Launch wrapper scripts inside container on all nodes in parallel + self.orch.exec_cmd_list( + [ + f'nohup {self.scripts_dir}/distributed_wrapper_script_{i}.sh > ' + f'{self.combo_log_dir}/out-node{i}/training.log 2>&1 &' + for i in range(n) + ] + ) + else: + # Write single-node wrapper script on bare host + self.orch.all.exec( + f'umask 077; echo {shlex.quote(self.job_cmd)} > {self.scripts_dir}/single_node_wrapper_script.sh ' + f'&& chmod 700 {self.scripts_dir}/single_node_wrapper_script.sh' + ) + # Launch inside container + self.orch.exec( + f'nohup {self.scripts_dir}/single_node_wrapper_script.sh > ' + f'{self.combo_log_dir}/out-node0/training.log 2>&1 &' + ) + time.sleep(50) + + def _read_last_node_log(self, tail_lines=0): + """Read the training log from the last node and return its output. + + Args: + tail_lines (int): If > 0, only the last N lines of the log are read. + + Returns: + str: Log text from the last node. + """ + n = len(self.orch.hosts) + last_host = self.orch.hosts[-1] + tail_suffix = f' | tail -{tail_lines}' if tail_lines > 0 else '' + out_dict = self.orch.exec( + f'cat {self.combo_log_dir}/out-node{n - 1}/training.log{tail_suffix}', hosts=[last_host] + ) + return out_dict.get(last_host) or '' + + def get_training_results_dict(self): + """Parse training log from the last node and extract key performance metrics. + + Reads the tail (summary lines) and full log, passing both to + _parse_training_results which handles primary summary-line parsing and + per-iteration fallback for models without shell-appended summary lines. + + Returns: + dict: A dictionary with lists of extracted values (strings) for each metric. + """ + tail_output = self._read_last_node_log(tail_lines=15) + + log.info('Extracting results from logs') + log.info('#===========================#') + log.info("%s", tail_output) + log.info('#===========================#') + + full_log = self._read_last_node_log() + training_results_dict = _parse_training_results(tail_output, full_log) + log.info("%s", training_results_dict) + return training_results_dict + + def scan_for_training_errors(self): + """Scan training logs from the last node for known error patterns. + + Returns: + bool: True if no error patterns found; False otherwise. + """ + log.info('Scan for training errors') + training_pass = True + + output = self._read_last_node_log() + for err_key in training_err_dict: + if re.search(f'{training_err_dict[err_key]}', output): + fail_test(f'ERROR {training_err_dict[err_key]} seen in training logs ..') + log.error('Aborting training log polling') + training_pass = False + return training_pass + + def poll_for_training_completion(self, time_between_iters=120): + """ + Periodically poll training logs to detect completion, surface errors, and validate results. + + Args: + time_between_iters (int | float): Seconds to sleep between each polling iteration. + + Behavior: + - Waits an initial 60s to allow training to start producing logs. + - For up to `self.iterations` loops: + * Invokes self.scan_for_training_errors(); aborts if it flags errors. + * Reads the consolidated training log from the "last" node in self.host_list. + * Checks for completion indicators (throughput per GPU or tokens/GPU/s). + - If not seen, prints a status and sleeps before next iteration. + - If seen, verifies that metrics do not contain NaN/Inf values. + - Fails on invalid values, else parses and stores results via get_training_results_dict(). + - Returns on success or failure (no explicit return value). + - Sleeps `time_between_iters` seconds between iterations (except when it early-sleeps 30s on in-progress). + + + Assumptions: + - self.host_list is non-empty; last node contains authoritative training logs. + - self.phdl.exec(cmd) returns {node: stdout_str}. + - self.scan_for_training_errors() returns True when OK, False on error patterns. + - self.get_training_results_dict() parses known metrics into self.training_results_dict. + - re, time, and fail_test are available in scope. + + Notes: + - The regex '[NaN|Inf]' uses a character class and will not match "NaN" or "Inf" as intended. + Consider using '(NaN|Inf)' to check for either token. + - The progress regex for tokens/GPU/s lacks a colon; consider 'tokens\\/GPU\\/s:\\s+[0-9]+'. + """ + + log.info('Poll for training completion ..') + time.sleep(80) + + # 10 additional iterations in case time per iteration is longer .. + for i in range(1, int(self.iterations) + 10): + log.info(f'Starting Iteration {i}') + if not self.scan_for_training_errors(): + fail_test('Failures seen in training logs, Aborting!!!') + return + output = self._read_last_node_log() + + if not _is_training_complete(output, self.iterations): + log.info('Training still in progress') + else: + if _has_nan_inf_results(output): + fail_test(f'ERROR - NaN or Inf values seen in training results {output}') + return + else: + time.sleep(30) + self.training_results_dict = self.get_training_results_dict() + log.info('Completed Training, returning !!!') + return + # Wait secs between every iteration + time.sleep(int(time_between_iters)) + + def verify_training_results( + self, + ): + """ + Validate collected training results and environment health after a training run. + + Behavior: + - Records end time of training for later log scanning. + - Scans parsed training_results_dict for NaN/Inf values in any reported metric. + - If distributed training is enabled: + * Collects RDMA and NIC (ethtool) stats after training. + * Verifies selected error counters did not increase vs. their pre-training baselines. + - Scans kernel logs (dmesg) between training start and end for known error patterns. + - Compares observed performance results against expected thresholds provided in + self.expected_result_dict and flags deviations. + + Assumptions: + - self.phdl.exec(cmd) returns a mapping of node -> command output (string). + - self.training_results_dict is populated before calling this method and structured + as: { metric_key: }. + - self.distributed_training indicates whether to collect/compare network-related stats. + - linux_utils.get_rdma_stats_dict and linux_utils.get_nic_ethtool_stats_dict return + per-node dictionaries of counters where values are numeric strings or ints. + - err_counters_pattern is a regex pattern for error counters to check. + - verify_dmesg_for_errors(phdl, start_time_dict, end_time_dict) is available and + scans logs between provided timestamps mapped by node. + - self.training_start_time and self.training_end_time are dicts keyed by node with + human-readable timestamps, compatible with verify_dmesg_for_errors. + - self.expected_result_dict contains numeric thresholds as strings or numbers. + + Side effects: + - Calls fail_test(...) to report errors and accumulate failure messages. + - Logs warnings for missing expected result keys. + + Returns: + None. Uses fail_test to record failures. + """ + + # across nodes what numbers we are getting - median variance, per iteration variance. + # Network errors + + # Record the training end time; used later for dmesg time-bounded scanning + self.training_end_time = self.orch.all.exec('date') + + log.info('#==================================================#') + log.info('\t\tTraining Results') + log.info("%s", self.training_results_dict) + log.info('#==================================================#') + # Check the parsed training results for invalid numeric values (NaN/Inf) + if not self.training_results_dict: + fail_test( + 'Failed to populate training results, training_results_dict is empty - please check logs for failures' + ) + return + + for result_key in self.training_results_dict.keys(): + for result_val in self.training_results_dict[result_key]: + if re.search('nan|inf', result_val, re.I): + fail_test( + f'Failures seen in training_result dict for {result_key}, numbers are either NaN or Inf - f{result_val}' + ) + + # Check if RDMA and Ethtool stats have errors .. + if self.distributed_training is True: + if self.verify_network_errors.lower() == "true": + self.rdma_stats_dict_after = linux_utils.get_rdma_stats_dict(self.orch.all) + self.ethtool_stats_dict_after = linux_utils.get_nic_ethtool_stats_dict(self.orch.all) + + # Compare RDMA error counters; fail if any error counter increased + for node in self.rdma_stats_dict_after.keys(): + for counter_nam in self.rdma_stats_dict_after[node]: + if re.search(f'{err_counters_pattern}', counter_nam, re.I): + if int(self.rdma_stats_dict_after[node][counter_nam]) > int( + self.rdma_stats_dict_before[node][counter_nam] + ): + fail_test( + f'Error counter {counter_nam} has gone up after training on node {node} \ + Before = {self.rdma_stats_dict_before[node][counter_nam]}, \ + After = {self.rdma_stats_dict_after[node][counter_nam]}' + ) + + # Compare NIC error counters; fail if any error counter increased + for node in self.ethtool_stats_dict_after.keys(): + for counter_nam in self.ethtool_stats_dict_after[node]: + if re.search(f'{err_counters_pattern}', counter_nam, re.I): + if int(self.ethtool_stats_dict_after[node][counter_nam]) > int( + self.ethtool_stats_dict_before[node][counter_nam] + ): + fail_test( + f'Error counter {counter_nam} has gone up after training on node {node} \ + Before = {self.ethtool_stats_dict_before[node][counter_nam]}, \ + After = {self.ethtool_stats_dict_after[node][counter_nam]}' + ) + + # Scan Dmesg for errors .. + verify_dmesg_for_errors(self.orch.all, self.training_start_time, self.training_end_time, till_end_flag=False) + + log.info('^^^^^^^^^^^^^^^^^^^^') + log.info('training_results_dict') + log.info('^^^^^^^^^^^^^^^^^^^^') + log.info("%s", self.training_results_dict) diff --git a/cvs/lib/training/megatron/utils/__init__.py b/cvs/lib/training/megatron/utils/__init__.py new file mode 100644 index 000000000..d3438a6e8 --- /dev/null +++ b/cvs/lib/training/megatron/utils/__init__.py @@ -0,0 +1,4 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' diff --git a/cvs/lib/training/megatron/utils/loss_curve.py b/cvs/lib/training/megatron/utils/loss_curve.py new file mode 100644 index 000000000..5ee92ee55 --- /dev/null +++ b/cvs/lib/training/megatron/utils/loss_curve.py @@ -0,0 +1,131 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Loss curve utilities for Megatron training log analysis. + +parse_all_loss_points — extract every (step, lm_loss) pair from a training log +sample_loss_curve — downsample points by stride and milestone steps +evaluate_loss_decreasing — slope-based smooth-decrease check (least-squares) +''' + +from __future__ import annotations + +import re +from typing import Dict, List, Optional, Tuple + + +def parse_all_loss_points(log_text: str) -> List[Dict]: + """Extract every (step, lm_loss) pair from a full Megatron training log. + + Each Megatron log line has the form: + iteration / | ... | lm loss: | ... + + Scans all iteration lines and returns them as a list of + ``{"step": int, "loss": float}`` dicts in log order. Only lines that + contain both an iteration number and a numeric ``lm loss`` value are + included. + + Args: + log_text: Full training log text. + + Returns: + List of ``{"step": int, "loss": float}`` dicts, one per parsed line. + """ + results = [] + pattern = re.compile( + r'iteration\s+(\d+)\s*/\s*\d+[^\n]*?\blm loss:\s*([0-9.eE+\-]+)', + re.I, + ) + for m in pattern.finditer(log_text): + results.append({"step": int(m.group(1)), "loss": float(m.group(2))}) + return results + + +def sample_loss_curve( + step_metrics: List[Dict], + sample_every: int = 10, + milestone_steps: Optional[List[int]] = None, +) -> List[Tuple[int, float]]: + """Downsample per-step training loss for the loss curve check. + + Keeps a point when its step is a multiple of ``sample_every``, is one of + the ``milestone_steps`` (e.g. 100/500/1k/5k), or is the first/last + recorded step. The first/last inclusion keeps short runs from producing + an empty curve. + + Args: + step_metrics: List of ``{"step": int, "loss": float}`` dicts as + returned by ``parse_all_loss_points``. + sample_every: Keep every Nth step (default 10). + milestone_steps: Additional steps to always include. + + Returns: + Ordered, de-duplicated list of ``(step, loss)`` tuples. Empty when + ``step_metrics`` is empty or contains no numeric loss values. + """ + milestones = set(milestone_steps or []) + every = sample_every if sample_every and sample_every > 0 else 1 + + loss_steps = [ + s for s in (step_metrics or []) if s.get("step") is not None and isinstance(s.get("loss"), (int, float)) + ] + if not loss_steps: + return [] + + first_step = loss_steps[0]["step"] + last_step = loss_steps[-1]["step"] + + picked: Dict[int, float] = {} + for s in loss_steps: + step = s["step"] + if step % every == 0 or step in milestones or step in (first_step, last_step): + picked[step] = s["loss"] + + return [(step, picked[step]) for step in sorted(picked)] + + +def evaluate_loss_decreasing( + points: List[Tuple[int, float]], + max_slope: float = 0.0, +) -> Optional[Tuple[bool, float, str]]: + """Decide whether a sampled loss curve trends downward using linear regression. + + Fits a least-squares line to ``points`` and treats the run as decreasing + when the slope is below ``max_slope`` (default 0.0, i.e. strictly negative). + Uses a dependency-free closed form: + + slope = (n*Sxy - Sx*Sy) / (n*Sxx - Sx²) + + Args: + points: Ordered list of ``(step, loss)`` tuples from + ``sample_loss_curve``. + max_slope: Slope threshold; slope < max_slope is considered decreasing. + + Returns: + ``(decreasing, slope, detail)`` or ``None`` when fewer than 2 points + are present or all steps are identical (degenerate case). Never raises. + """ + if not points or len(points) < 2: + return None + + n = len(points) + sx = sum(p[0] for p in points) + sy = sum(p[1] for p in points) + sxx = sum(p[0] * p[0] for p in points) + sxy = sum(p[0] * p[1] for p in points) + + denom = n * sxx - sx * sx + if denom == 0: + return None + + slope = (n * sxy - sx * sy) / denom + decreasing = slope < max_slope + detail = ( + f"loss slope {slope:.6g}/step over {n} points " + f"(first={points[0][1]:.4f}@step{points[0][0]}, " + f"last={points[-1][1]:.4f}@step{points[-1][0]}); " + f"{'decreasing' if decreasing else 'NOT decreasing'} " + f"(threshold max_slope={max_slope})" + ) + return (decreasing, slope, detail) diff --git a/cvs/lib/training/megatron/utils/loss_curve_plot.py b/cvs/lib/training/megatron/utils/loss_curve_plot.py new file mode 100644 index 000000000..d9cdab6ed --- /dev/null +++ b/cvs/lib/training/megatron/utils/loss_curve_plot.py @@ -0,0 +1,60 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Loss curve PNG rendering for Megatron training suites. +''' + +from __future__ import annotations + +from cvs.lib import globals + +log = globals.log + + +def render_loss_curve_png(points, out_path, title=None): + """Render a training loss curve to a PNG file. + + Args: + points: Ordered list of ``(step, loss)`` tuples from + ``loss_curve.sample_loss_curve``. + out_path: Destination PNG path (str or Path). + title: Optional plot title. + + Returns: + ``out_path`` as str on success, or ``None`` if there is nothing to plot + or matplotlib is unavailable / rendering failed. Never raises. + """ + if not points: + log.info("loss curve: no points to plot, skipping PNG") + return None + + try: + import matplotlib + + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except Exception as e: + log.warning("loss curve: matplotlib unavailable, skipping PNG (%s)", e) + return None + + try: + steps = [p[0] for p in points] + losses = [p[1] for p in points] + + fig, ax = plt.subplots(figsize=(8, 4.5)) + ax.plot(steps, losses, marker="o", markersize=3, linewidth=1.5, color="#1f77b4") + ax.set_xlabel("step") + ax.set_ylabel("lm_loss") + ax.set_title(title or "Training Loss Curve") + ax.grid(True, linestyle="--", alpha=0.4) + fig.tight_layout() + + out_path = str(out_path) + fig.savefig(out_path, dpi=100) + plt.close(fig) + log.info("loss curve: wrote PNG %s (%d points)", out_path, len(points)) + return out_path + except Exception as e: + log.warning("loss curve: failed to render PNG (%s)", e) + return None diff --git a/cvs/lib/training/megatron/utils/model_registry.py b/cvs/lib/training/megatron/utils/model_registry.py new file mode 100644 index 000000000..0aca43cc4 --- /dev/null +++ b/cvs/lib/training/megatron/utils/model_registry.py @@ -0,0 +1,63 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. +The year included in the foregoing notice is the year of creation of the work. +All code contained here is Property of Advanced Micro Devices, Inc. + +Model registry for Megatron training — all model-specific lookup tables live here. + +Adding a new model family: add one entry to each of TRAINING_SCRIPTS, +PRECISION_FLAGS, and BATCH_SIZE_FLAGS. No changes to megatron_lib.py needed. + +Key ordering note: qwen3 must appear before qwen2 in every dict so that +tokenizer names containing "Qwen3" do not match the "qwen2" pattern first. +''' + +# Training script paths relative to megatron_root, keyed by tokenizer family regex. +TRAINING_SCRIPTS = { + 'llama-3': 'examples/llama/train_llama3.sh', + 'llama-2': 'examples/llama/train_llama2.sh', + 'deepseek': 'examples/deepseek_v2/train_deepseekv2.sh', + 'mixtral': 'examples/mixtral/train_mixtral_moe.sh', + 'qwen3': 'examples/qwen3/train_qwen3.sh', + 'qwen2': 'examples/qwen2/train_qwen2.sh', +} + +# Precision env var strings per model family and precision name. +PRECISION_FLAGS = { + 'llama': { + 'FP8': 'TE_FP8=1', + 'BF16': 'TE_FP8=0 TE_FP4=0', + 'MXFP4': 'TE_FP4=1 TE_FP4_RECIPE=mxfp4', + 'MXFP8': 'TE_FP8=1 TE_FP8_RECIPE=mxfp8', + }, + 'deepseek': { + 'FP8': 'PR=fp8', + 'FP16': 'PR=fp16', + 'BF16': 'PR=bf16', + }, + 'qwen3': { + 'FP8': 'PR=fp8', + 'BF16': 'PR=bf16', + 'MXFP8': 'PR=fp8 FP8_RECIPE=mxfp8', + }, + 'qwen2': { + 'FP8': 'TE_FP8=1', + 'BF16': 'TE_FP8=0', + }, + 'mixtral': { + 'FP8': 'PR=fp8', + 'FP16': 'PR=fp16', + 'BF16': 'PR=bf16', + }, +} + +# Batch size env var names per model family. +# mbs → micro batch size env var name, gbs → global batch size env var name. +BATCH_SIZE_FLAGS = { + 'llama': {'mbs': 'MBS', 'gbs': 'BS'}, + 'deepseek': {'mbs': 'MBS', 'gbs': 'GBS'}, + 'qwen3': {'mbs': 'MICRO_BATCH_SIZE', 'gbs': 'GLOBAL_BATCH_SIZE'}, + 'qwen2': {'mbs': 'MBS', 'gbs': 'BS'}, + 'mixtral': {'mbs': 'MBS', 'gbs': 'GBS'}, +} diff --git a/cvs/lib/training/megatron/utils/scaling.py b/cvs/lib/training/megatron/utils/scaling.py new file mode 100644 index 000000000..c80cdfb93 --- /dev/null +++ b/cvs/lib/training/megatron/utils/scaling.py @@ -0,0 +1,38 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Scaling efficiency utilities for Megatron distributed training analysis. +''' + +from __future__ import annotations + +from typing import Optional + + +def compute_scaling_efficiency( + tokens_per_sec_total: Optional[float], + num_nodes: Optional[int], + baseline_tokens_per_sec_total: Optional[float], + baseline_num_nodes: int = 1, +) -> Optional[float]: + """Scaling efficiency % for a training run. + + efficiency % = throughput_N / ((N / ref_N) * throughput_ref) * 100 + + where throughput_N is this run's total tokens/sec on `num_nodes` nodes and + throughput_ref is the reference (typically 1-node) total tokens/sec measured + on `baseline_num_nodes` nodes. 100% means perfectly linear scaling; lower + means communication/straggler overhead is eating into the added nodes. + + Returns None (record-only) when any input is missing or non-positive so an + uncalibrated baseline never produces a misleading number or a crash. + """ + if not tokens_per_sec_total or not baseline_tokens_per_sec_total: + return None + if not num_nodes or not baseline_num_nodes: + return None + ideal = (num_nodes / baseline_num_nodes) * baseline_tokens_per_sec_total + if ideal <= 0: + return None + return tokens_per_sec_total / ideal * 100.0 diff --git a/cvs/lib/training/megatron/utils/training_config_loader.py b/cvs/lib/training/megatron/utils/training_config_loader.py new file mode 100644 index 000000000..b84cc7c47 --- /dev/null +++ b/cvs/lib/training/megatron/utils/training_config_loader.py @@ -0,0 +1,218 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Training-specific config schema for Megatron suites (single-node and distributed). + +The framework-agnostic machinery (ContainerSpec, RuntimeSpec, placeholder +substitution, threshold file discovery) lives in `cvs.lib.utils.config_loader`. +This module holds the training half: MegatronSweepCombo, MegatronSweep, +MegatronVariantConfig, and load_training_variant. + +Thresholds live in a sibling *threshold.json file (not inline in result_dict). +The threshold file is discovered via the `threshold_json` field in the config or +auto-discovered as the sole *threshold.json sibling. Cell keys in the threshold +file must match the combination keys in sweep.combinations exactly. + +enforce_thresholds gates whether threshold specs are asserted in test_metric. + +Both megatron_single and megatron_distributed are covered by MegatronVariantConfig +via the framework field, which is a validated schema tag / config discriminator. +''' + +from __future__ import annotations + +import warnings +from collections import Counter +from typing import Any, Dict, List + +from pydantic import Field, model_validator +from typing_extensions import Literal + +from cvs.lib.utils.config_loader import ( + ContainerSpec, + _Forbid, + substitute_config, +) + + +# ---------- pydantic models (training) ---------- + + +class MegatronSweepCombo(_Forbid): + name: str + micro_batch_size: str + global_batch_size: str + precision: str = "" + + +def validate_sweep_selector(combo_keys, run_refs): + """The sweep-selector rule: combination keys unique, every run references one. + + Single home for this check, shared by the typed MegatronSweep validator + (load time) and pytest_generate_tests (collection time, which reads raw + JSON before the loader runs) so the two can never drift. + + Without it a typo'd run key is a silently-dropped cell — the sweep runs + a different matrix than the config reads. + """ + counts = Counter(combo_keys) + dupes = sorted(k for k, count in counts.items() if count > 1) + if dupes: + raise ValueError(f"duplicate sweep.combinations keys: {dupes}") + known = set(counts) + unknown = sorted(r for r in run_refs if r not in known) + if unknown: + raise ValueError(f"sweep.runs references unknown combinations: {unknown} (known: {sorted(known)})") + + +def validate_thresholds_cover_sweep( + *, + expected_cells, + thresholds, + enforce_thresholds: bool, + gated_metrics=None, +) -> None: + """Shared sweep/threshold coverage check for training variant configs. + + Checks every sweep cell has a threshold entry and no threshold key is + orphaned. Individual metrics within a cell are optional — absent specs + are skipped in test_metric (record-only for that metric). + """ + expected = set(expected_cells) + present = set(thresholds.keys()) + missing = sorted(expected - present) + extra = sorted(present - expected) + problems = [] + if missing: + problems.append(f"sweep cells with no threshold entry: {missing}") + if extra: + problems.append(f"threshold keys matching no sweep cell (typo?): {extra}") + gated = gated_metrics if gated_metrics is not None else set() + gated_keys = [f"training.{m}" for m in sorted(gated)] + gated_gaps = {} + for cell in sorted(expected & present): + specs = thresholds.get(cell) or {} + absent = [k for k in gated_keys if k not in specs] + if absent: + gated_gaps[cell] = absent + if gated_gaps: + problems.append(f"cells missing gated-metric specs: {gated_gaps}") + if problems: + msg = "threshold.json does not match the sweep matrix; " + "; ".join(problems) + if enforce_thresholds: + raise ValueError(msg) + warnings.warn(f"{msg} (enforce_thresholds=false -> record-only)", stacklevel=3) + + +class MegatronSweep(_Forbid): + combinations: Dict[str, MegatronSweepCombo] + runs: List[str] + + @model_validator(mode="after") + def _check_runs_reference_known_combos(self): + validate_sweep_selector( + list(self.combinations.keys()), + self.runs, + ) + return self + + +class ScalingBaseline(_Forbid): + tokens_per_sec_total: float = 0.0 + num_nodes: int = 1 + + +class LossCurveConfig(_Forbid): + sample_every: int = 10 + milestone_steps: List[int] = Field(default_factory=lambda: [100, 500, 1000, 5000]) + max_slope: float = 0.0 + enforce: bool = True + + +class MegatronVariantConfig(_Forbid): + schema_version: Literal[1] + framework: Literal["megatron_single", "megatron_distributed"] + gpu_arch: str + enforce_thresholds: bool = True + threshold_json: str = "" + scaling_baseline: ScalingBaseline = Field(default_factory=ScalingBaseline) + loss_curve: LossCurveConfig = Field(default_factory=LossCurveConfig) + config: Dict[str, Any] # training knobs: megatron_root, nccl_*, nic_type, ... + model_params: Dict[str, Any] # model knobs: model_name, precision, tp, pp, ... + container: ContainerSpec + sweep: MegatronSweep + thresholds: Dict[str, Dict[str, Any]] = Field(default_factory=dict) + + def cell_key(self, combo_key: str) -> str: + """Canonical threshold lookup key for a sweep combo. + + Constructs a key from the combo's micro_batch_size, global_batch_size, + and precision — must match the top-level keys in the threshold file exactly. + """ + combo = self.sweep.combinations[combo_key] + return f"MBS={combo.micro_batch_size},GBS={combo.global_batch_size},PRECISION={combo.precision}" + + def expected_cells(self) -> List[str]: + """Return the threshold cell key for every run in sweep.runs.""" + return [self.cell_key(k) for k in self.sweep.runs] + + @model_validator(mode="after") + def _check_thresholds_cover_sweep(self): + """Every sweep cell must have a threshold entry; no metric within it is + mandatory. test_metric treats an absent ``training.*`` spec as + "don't gate this metric" (skips the assertion), so a threshold.json + is free to gate only the metrics an operator cares about. + """ + validate_thresholds_cover_sweep( + expected_cells=self.expected_cells(), + thresholds=self.thresholds, + enforce_thresholds=self.enforce_thresholds, + gated_metrics=set(), + ) + return self + + +# ---------- public API (training) ---------- + + +def _check_no_changeme(node, path="", _offenders=None): + """Recursively collect config fields whose value still contains ''. + + Collects all offending dotted paths so the caller can report them all at once. + """ + if _offenders is None: + _offenders = [] + if isinstance(node, dict): + for k, v in node.items(): + _check_no_changeme(v, f"{path}.{k}" if path else k, _offenders) + elif isinstance(node, list): + for i, v in enumerate(node): + _check_no_changeme(v, f"{path}[{i}]", _offenders) + elif isinstance(node, str) and "" in node: + _offenders.append(path) + if not path: + if _offenders: + raise ValueError(f"config has unfilled placeholder '' in: {', '.join(_offenders)}") + + +def load_training_variant(config_path, cluster_dict) -> MegatronVariantConfig: + """Load and validate a Megatron training variant config + its threshold file. + + Delegates file read, placeholder substitution, and threshold file discovery + to the generic substitute_config. The threshold file is located via the + threshold_json field in the config (relative to the config file's directory) + or auto-discovered as the sole *threshold.json sibling. + + Cell keys in the threshold file must match MegatronVariantConfig.cell_key() + output exactly — MBS=,GBS=,PRECISION=. A load-time + validator checks that every sweep cell has a threshold entry and no key is + orphaned. + """ + raw, thresholds = substitute_config(config_path, cluster_dict) + + _check_no_changeme(raw) + + known = {k: v for k, v in raw.items() if k in MegatronVariantConfig.model_fields} + known["thresholds"] = thresholds + return MegatronVariantConfig(**known) diff --git a/cvs/lib/training/torchtitan/__init__.py b/cvs/lib/training/torchtitan/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cvs/lib/training/torchtitan/model_registry.py b/cvs/lib/training/torchtitan/model_registry.py new file mode 100644 index 000000000..596918748 --- /dev/null +++ b/cvs/lib/training/torchtitan/model_registry.py @@ -0,0 +1,145 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. +The year included in the foregoing notice is the year of creation of the work. +All code contained here is Property of Advanced Micro Devices, Inc. + +Model registry for TorchTitan training — all model-specific lookup tables live here. + +Adding a new model family: add one entry to each of MODEL_FLAVORS and +PRECISION_FLAGS. No changes to torchtitan_lib.py needed. + +TorchTitan uses TOML config files instead of shell scripts, so no training +script lookup is needed (unlike Megatron). +''' + +# Model flavor mappings: maps model_name to TorchTitan model flavor +# TorchTitan format: model.name = "llama3", model.flavor = "8B" +MODEL_FLAVORS = { + 'llama3_1_8b': { + 'name': 'llama3', + 'flavor': '8B', + 'module': 'llama3', + 'model_size': '8B', + 'tokenizer_path': 'meta-llama/Llama-3.1-8B', + 'hf_assets_subdir': 'llama3', + }, + 'llama3_1_70b': { + 'name': 'llama3', + 'flavor': '70B', + 'module': 'llama3', + 'model_size': '70B', + 'tokenizer_path': 'meta-llama/Llama-3.1-70B', + 'hf_assets_subdir': 'llama3', + }, + 'llama3_1_405b': { + 'name': 'llama3', + 'flavor': '405B', + 'module': 'llama3', + 'model_size': '405B', + 'tokenizer_path': 'meta-llama/Llama-3.1-405B', + 'hf_assets_subdir': 'llama3', + }, + 'llama3_3_70b': { + 'name': 'llama3', + 'flavor': '70B', + 'module': 'llama3', + 'model_size': '70B', + 'tokenizer_path': 'meta-llama/Llama-3.3-70B-Instruct', + 'hf_assets_subdir': 'llama3', + }, + 'deepseek_v2_lite': { + 'name': 'deepseek', + 'flavor': 'lite', + 'module': 'deepseek', + 'model_size': 'lite', + 'tokenizer_path': 'deepseek-ai/DeepSeek-V2-Lite', + 'hf_assets_subdir': 'deepseek', + }, + 'deepseek_v3_16b': { + 'name': 'deepseek_v3', + 'flavor': '16b', + 'module': 'deepseek_v3', + 'model_size': '16b', + 'tokenizer_path': 'deepseek-ai/DeepSeek-V3', + 'hf_assets_subdir': 'deepseek', + }, + 'qwen3_32b': { + 'name': 'qwen3', + 'flavor': '32B', + 'module': 'qwen3', + 'model_size': '32B', + 'tokenizer_path': 'Qwen/Qwen2.5-32B', + 'hf_assets_subdir': 'qwen', + }, + 'mixtral_8x22b': { + 'name': 'mixtral', + 'flavor': '8x22B', + 'module': 'mixtral', + 'model_size': '8x22B', + 'tokenizer_path': 'mistralai/Mixtral-8x22B-v0.1', + 'hf_assets_subdir': 'mixtral', + }, +} + +# Precision/dtype mappings per precision type +# TorchTitan format: keyed by precision name, returns dtype config +PRECISION_FLAGS = { + 'bf16': { + 'dtype': 'bfloat16', + 'enable_float8': False, + 'converters': {}, + }, + 'fp8': { + 'dtype': 'bfloat16', + 'enable_float8': True, + 'converters': {'enable_fsdp_float8_all_gather': True, 'precompute_float8_dynamic_scale_for_fsdp': True}, + }, + 'BF16': { + 'dtype': 'bfloat16', + 'enable_float8': False, + 'converters': {}, + }, + 'FP8': { + 'dtype': 'bfloat16', + 'enable_float8': True, + 'converters': {'enable_fsdp_float8_all_gather': True, 'precompute_float8_dynamic_scale_for_fsdp': True}, + }, +} + +# Float8 config flags per precision +# TorchTitan enables float8 via [quantize.linear.float8] section +FLOAT8_CONFIG = { + 'fp8': { + 'enable_fsdp_float8_all_gather': True, + 'precompute_float8_dynamic_scale_for_fsdp': True, + }, + 'bf16': { + 'enable_fsdp_float8_all_gather': False, + 'precompute_float8_dynamic_scale_for_fsdp': False, + }, +} + + +# TorchTitan model configurations - maps model_name to complete model config +# This is a compatibility layer for torchtitan_lib.py +TORCHTITAN_MODELS = { + 'llama3_1_8b': MODEL_FLAVORS['llama3_1_8b'], + 'llama3_1_70b': MODEL_FLAVORS['llama3_1_70b'], + 'llama3_1_405b': MODEL_FLAVORS['llama3_1_405b'], + 'llama3_3_70b': MODEL_FLAVORS['llama3_3_70b'], + 'deepseek_v2_lite': MODEL_FLAVORS['deepseek_v2_lite'], + 'deepseek_v3_16b': MODEL_FLAVORS['deepseek_v3_16b'], + 'qwen3_32b': MODEL_FLAVORS['qwen3_32b'], + 'mixtral_8x22b': MODEL_FLAVORS['mixtral_8x22b'], +} + +# Default training parameters for TorchTitan TOML config generation +DEFAULT_TRAINING_PARAMS = { + 'training_iterations': '10', + 'warmup_steps': '200', + 'lr': '3e-4', + 'activation_checkpointing': 'selective', + 'compile': 'false', + 'dataset': 'c4', +} diff --git a/cvs/lib/training/torchtitan/torchtitan_lib.py b/cvs/lib/training/torchtitan/torchtitan_lib.py new file mode 100644 index 000000000..9474fe16e --- /dev/null +++ b/cvs/lib/training/torchtitan/torchtitan_lib.py @@ -0,0 +1,585 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +TorchTitan training job orchestration library. + +Adapted from megatron_lib.py with TorchTitan-specific implementation: +- Uses torchrun instead of mpirun +- Generates TOML config files instead of CLI arguments +- Parses TorchTitan-specific metrics (tokens_per_sec, loss) +- Supports single-node and multi-node distributed training +''' + +import re +import shlex +import time + +from cvs.lib import globals +from cvs.lib.utils_lib import * +from cvs.lib.verify_lib import * +from cvs.lib import linux_utils +from cvs.lib.training.torchtitan.model_registry import ( + TORCHTITAN_MODELS, + PRECISION_FLAGS, + DEFAULT_TRAINING_PARAMS, +) + +log = globals.log + + +training_err_dict = { + 'NCCL ERROR': 'NCCL ERROR|NCCL timeout|ncclRemoteError: A call failed possibly due to a network error|NCCL error:', + 'GPU HW ERROR': 'HW Exception by GPU|GPU Hang|Uncorrectable error|GPU Reset', + 'torch': 'torch.distributed.elastic.multiprocessing.errors', +} + +err_counters_pattern = 'err|retransmit|drop|discard|naks|invalid|oflow|out_of_buffer|reset|fail' + + +# Ordered fallback chains for parsing TorchTitan training output +TRAINING_RESULT_PATTERNS = { + 'tokens_per_sec': [r'tps:\s+([0-9,\.]+)', r'tok/s:\s+([0-9\.]+)'], + 'loss': [r'loss:\s+([0-9\.]+)'], + 'mem_usage_gb': [r'memory:\s+([0-9\.]+)\s*GiB', r'mem:\s+([0-9\.]+)\s+GB'], +} + +TRAINING_PROGRESS_PATTERNS = [ + r'step:\s+\d+', + r'tps:\s+[0-9,\.]+', + r'loss:\s+[0-9\.]+', +] + +TRAINING_NAN_PATTERNS = [ + r'tok/s:\s+(?:NaN|Inf)', + r'loss:\s+(?:NaN|Inf)', +] + + +def _parse_training_results(output): + """Extract metric values from training-log text using ordered fallback chains.""" + out = {} + for metric, patterns in TRAINING_RESULT_PATTERNS.items(): + out[metric] = [] + for pat in patterns: + matches = re.findall(pat, output, re.I) + if matches: + # TorchTitan may emit comma-grouped numbers + out[metric] = [m.replace(',', '') for m in matches] + break + return out + + +def _is_training_complete(output, iterations): + """Return True if training log shows the configured final step.""" + final_step_pattern = rf'step:\s+{iterations}\b' + return bool(re.search(final_step_pattern, output, re.I)) + + +def _has_nan_inf_results(output): + """Return True if training log shows NaN/Inf results.""" + return any(re.search(p, output, re.I) for p in TRAINING_NAN_PATTERNS) + + +def detect_rocm_path(orch, config_rocm_path): + """ + Detect the ROCm installation path inside the container. + """ + if config_rocm_path and config_rocm_path != '': + log.info(f'Using configured ROCm path: {config_rocm_path}') + return config_rocm_path + + log.info('Auto-detecting ROCm path inside container...') + + # Try new ROCm layout first (/opt/rocm/core-X.Y) + out_dict = orch.exec('ls -d /opt/rocm/core-* 2>/dev/null | sort -V | tail -1') + for node, output in out_dict.items(): + if output and '/opt/rocm/core-' in output: + rocm_path = output.strip() + validate_dict = orch.exec( + f'test -d {rocm_path}/lib && ls {rocm_path}/lib/libamdhip64.so* 2>/dev/null | head -1' + ) + for _, lib_output in validate_dict.items(): + if lib_output.strip() and 'libamdhip64.so' in lib_output: + log.info(f'Detected ROCm path (new layout): {rocm_path}') + return rocm_path + + # Fall back to legacy /opt/rocm + out_dict = orch.exec('test -d /opt/rocm/lib && ls /opt/rocm/lib/libamdhip64.so* 2>/dev/null | head -1') + for node, output in out_dict.items(): + if output.strip() and 'libamdhip64.so' in output: + log.info('Detected ROCm path (legacy layout): /opt/rocm') + return '/opt/rocm' + + log.warning('Could not detect ROCm path, defaulting to /opt/rocm') + return '/opt/rocm' + + +class TorchTitanTrainingJob: + """ + Orchestrates a TorchTitan training job across one or more nodes. + + Similar to MegatronTrainingJob but adapted for TorchTitan: + - Uses torchrun instead of mpirun + - Generates TOML config files + - Parses TorchTitan-specific metrics + """ + + def __init__( + self, + orch, + variant_config, + hf_token, + micro_batch_size=None, + global_batch_size=None, + precision='', + result_dict=None, + distributed_training=True, + tune_model_params=True, + scripts_dir=None, + run_label=None, + ): + self.orch = orch + self.variant_config = variant_config + self.hf_token = hf_token + self.distributed_training = distributed_training + self.tune_model_params = tune_model_params + self.run_label = run_label + + self.job_cmd = '' + self.job_cmd_list = [] + self.training_results_dict = {} + + # Get config and model params + self.config = variant_config.config + self.model_params = variant_config.model_params + self.gpu_arch = variant_config.gpu_arch + + # Training configs with defaults + self.container_image = self.config.get('container_image', 'rocm/pytorch:latest') + self.container_name = self.config.get('container_name', 'torchtitan_training') + self.torchtitan_root = self.config.get('torchtitan_root', '/workspace/Primus/third_party/torchtitan') + self.iterations = int(self.config.get('training_iterations', 30)) + self.nnodes = int(self.config.get('nnodes', 1)) + self.nic_type = self.config.get('nic_type', 'thor2') + self.hca_id_pattern = self.config.get('hca_id_pattern', 'bnxt_|rocep') + self.nccl_ib_hca_list = self.config.get('nccl_ib_hca_list', '') + self.nccl_ib_hca = self.config.get('nccl_ib_hca', '') + self.nccl_socket_ifname = self.config.get('nccl_socket_ifname', '') + self.gloo_socket_ifname = self.config.get('gloo_socket_ifname', '') + self.nccl_ib_gid_index = self.config.get('nccl_ib_gid_index', '3') + self.nccl_debug = self.config.get('nccl_debug', 'ERROR') + self.data_cache_dir = self.config.get('data_cache_dir', '/tmp/cache') + self.log_dir = self.config.get('log_dir', '/tmp/logs') + self.scripts_dir = scripts_dir if scripts_dir is not None else self.config.get('scripts_dir', '/tmp/scripts') + self.master_address = self.config.get('master_address', list(orch.hosts)[0] if orch.hosts else 'localhost') + self.verify_network_errors = self.config.get('verify_network_errors', 'False') + self.rocm_path = detect_rocm_path(self.orch, self.config.get('rocm_dir', '')) + self.use_generated_config = self.config.get('use_generated_config', 'True') == 'True' + self.hf_token_file = self.config.get('hf_token_file', '/tmp/.hf_token') + + # Model params with defaults + model_name = self.model_params.get('model_name', 'llama3_3_70b') + self.model_config = TORCHTITAN_MODELS.get(model_name, TORCHTITAN_MODELS['llama3_3_70b']) + self.model_name = model_name + self.tt_module = self.model_config['module'] + self.model_size = self.model_config['model_size'] + self.tokenizer_path = self.model_config['tokenizer_path'] + + # Override batch sizes if provided + if micro_batch_size is not None: + self.micro_batch_size = str(micro_batch_size) + else: + self.micro_batch_size = str(self.model_params.get('micro_batch_size', '1')) + + if global_batch_size is not None: + self.global_batch_size = str(global_batch_size) + else: + self.global_batch_size = str(self.model_params.get('global_batch_size', '32')) + + # Precision settings + if precision: + self.precision = precision + else: + self.precision = self.model_params.get('precision', 'bf16') + + prec_flags = PRECISION_FLAGS.get(self.precision, PRECISION_FLAGS['bf16']) + self.dtype = prec_flags['dtype'] + self.enable_float8 = prec_flags['enable_float8'] + self.converters = prec_flags['converters'] + + # TorchTitan config name for fallback to canned TOMLs + self.tt_config = f'{self.tt_module}_{self.model_size}' + + # HF assets path for model downloads + self.hf_assets_path = self.model_params.get( + 'hf_assets_path', + f'./assets/hf/{self.model_config["hf_assets_subdir"]}/{self.tokenizer_path.split("/")[-1]}', + ) + + # Other training params with defaults from DEFAULT_TRAINING_PARAMS + for key, default_val in DEFAULT_TRAINING_PARAMS.items(): + setattr(self, key, self.model_params.get(key, default_val)) + + # Sequence length + self.sequence_length = str(self.model_params.get('sequence_length', '8192')) + + # Parallelism degrees + self.data_parallel_shard_degree = str(self.model_params.get('data_parallel_shard_degree', '8')) + self.tensor_parallel_degree = str(self.model_params.get('tensor_parallel_degree', '1')) + self.pipeline_parallel_degree = str(self.model_params.get('pipeline_parallel_degree', '1')) + self.context_parallel_degree = str(self.model_params.get('context_parallel_degree', '1')) + self.expert_parallel_degree = str(self.model_params.get('expert_parallel_degree', '1')) + self.enable_async_tensor_parallel = str(self.model_params.get('enable_async_tensor_parallel', 'false')).lower() + self.precompute_float8_dynamic_scale_for_fsdp = str( + self.model_params.get('precompute_float8_dynamic_scale_for_fsdp', 'false') + ).lower() + + # Result expectations + self.expected_result_dict = result_dict or {} + + # Initialize stats dicts + self.rdma_stats_dict_before = {} + self.ethtool_stats_dict_before = {} + self.rdma_stats_dict_after = {} + self.ethtool_stats_dict_after = {} + self.training_start_time = None + self.training_end_time = None + + # Create scripts directory (owner-only for security - contains HF tokens) + self.orch.exec(f'rm -rf {self.scripts_dir}') + time.sleep(1) + self.orch.exec(f'mkdir -p {self.scripts_dir}') + time.sleep(1) + self.orch.exec(f'chmod 700 {self.scripts_dir}') + + # Adjust batch size for distributed if needed + if self.tune_model_params and self.distributed_training: + gpus_per_node = 8 + total_gpus = self.nnodes * gpus_per_node + if int(self.global_batch_size) > 32: + if int(self.global_batch_size) % 32 == 0: + per_gpu_batch_size = int(self.global_batch_size) / 32 + self.global_batch_size = str(int(per_gpu_batch_size * total_gpus)) + + def run_pretraining_tasks(self): + """Snapshot network stats before training (distributed only).""" + if self.distributed_training: + self.rdma_stats_dict_before = linux_utils.get_rdma_stats_dict(self.orch) + self.ethtool_stats_dict_before = linux_utils.get_nic_ethtool_stats_dict(self.orch) + + def download_hf_assets(self): + """Download HuggingFace model assets if needed. + + Uses TorchTitan's download_hf_assets.py script to fetch model weights + and tokenizers from HuggingFace Hub. Idempotent - skips if already present. + """ + if not self.use_generated_config: + # Canned configs expect assets in ./assets/hf/ + local_dir = './assets/hf/' + else: + # Generated configs use hf_assets_path, but download needs base dir only + # (download script adds repo name automatically) + local_dir = f'./assets/hf/{self.model_config["hf_assets_subdir"]}' + + log.info(f'Downloading HF assets for {self.tokenizer_path} to {local_dir}') + + download_cmd = ( + f'cd {self.torchtitan_root}; ' + f'export HF_TOKEN={self.hf_token}; ' + f'python scripts/download_hf_assets.py --repo_id {self.tokenizer_path} ' + f'--local_dir {local_dir} --all' + ) + + out_dict = self.orch.exec(download_cmd) + for node, output in out_dict.items(): + if 'error' in (output or '').lower(): + log.warning(f'Potential download error on {node}: {output}') + + def exec_nic_setup_scripts(self): + """Setup NICs for distributed training (Broadcom/Thor only).""" + if not self.distributed_training: + return + + if re.search('broadcom|thor', self.nic_type, re.I): + self.nccl_ib_gid_index = '3' + out_dict = self.orch.exec( + 'sudo cp /usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so.host ' + '/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so && ' + 'sleep 2 && ibv_devinfo' + ) + + segments = [re.escape(s.strip()) for s in self.hca_id_pattern.split('|') if s.strip()] + if not segments: + fail_test(f'hca_id_pattern invalid: {self.hca_id_pattern}') + + hca_id_regex = rf'hca_id:\s+({"|".join(segments)})' + for node, output in out_dict.items(): + if not re.search(hca_id_regex, output or '', re.I): + fail_test(f'Broadcom RDMA device not detected on {node}') + + def _build_toml_config(self): + """Generate TorchTitan TOML configuration.""" + # Use hf_assets_path for model location + hf_path = self.hf_assets_path + + # Build quantization converters list + self.converters if isinstance(self.converters, str) else '[]' + + lines = [ + "[model]", + f'name = "{self.tt_module}"', + f'flavor = "{self.model_size.upper()}"', + f'hf_assets_path = "{hf_path}"', + "", + "[training]", + f'dataset = "{self.dataset}"', + f'local_batch_size = {self.micro_batch_size}', + f'global_batch_size = {self.global_batch_size}', + f'seq_len = {self.sequence_length}', + f'steps = {self.iterations}', + f'dtype = "{self.dtype}"', + "", + "[optimizer]", + f'lr = {self.lr}', + "", + "[lr_scheduler]", + f'warmup_steps = {self.warmup_steps}', + "", + "[parallelism]", + f'data_parallel_shard_degree = {self.data_parallel_shard_degree}', + f'tensor_parallel_degree = {self.tensor_parallel_degree}', + f'pipeline_parallel_degree = {self.pipeline_parallel_degree}', + f'context_parallel_degree = {self.context_parallel_degree}', + f'expert_parallel_degree = {self.expert_parallel_degree}', + f'enable_async_tensor_parallel = {self.enable_async_tensor_parallel}', + "", + "[activation_checkpoint]", + f'mode = "{self.activation_checkpointing}"', + "", + "[compile]", + f'enable = {self.compile}', + "", + "[quantize.linear.float8]", + f'enable_fsdp_float8_all_gather = {str(self.enable_float8).lower()}', + f'precompute_float8_dynamic_scale_for_fsdp = {self.precompute_float8_dynamic_scale_for_fsdp}', + # converters not supported in this TorchTitan version + # f'converters = {converters_str}', + 'filter_fqns = ["output"]', + "", + "[comm]", + 'init_timeout_seconds = 3600', + ] + return "\n".join(lines) + "\n" + + def _write_generated_toml(self, dest_path): + """Write TOML config to destination path on all nodes.""" + toml_content = self._build_toml_config() + log.info('Generated TorchTitan TOML config') + + # Use printf to write multi-line content + escaped = toml_content.replace('\\', '\\\\').replace('$', '\\$').replace('"', '\\"') + write_cmd = f'printf "%s" "{escaped}" > {dest_path}' + self.orch.exec(write_cmd) + + def build_training_job_cmd(self): + """Build torchrun commands for training.""" + # Base environment setup + cmd = f'cd {self.torchtitan_root}; ' + cmd += f'export HF_TOKEN={self.hf_token}; ' + cmd += 'export HSA_FORCE_FINE_GRAIN_PCIE=1; ' + cmd += 'export PYTORCH_HIP_ALLOC_CONF=expandable_segments:True; ' + # Add TorchTitan to PYTHONPATH so it can be imported as a module + cmd += f'export PYTHONPATH={self.torchtitan_root}:$PYTHONPATH; ' + + # Config file path - supports both generated and canned TOMLs + if self.use_generated_config: + config_file_path = f'{self.scripts_dir}/run_config.toml' + self._write_generated_toml(config_file_path) + else: + # Fallback to canned TOML shipped with TorchTitan + config_file_path = f'./train_configs/{self.tt_config}.toml' + log.info(f'Using canned TOML config: {config_file_path}') + + # Distributed env vars + if self.distributed_training: + cmd += f'export NCCL_IB_HCA={self.nccl_ib_hca_list}; ' + cmd += f'export NCCL_SOCKET_IFNAME={self.nccl_socket_ifname}; ' + cmd += f'export GLOO_SOCKET_IFNAME={self.gloo_socket_ifname}; ' + cmd += f'export NCCL_DEBUG={self.nccl_debug}; ' + cmd += f'export NCCL_IB_GID_INDEX={self.nccl_ib_gid_index}; ' + + nproc_per_node = 8 + + if self.distributed_training: + for i in range(self.nnodes): + torchrun_cmd = ( + f'torchrun --nnodes {self.nnodes} --node_rank={i} --nproc_per_node {nproc_per_node} ' + f'--rdzv_id 101 --rdzv_backend c10d ' + f'--rdzv_endpoint "{self.master_address}:29500" ' + f'--role rank --tee 3 ' + f'--module torchtitan.train --job.config_file {config_file_path}' + ) + + log_path = f'{self.log_dir}/torchtitan-logs/out-node{i}/training.log' + self.orch.exec(f'mkdir -p $(dirname {log_path})') + + full_cmd = cmd + f': > {log_path}; nohup {torchrun_cmd} > {log_path} 2>&1 & disown' + + script_cmd = ( + f"cat > {self.scripts_dir}/distributed_wrapper_script_{i}.sh << 'WRAPPER_EOF'\n" + f"#!/bin/bash\n{full_cmd}\nWRAPPER_EOF\n; " + f'chmod 600 {self.scripts_dir}/distributed_wrapper_script_{i}.sh' + ) + self.job_cmd_list.append(script_cmd) + else: + torchrun_cmd = ( + f'torchrun --nnodes 1 --node_rank=0 --nproc_per_node {nproc_per_node} ' + f'--rdzv_id 101 --rdzv_backend c10d ' + f'--rdzv_endpoint "{self.master_address}:29500" ' + f'--role rank --tee 3 ' + f'--module torchtitan.train --job.config_file {config_file_path}' + ) + + log_path = f'{self.log_dir}/torchtitan-logs/out-node0/training.log' + self.orch.exec(f'mkdir -p $(dirname {log_path})') + + self.job_cmd = cmd + f': > {log_path}; nohup {torchrun_cmd} > {log_path} 2>&1 & disown' + + def start_training_job(self, timeout=500): + """Launch the training job.""" + # Capture start time for dmesg verification + self.training_start_time = self.orch.exec('date') + + if self.distributed_training: + for i, cmd in enumerate(self.job_cmd_list): + log.info(f'Writing wrapper script for node {i}') + self.orch.exec(cmd) + + time.sleep(2) + + for i in range(self.nnodes): + script_path = f'{self.scripts_dir}/distributed_wrapper_script_{i}.sh' + log.info(f'Launching training on node {i}') + self.orch.exec(f'bash {script_path}', hosts=[list(self.orch.hosts)[i]]) + time.sleep(1) + else: + log.info('Launching single-node training') + self.orch.exec(f'bash -c {shlex.quote(self.job_cmd)}') + + def get_training_results_dict(self): + """Parse training results from logs.""" + if self.distributed_training: + log_files = [f'{self.log_dir}/torchtitan-logs/out-node{i}/training.log' for i in range(self.nnodes)] + else: + log_files = [f'{self.log_dir}/torchtitan-logs/out-node0/training.log'] + + all_results = {} + for log_file in log_files: + out_dict = self.orch.exec(f'cat {log_file}') + for host, output in out_dict.items(): + if output: + parsed = _parse_training_results(output) + for metric, values in parsed.items(): + if metric not in all_results: + all_results[metric] = [] + all_results[metric].extend(values) + + return all_results + + def scan_for_training_errors(self): + """Scan training logs for known error patterns.""" + if self.distributed_training: + log_files = [f'{self.log_dir}/torchtitan-logs/out-node{i}/training.log' for i in range(self.nnodes)] + else: + log_files = [f'{self.log_dir}/torchtitan-logs/out-node0/training.log'] + + for log_file in log_files: + out_dict = self.orch.exec(f'tail -1000 {log_file}') + for host, output in out_dict.items(): + if not output: + continue + for err_type, pattern in training_err_dict.items(): + if re.search(pattern, output, re.I): + fail_test(f'{err_type} detected in training log on {host}') + + def poll_for_training_completion(self, time_between_iters=120): + """Poll training logs until completion.""" + max_iters = 60 + + if self.distributed_training: + log_file = f'{self.log_dir}/torchtitan-logs/out-node0/training.log' + else: + log_file = f'{self.log_dir}/torchtitan-logs/out-node0/training.log' + + for iteration in range(max_iters): + time.sleep(time_between_iters) + log.info(f'Polling iteration {iteration + 1}/{max_iters}') + + out_dict = self.orch.exec(f'tail -500 {log_file}') + for host, output in out_dict.items(): + if output and _is_training_complete(output, self.iterations): + log.info(f'Training completed on {host}') + return + + if output and _has_nan_inf_results(output): + fail_test(f'NaN/Inf detected in training output on {host}') + + fail_test(f'Training did not complete within {max_iters * time_between_iters} seconds') + + def verify_training_results(self): + """Verify training results meet expectations.""" + # Capture end time for dmesg verification + self.training_end_time = self.orch.exec('date') + + self.training_results_dict = self.get_training_results_dict() + log.info(f'Training results: {self.training_results_dict}') + + # Scan for errors + self.scan_for_training_errors() + + # Check for NaN/Inf in results + for metric, values in self.training_results_dict.items(): + for val in values: + try: + float_val = float(val) + if str(float_val).lower() in ['nan', 'inf', '-inf']: + fail_test(f'Invalid value {val} for metric {metric}') + except ValueError: + fail_test(f'Cannot parse value {val} for metric {metric}') + + # Check network errors if requested + if self.distributed_training and self.verify_network_errors == 'True': + self.rdma_stats_dict_after = linux_utils.get_rdma_stats_dict(self.orch) + self.ethtool_stats_dict_after = linux_utils.get_nic_ethtool_stats_dict(self.orch) + + # Compare RDMA error counters; fail if any error counter increased + for node in self.rdma_stats_dict_after.keys(): + for counter_name in self.rdma_stats_dict_after[node]: + if re.search(err_counters_pattern, counter_name, re.I): + if int(self.rdma_stats_dict_after[node][counter_name]) > int( + self.rdma_stats_dict_before[node][counter_name] + ): + fail_test( + f'Error counter {counter_name} has gone up after training on node {node} ' + f'Before = {self.rdma_stats_dict_before[node][counter_name]}, ' + f'After = {self.rdma_stats_dict_after[node][counter_name]}' + ) + + # Compare NIC error counters; fail if any error counter increased + for node in self.ethtool_stats_dict_after.keys(): + for counter_name in self.ethtool_stats_dict_after[node]: + if re.search(err_counters_pattern, counter_name, re.I): + if int(self.ethtool_stats_dict_after[node][counter_name]) > int( + self.ethtool_stats_dict_before[node][counter_name] + ): + fail_test( + f'Error counter {counter_name} has gone up after training on node {node} ' + f'Before = {self.ethtool_stats_dict_before[node][counter_name]}, ' + f'After = {self.ethtool_stats_dict_after[node][counter_name]}' + ) + + # Scan dmesg for errors during training window + verify_dmesg_for_errors(self.orch, self.training_start_time, self.training_end_time, till_end_flag=False) + + update_test_result() diff --git a/cvs/lib/training/torchtitan/training_config_loader.py b/cvs/lib/training/torchtitan/training_config_loader.py new file mode 100644 index 000000000..364d489a7 --- /dev/null +++ b/cvs/lib/training/torchtitan/training_config_loader.py @@ -0,0 +1,210 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Training-specific config schema for TorchTitan suites (single-node and distributed). + +The framework-agnostic machinery (ContainerSpec, RuntimeSpec, placeholder +substitution, threshold file discovery) lives in `cvs.lib.utils.config_loader`. +This module holds the training half: TorchTitanSweepCombo, TorchTitanSweep, +TorchTitanVariantConfig, and load_training_variant. + +Thresholds live in a sibling *threshold.json file (not inline in result_dict). +The threshold file is discovered via the `threshold_json` field in the config or +auto-discovered as the sole *threshold.json sibling. Cell keys in the threshold +file must match the combination keys in sweep.combinations exactly. + +enforce_thresholds gates whether threshold specs are asserted in test_metric. + +Both torchtitan_single and torchtitan_distributed are covered by TorchTitanVariantConfig +via the framework field, which is a validated schema tag / config discriminator. +''' + +from __future__ import annotations + +import warnings +from collections import Counter +from typing import Any, Dict, List + +from pydantic import Field, model_validator +from typing_extensions import Literal + +from cvs.lib.utils.config_loader import ( + ContainerSpec, + _Forbid, + substitute_config, +) + + +# ---------- pydantic models (training) ---------- + + +class TorchTitanSweepCombo(_Forbid): + name: str + micro_batch_size: str + global_batch_size: str + precision: str = "" + + +def validate_sweep_selector(combo_keys, run_refs): + """The sweep-selector rule: combination keys unique, every run references one. + + Single home for this check, shared by the typed TorchTitanSweep validator + (load time) and pytest_generate_tests (collection time, which reads raw + JSON before the loader runs) so the two can never drift. + + Without it a typo'd run key is a silently-dropped cell — the sweep runs + a different matrix than the config reads. + """ + counts = Counter(combo_keys) + dupes = sorted(k for k, count in counts.items() if count > 1) + if dupes: + raise ValueError(f"duplicate sweep.combinations keys: {dupes}") + known = set(counts) + unknown = sorted(r for r in run_refs if r not in known) + if unknown: + raise ValueError(f"sweep.runs references unknown combinations: {unknown} (known: {sorted(known)})") + + +def validate_thresholds_cover_sweep( + *, + expected_cells, + thresholds, + enforce_thresholds: bool, + gated_metrics=None, +) -> None: + """Shared sweep/threshold coverage check for training variant configs. + + Checks every sweep cell has a threshold entry and no threshold key is + orphaned. Individual metrics within a cell are optional — absent specs + are skipped in test_metric (record-only for that metric). + """ + expected = set(expected_cells) + present = set(thresholds.keys()) + missing = sorted(expected - present) + extra = sorted(present - expected) + problems = [] + if missing: + problems.append(f"sweep cells with no threshold entry: {missing}") + if extra: + problems.append(f"threshold keys matching no sweep cell (typo?): {extra}") + gated = gated_metrics if gated_metrics is not None else set() + gated_keys = [f"training.{m}" for m in sorted(gated)] + gated_gaps = {} + for cell in sorted(expected & present): + specs = thresholds.get(cell) or {} + absent = [k for k in gated_keys if k not in specs] + if absent: + gated_gaps[cell] = absent + if gated_gaps: + problems.append(f"cells missing gated-metric specs: {gated_gaps}") + if problems: + msg = "threshold.json does not match the sweep matrix; " + "; ".join(problems) + if enforce_thresholds: + raise ValueError(msg) + warnings.warn(f"{msg} (enforce_thresholds=false -> record-only)", stacklevel=3) + + +class TorchTitanSweep(_Forbid): + combinations: Dict[str, TorchTitanSweepCombo] + runs: List[str] + + @model_validator(mode="after") + def _check_runs_reference_known_combos(self): + validate_sweep_selector( + list(self.combinations.keys()), + self.runs, + ) + return self + + +class ScalingBaseline(_Forbid): + tokens_per_sec_total: float = 0.0 + num_nodes: int = 1 + + +class TorchTitanVariantConfig(_Forbid): + schema_version: Literal[1] + framework: Literal["torchtitan_single", "torchtitan_distributed"] + gpu_arch: str + enforce_thresholds: bool = True + threshold_json: str = "" + scaling_baseline: ScalingBaseline = Field(default_factory=ScalingBaseline) + config: Dict[str, Any] # training knobs: torchtitan_root, nccl_*, nic_type, ... + model_params: Dict[str, Any] # model knobs: model_name, precision, tp, pp, ... + container: ContainerSpec + sweep: TorchTitanSweep + thresholds: Dict[str, Dict[str, Any]] = Field(default_factory=dict) + + def cell_key(self, combo_key: str) -> str: + """Canonical threshold lookup key for a sweep combo. + + Constructs a key from the combo's micro_batch_size, global_batch_size, + and precision — must match the top-level keys in the threshold file exactly. + """ + combo = self.sweep.combinations[combo_key] + return f"MBS={combo.micro_batch_size},GBS={combo.global_batch_size},PRECISION={combo.precision}" + + def expected_cells(self) -> List[str]: + """Return the threshold cell key for every run in sweep.runs.""" + return [self.cell_key(k) for k in self.sweep.runs] + + @model_validator(mode="after") + def _check_thresholds_cover_sweep(self): + """Every sweep cell must have a threshold entry; no metric within it is + mandatory. test_metric treats an absent ``training.*`` spec as + "don't gate this metric" (skips the assertion), so a threshold.json + is free to gate only the metrics an operator cares about. + """ + validate_thresholds_cover_sweep( + expected_cells=self.expected_cells(), + thresholds=self.thresholds, + enforce_thresholds=self.enforce_thresholds, + gated_metrics=set(), + ) + return self + + +# ---------- public API (training) ---------- + + +def _check_no_changeme(node, path="", _offenders=None): + """Recursively collect config fields whose value still contains ''. + + Collects all offending dotted paths so the caller can report them all at once. + """ + if _offenders is None: + _offenders = [] + if isinstance(node, dict): + for k, v in node.items(): + _check_no_changeme(v, f"{path}.{k}" if path else k, _offenders) + elif isinstance(node, list): + for i, v in enumerate(node): + _check_no_changeme(v, f"{path}[{i}]", _offenders) + elif isinstance(node, str) and "" in node: + _offenders.append(path) + if not path: + if _offenders: + raise ValueError(f"config has unfilled placeholder '' in: {', '.join(_offenders)}") + + +def load_training_variant(config_path, cluster_dict) -> TorchTitanVariantConfig: + """Load and validate a TorchTitan training variant config + its threshold file. + + Delegates file read, placeholder substitution, and threshold file discovery + to the generic substitute_config. The threshold file is located via the + threshold_json field in the config (relative to the config file's directory) + or auto-discovered as the sole *threshold.json sibling. + + Cell keys in the threshold file must match TorchTitanVariantConfig.cell_key() + output exactly — MBS=,GBS=,PRECISION=. A load-time + validator checks that every sweep cell has a threshold entry and no key is + orphaned. + """ + raw, thresholds = substitute_config(config_path, cluster_dict) + + _check_no_changeme(raw) + + known = {k: v for k, v in raw.items() if k in TorchTitanVariantConfig.model_fields} + known["thresholds"] = thresholds + return TorchTitanVariantConfig(**known) diff --git a/cvs/lib/unittests/test_globals.py b/cvs/lib/unittests/test_globals.py new file mode 100644 index 000000000..25f0e1a44 --- /dev/null +++ b/cvs/lib/unittests/test_globals.py @@ -0,0 +1,94 @@ +""" +test_globals.py + +Unit tests for the pssh host_logger suppression applied at cvs.lib.globals +import time. +""" + +import logging +import unittest + +import cvs.lib.globals # noqa: F401 -- imported for its host_logger suppression side effect + +HOST_LOGGER = 'pssh.host_logger' + + +class _CollectingHandler(logging.Handler): + """Records every LogRecord it is handed.""" + + def __init__(self): + super().__init__() + self.records = [] + + def emit(self, record): + self.records.append(record) + + +def _emit_host_line(): + """Emit a line the way pssh/clients/base/single.py does.""" + logging.getLogger(HOST_LOGGER).info("[%s]%s\t%s", '10.0.0.1', '', 'remote stdout line') + + +class TestHostLoggerSuppression(unittest.TestCase): + def test_handler_attached_directly_to_host_logger_receives_nothing(self): + # A handler bolted straight onto pssh.host_logger bypasses propagation + # entirely. This is what pytest's catching_logs does, so suppressing by + # detaching the logger from root is not enough -- the suppression has to + # stop the record before any handler is consulted. + handler = _CollectingHandler() + host_logger = logging.getLogger(HOST_LOGGER) + root = logging.getLogger() + orig_level = root.level + host_logger.addHandler(handler) + # host_logger sets no level of its own, so without this the INFO record + # is never created and the test would pass without exercising anything. + root.setLevel(logging.INFO) + try: + _emit_host_line() + finally: + root.setLevel(orig_level) + host_logger.removeHandler(handler) + + self.assertEqual(handler.records, []) + + def test_no_duplicate_captured_under_pytest_catching_logs(self): + # The real mechanism: _pytest.logging.catching_logs attaches its handler + # to root AND to every non-propagating logger. + try: + from _pytest.logging import catching_logs + except ImportError: # pragma: no cover - pytest is a declared dependency + self.skipTest("pytest not installed") + + handler = _CollectingHandler() + with catching_logs(handler, level=logging.INFO): + _emit_host_line() + + host_lines = [r for r in handler.records if r.name == HOST_LOGGER] + self.assertEqual(host_lines, []) + + def test_other_loggers_are_still_captured(self): + # Guards against suppressing more than the one third-party logger. + try: + from _pytest.logging import catching_logs + except ImportError: # pragma: no cover - pytest is a declared dependency + self.skipTest("pytest not installed") + + handler = _CollectingHandler() + with catching_logs(handler, level=logging.INFO): + logging.getLogger('cvs.some.module').info("kept") + + self.assertEqual([r.getMessage() for r in handler.records], ["kept"]) + + def test_filter_is_identifiable_by_name(self): + # The filter has to be recognisable in + # logging.getLogger('pssh.host_logger').filters when someone is + # debugging log routing on a live node. An anonymous lambda shows up + # there as a bare > with no hint of what installed it + # or why, so the suppression is pinned to a named callable. + installed = logging.getLogger(HOST_LOGGER).filters + names = [getattr(f, '__name__', '') for f in installed] + self.assertIn('_suppress_pssh_host_logger', names, f"no named suppression filter among {installed}") + + +if __name__ == '__main__': + unittest.main() diff --git a/cvs/lib/utils/AGENTS.md b/cvs/lib/utils/AGENTS.md new file mode 100644 index 000000000..7eeb70d64 --- /dev/null +++ b/cvs/lib/utils/AGENTS.md @@ -0,0 +1,265 @@ +# cvs/lib/utils — framework-agnostic config machinery + +**Boundary**: if every CVS suite (inference, training, ...) needs it, it belongs here. +Inference-only symbols belong in `cvs/lib/inference/utils/`; single-framework symbols in `cvs/lib//utils/`. + +--- + +## Files + +### `config_loader.py` + +#### Schema classes + +| Class | Models | Extra keys | +|---|---|---| +| `_Forbid` | Base: extra keys forbidden | — | +| `_Allow` | Base: extra keys allowed | — | +| `Paths` | Shared filesystem paths | `_Forbid` | +| `ModelSpec` | Model identity and fetch mode | `_Forbid` | +| `RuntimeSpec` | Orchestrator runtime (name + open-ended args) | `_Allow` | +| `ContainerSpec` | Container lifecycle and image | `_Forbid` | +| `BaseVariantConfig` | Framework-agnostic skeleton all suites share | `_Forbid` | + +**`Paths`** — `shared_fs: str`, `models_dir: str`, `log_dir: str`, `hf_token_file: str` + +**`ModelSpec`** — `id: str`, `remote: Literal[0, 1]` + +**`RuntimeSpec`** (`_Allow`) — `name: str`, `args: Dict[str, Any]` (defaults to `{}`). +`_Allow` because orchestrator runtime options are framework-specific. + +**`ContainerSpec`** (`_Forbid`): + +| Field | Type | Default | Meaning | +|---|---|---|---| +| `lifetime` | `Literal["no_launch", "per_run", "persistent"]` | `"per_run"` | `"no_launch"` — skip container management entirely; `"per_run"` — tear down and re-create each run; `"persistent"` — reuse an already-running container | +| `name` | `str` | required | Container name | +| `image` | `str` | required | Declared once here; no separate top-level image block | +| `runtime` | `RuntimeSpec` | required | Nested `RuntimeSpec`; its serialised form inside `container.model_dump()` is `{name, args}` — the full container dump is `{lifetime, name, image, runtime: {name, args}}` | + +**`BaseVariantConfig`** (`_Forbid`) shared fields: + +| Field | Type | Default | Notes | +|---|---|---|---| +| `schema_version` | `Literal[1]` | required | — | +| `enforce_thresholds` | `bool` | `True` | `False` → coverage failures become warnings; test runs record-only | +| `threshold_json` | `str` | required | Literal absolute path; see contract below | +| `paths` | `Paths` | required | — | +| `model` | `ModelSpec` | required | — | +| `container` | `ContainerSpec` | required | — | +| `thresholds` | `Dict[str, Dict[str, Any]]` | `{}` | Populated by the loader, not the config file | + +`BaseVariantConfig` carries one `@model_validator(mode="after")`: + +**`_check_remote_not_implemented`** — raises `NotImplementedError` when `model.remote == 1`. +Runs first (parent-class validators precede subclass validators), so a remote config fails fast +before any subclass coverage check runs on a config that will be rejected anyway. + +--- + +#### `substitute_config(config_path, cluster_dict) -> (raw_dict, thresholds)` + +- **Accepts**: path to a variant `_config.json` + a resolved cluster dict +- **`threshold_json` handling**: read as a literal absolute path from the raw (un-substituted) config + before any substitution pass runs — **no placeholder substitution of any kind is applied to it** +- **3-pass substitution** (see `docs/placeholder-substitution.md` for worked example): + 1. Cluster placeholders (`{user-id}`, etc.) resolved everywhere in the document + 2. Self-reference within the `paths` block (`{shared_fs}` expanded inside other `paths.*` values) + 3. Cross-block references (`{paths.models_dir}`, etc.) resolved anywhere in the document +- **Strips** `_`-prefixed comment keys from thresholds before returning +- **Returns**: substituted-but-**unvalidated** dict + parsed thresholds +- **Does NOT**: validate, type-coerce, or build a typed config — that is the caller's job +- **Unknown `{token}`**: left verbatim (no error; typo surfaces as a literal brace in a path) + +--- + +#### `_resolve_cluster_mapping(cluster_dict)` + +- Returns `{"user-id": }` +- Falls back to `getpass.getuser()` when `cluster_dict` has no `username` key (or it is falsy) +- This is how `{user-id}` resolves on clusters without an explicit `username` field + +--- + +### `verdict.py` + +**`ThresholdViolation(Exception)`** +- `.violations: list[str]` — all failure strings +- Exception message is the violation strings joined by newlines + +**`evaluate_all(actuals, thresholds)`** + +| Situation | Behaviour | +|---|---| +| Metric in `thresholds` but not in `actuals` | Violation string (not `KeyError`) | +| Metric present in `actuals` with value `None` | Loud violation string (not `float(None)` crash) | +| Metric present in `actuals` with a non-numeric, non-None value | Uncaught `ValueError` from `float()` — callers must ensure actuals values are numeric or `None` | +| `min_ratio` spec | `evaluate_all` injects `_actuals` into the spec dict before calling `_check_one`; callers never set `_actuals` | +| `min_ratio` — **reference** metric value is `None` | Caught inside `_check_one` (not in `evaluate_all`'s per-metric guard); `_check_one` returns a violation string when `actuals[ref_metric] is None`. The `evaluate_all` `None` guard covers only the **primary** metric being checked, not the reference metric for ratio specs. | +| Multiple failures | Raises `ThresholdViolation` listing ALL failures, not just the first | + +- `actuals`: `{metric_name: value_or_None}` +- `thresholds`: `{metric_name: spec_dict}` + +See `docs/threshold-kinds.md` for the full threshold kind reference. + +--- + +### `gpu.py` + +GPU metrics polling library. No side-effects at import time; safe to import in any suite — +inference or training. Shells out to `amd-smi metric --json` via an `Orchestrator`; no +suite-specific logic. See `docs/gpu-metrics.md` for the integration guide. + +**When to use**: add GPU utilisation rows to any suite's HTML report. +Do not copy-paste this logic — import it. + +#### Public API + +| Symbol | Kind | Purpose | +|---|---|---| +| `GPU_METRICS` | `list[tuple[str, str]]` | 5 derived metric keys + units, in display order. Iterate to register `test_gpu_metric` parametrize IDs and threshold keys. | +| `GPU_METRIC_UNITS` | `dict[str, str]` | `{key: unit}` convenience dict built from `GPU_METRICS`. | +| `capture_gpu_metrics(orch, nodes=None)` | function | One `amd-smi metric --json` exec round. Returns `{gpu.*: value_or_None}` merged snapshot. | +| `agg_readings(readings)` | function | Aggregates a list of raw snapshots → `{peak_gpu_memory_mb, gpu_compute_util_pct, gpu_bandwidth_util_pct}`. | +| `poll_gpu_metrics(orch, is_done_fn, ...)` | function | Polling loop. Returns list of raw snapshots. Never raises. | + +#### Single-node vs multi-node + +`capture_gpu_metrics` and `poll_gpu_metrics` both take an optional `nodes` parameter. + +- **`nodes=None` (default, single-node)**: `orch` must implement `.exec_on_head(cmd) -> {host: str}`. + amd-smi runs once, on the orchestrator's head node. +- **`nodes` provided (multi-node)**: `nodes` is a `list[(label, hosts)]`, where `hosts` is a + list of hostnames. `orch` must implement `.exec(cmd, hosts=hosts) -> {host: str}` — every + `Orchestrator` subclass (`BaremetalOrchestrator`, `ContainerOrchestrator`, ...) already + supports this. One `amd-smi` exec runs per `(label, hosts)` pair per poll; all nodes' GPU + entries are merged into a single snapshot before aggregation, and the last successful + per-node VRAM reading is tracked separately for the summary block. + See `cvs/lib/inference/sglang_disagg_lib.py::sglang_disagg_gpu_counts` for a role-based + usage example (prefill/decode/router/benchmark node groups). + +Do not construct raw `Pssh`/ssh handles per node — pass hostnames through `nodes` and let +`orch.exec(cmd, hosts=...)` route the call; this keeps polling orchestrator-agnostic. + +#### `poll_gpu_metrics` parameters + +| Parameter | Default | Notes | +|---|---|---| +| `orch` | — | `Orchestrator`; must have `.exec_on_head(cmd)` and, for multi-node, `.exec(cmd, hosts=...)` | +| `is_done_fn` | — | Callable returning `bool`; polling stops when it returns `True`. Runs outside the amd-smi try/except, so an exception here always propagates and is never misattributed as a polling failure. | +| `poll_interval_s` | `15` | Seconds between polls | +| `label` | `"poll"` | Log-line prefix tag | +| `log_path` | `None` | If given, writes `gpu_poll.log` to this path | +| `max_consecutive_failures` | `3` | Stops early after this many back-to-back `amd-smi` failures | +| `model_load_s` | `None` | Passed through into the summary block of `gpu_poll.log` | +| `model_load_memory_mb` | `None` | Passed through into the summary block of `gpu_poll.log` | +| `nodes` | `None` | Optional `list[(label, hosts)]` for multi-node polling; see above | + +`poll_gpu_metrics` returns the raw readings list. The caller computes the 5 derived +metrics by combining `agg_readings(readings)` with the separately-measured +`model_load_s` and `model_load_memory_mb` scalars. + +#### The 5 derived metrics and how they are computed + +| Key | Source | Aggregation | +|---|---|---| +| `peak_gpu_memory_mb` | `agg_readings` | `max(used_vram)` over polls, each poll summed across GPUs/nodes | +| `model_load_memory_mb` | caller-measured | `post_load_snap["gpu.used_vram"] - pre_load_snap["gpu.used_vram"]` | +| `model_load_s` | caller-measured | wall-clock elapsed while server starts | +| `gpu_bandwidth_util_pct` | `agg_readings` | `mean(umc_activity)` over polls, each poll averaged across GPUs/nodes | +| `gpu_compute_util_pct` | `agg_readings` | `mean(gfx_activity)` over polls, each poll averaged across GPUs/nodes | + +Store as `inf_res_dict[f"gpu.{key}"]` so a `test_gpu_metric`-style test can retrieve them. + +#### Gotchas + +- **`amd-smi` runs on the host, not in the container.** Single-node: use `orch.exec_on_head(...)`, + never `orch.exec_in_container(...)`. Multi-node: use `orch.exec(cmd, hosts=[...])` — same + host-side constraint, just targeted at a specific host subset. +- **`capture_gpu_metrics` can raise**; only `poll_gpu_metrics` guarantees never-raises. + Wrap one-shot snapshot calls in a `try/except` that returns `{}`. +- **`model_load_memory_mb` should be `None` when VRAM data is unavailable**, not `0`. + Use `... or None` after the subtraction so a missing-data case is skipped rather than + gated as a zero value. +- **`agg_readings` only returns 3 of the 5 metrics.** `model_load_memory_mb` and + `model_load_s` come from the caller's timing and snapshot code, not from the poll loop. +- **All poll readings use raw `gpu.*` keys** (e.g. `gpu.used_vram`), not derived metric + keys (e.g. `peak_gpu_memory_mb`). Do not pass raw snapshots to `evaluate_all`. +- **Multi-node degrades per label, not globally.** If `orch.exec` raises for one node in + `nodes`, that node's entries are excluded from the merged snapshot and its per-node VRAM + is `None`; other nodes' data is unaffected. + +--- + +## The boundary rule + +| Question | Answer | +|---|---| +| Does every CVS suite need it? | `cvs/lib/utils/` | +| Do only serving/inference suites need it? | `cvs/lib/inference/utils/` | +| Does only one framework (vllm, megatron, jax) need it? | `cvs/lib//utils/` | + +When in doubt: "does any other suite need this?" → move it up one layer. + +--- + +## Subclassing `BaseVariantConfig` + +Contract for new suite authors: + +**Must add:** +- `framework: Literal["your_name"]` +- `params` — your framework's CLI flags schema +- `sweep` — your sweep schema + +**Must implement:** +- `cell_key(...)` — returns a string key matching threshold.json top-level keys +- `expected_cells()` — returns a list of all cell keys the sweep produces + +**Must add:** +- A `@model_validator(mode="after")` that performs threshold-coverage checking + (equivalent to `_check_thresholds_cover_sweep` in `inferencing_config_loader.py`). + The check must cover **two axes**: + 1. **Cell coverage** — sweep cells with no threshold entry AND threshold keys + that match no sweep cell (both directions; a one-way check silently skips + orphaned threshold entries). + 2. **Gated-metric coverage** — for every cell that is present in both the sweep + and the threshold file, every member of the framework's gated-metric set must + have a spec. Without this axis, a gated metric with no spec falls through the + `spec is None` record-only branch and silently reports a green PASS with zero + assertions even under `enforce_thresholds=True`. For the vllm/inference + framework this set is `GATED_METRICS` imported from + `cvs.lib.inference.utils.vllm_parsing`; a new framework author must define an + equivalent set. + +**Validator ordering:** parent-class validators run before subclass validators. +`_check_remote_not_implemented` always fires first — do not add a base validator that +assumes a valid config before this check passes. + +**`load_variant`:** always delegate to `substitute_config` — never reimplement file-read +or substitution. After calling `substitute_config`, attach `thresholds`, then build +`YourVariantConfig(**raw)`. + +--- + +## Gotchas + +- **`threshold_json` is a literal absolute path** — not a glob, not relative to the config + file. It is read from the raw un-substituted config before Pass 1 runs, so no placeholder + substitution (not even `{user-id}`) applies to it. If your threshold path needs to vary + by user, it must be pre-resolved before being written into the config file. +- **Unknown `{token}` left verbatim** — a typo surfaces as a literal brace in a path at + runtime, not a load failure. Check paths block values after loading if substitution is + suspected to have silently failed. +- **`_Forbid` vs `_Allow`**: never loosen `_Forbid` to silence an "extra key" validation + error — add the field explicitly. +- **Validator ordering**: `BaseVariantConfig` validators run before subclass validators; + `_check_remote_not_implemented` always fires first. Do not add a subclass validator that + assumes `model.remote == 0` without relying on this ordering guarantee. +- **`_resolve_cluster_mapping` fallback**: if the running user differs from the cluster user, + verify `cluster_dict` has a `username` key; omitting it silently resolves `{user-id}` to + the local OS user. +- **`container.model_dump()` is the orchestrator contract** — serialises to + `{lifetime, name, image, runtime: {name, args}}` that `OrchestratorConfig.from_configs` + consumes; do not reshape the dict before passing it. diff --git a/cvs/lib/utils/__init__.py b/cvs/lib/utils/__init__.py new file mode 100644 index 000000000..d3438a6e8 --- /dev/null +++ b/cvs/lib/utils/__init__.py @@ -0,0 +1,4 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' diff --git a/cvs/lib/utils/config_loader.py b/cvs/lib/utils/config_loader.py new file mode 100644 index 000000000..c1198fe69 --- /dev/null +++ b/cvs/lib/utils/config_loader.py @@ -0,0 +1,224 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Framework-agnostic config machinery shared by every CVS suite. + +Holds the generic half of what used to be one monolithic loader: the +container/paths/model/image schema, the 3-pass placeholder substitution, the +`enforce_thresholds` gate carried on `BaseVariantConfig`, and the +`substitute_config` helper that reads a variant `config.json` + sibling +`*threshold.json` and resolves placeholders. A per-framework module subclasses +`BaseVariantConfig` and adds its own `Params`/`Sweep`/`cell_key` (see +`cvs.lib.inference.utils.vllm_config_loader` for the vllm flavour). + +3-pass placeholder substitution: + 1. cluster placeholders (`{user-id}`) anywhere + 2. self-reference within `paths` (e.g. `{shared_fs}`) + 3. cross-block (`{paths.models_dir}`, etc.) into the rest of the doc + +A loaded variant's `container` field (`lifetime`, `name`, `image`, `runtime`) +matches the dict shape that `cvs.core.orchestrators.factory.OrchestratorConfig` +already understands. `runtime` is a nested `RuntimeSpec`, not a flat dict; +`container.model_dump()` serialises it to the `runtime: {name, args}` shape the +factory consumes (the vllm conftest does exactly this before building the +orchestrator). The loaded variant also carries a `thresholds` field with the +parsed threshold contents. + +`model.remote=1` raises NotImplementedError -- schema is present, but the +download/resolve logic lives in cvs-dtni-v1's `resource_resolver.py` and is +out of scope for this PoC. +''' + +from __future__ import annotations + +import getpass +import json +import re +from pathlib import Path +from typing import Any, Dict + +from pydantic import BaseModel, ConfigDict, Field, model_validator +from typing_extensions import Literal + + +# ---------- pydantic models (generic) ---------- + + +class _Forbid(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class _Allow(BaseModel): + model_config = ConfigDict(extra="allow") + + +class Paths(_Forbid): + shared_fs: str + models_dir: str + log_dir: str + hf_token_file: str + + +class ModelSpec(_Forbid): + id: str + remote: Literal[0, 1] + precision: str = "" + + +class RuntimeSpec(_Allow): + name: str + args: Dict[str, Any] = Field(default_factory=dict) + + +class ContainerSpec(_Forbid): + lifetime: Literal["no_launch", "per_run", "persistent"] = "per_run" + name: str + image: str + runtime: RuntimeSpec + + +class BaseVariantConfig(_Forbid): + """The framework-agnostic skeleton of a variant config. + + Carries the fields every suite shares (schema/paths/model/image/container/ + thresholds + the enforce gate) and the remote-not-implemented guard. + Per-framework subclasses add their own `framework`/`Params`/`Sweep` and the + `cell_key`/coverage-check pair that depend on them. + """ + + schema_version: Literal[1] + # When false, the threshold-coverage gate warns instead of raising and the + # test records metrics without asserting pass/fail (record-only). Use for + # un-calibrated shapes (e.g. a throughput characterization whose published + # numbers are curves, not tabulated values). Default true keeps the gate + # strict for calibrated configs -- no regression to the remediation work. + enforce_thresholds: bool = True + threshold_json: str = "" + paths: Paths + model: ModelSpec + # The container image is declared once, on container.image (ContainerSpec). + # There is no separate top-level image block. + container: ContainerSpec + thresholds: Dict[str, Dict[str, Any]] = Field(default_factory=dict) + + # pydantic runs @model_validator(mode="after") hooks in definition order, + # parent-class hooks before subclass hooks. This remote check is intentionally + # the first to run: an unimplemented remote config fails fast + # (NotImplementedError) before any subclass's threshold-coverage check runs, + # which is meaningless for a config we are going to reject anyway. + @model_validator(mode="after") + def _check_remote_not_implemented(self): + if self.model.remote == 1: + raise NotImplementedError( + "model.remote=1 (remote model download) is not implemented in the PoC. " + "Port from cvs-dtni-v1/resource_resolver.py before enabling." + ) + return self + + +# ---------- placeholder substitution ---------- + +_PLACEHOLDER_RE = re.compile(r"\{([a-zA-Z0-9_.\-]+)\}") + + +def _walk_substitute(node, mapping): + if isinstance(node, str): + + def repl(m): + key = m.group(1) + if key in mapping: + return str(mapping[key]) + return m.group(0) + + return _PLACEHOLDER_RE.sub(repl, node) + if isinstance(node, list): + return [_walk_substitute(x, mapping) for x in node] + if isinstance(node, dict): + return {k: _walk_substitute(v, mapping) for k, v in node.items()} + return node + + +def _flatten_paths(d, prefix=""): + out = {} + for k, v in d.items(): + key = f"{prefix}.{k}" if prefix else k + if isinstance(v, dict): + out.update(_flatten_paths(v, key)) + elif isinstance(v, (str, int, float)): + out[key] = str(v) + return out + + +def _resolve_cluster_mapping(cluster_dict): + raw = cluster_dict.get("username") or "{user-id}" + user = getpass.getuser() if raw == "{user-id}" else raw + return {"user-id": user} + + +# ---------- public API (generic) ---------- + + +def substitute_config(config_path, cluster_dict): + """Read a variant config + sibling threshold file and resolve placeholders. + + Returns `(raw_dict, thresholds)`: the substituted config dict (NOT yet + validated -- the caller's per-framework `VariantConfig(**raw)` does that) + and the parsed, comment-stripped threshold dict. + + Threshold discovery supports both layouts: + - ``threshold_json`` in the config (literal path; vllm_single style), or + - a sole ``*threshold.json`` sibling next to the config (atom style). + + This is the framework-neutral body of the old `load_variant`: file read + + 3-pass substitution + threshold read. Per-framework loaders call it, attach + ``thresholds``, then build their typed config. + """ + config_path = Path(config_path) + if not config_path.is_file(): + raise FileNotFoundError(f"variant config not found: {config_path}") + + raw = json.loads(config_path.read_text()) + + threshold_json = (raw.get("threshold_json") or "").strip() + if threshold_json: + threshold_path = Path(threshold_json) + if not threshold_path.is_absolute(): + threshold_path = (config_path.parent / threshold_path).resolve() + if not threshold_path.is_file(): + raise FileNotFoundError(f"threshold_json not found: {threshold_path}") + else: + threshold_candidates = sorted(config_path.parent.glob("*threshold.json")) + if not threshold_candidates: + raise FileNotFoundError(f"no *threshold.json next to config: {config_path.parent}") + if len(threshold_candidates) > 1: + raise ValueError(f"multiple *threshold.json files next to config (ambiguous): {threshold_candidates}") + threshold_path = threshold_candidates[0] + thresholds = json.loads(threshold_path.read_text()) + + # Pass 1: cluster placeholders ({user-id}) everywhere. + cluster_map = _resolve_cluster_mapping(cluster_dict) + raw = _walk_substitute(raw, cluster_map) + + # Pass 2: self-reference within paths ({shared_fs} inside paths.*). + paths_block = raw.get("paths", {}) + if isinstance(paths_block, dict): + for _ in range(len(paths_block) + 1): + new = { + k: _walk_substitute(v, {pk: pv for pk, pv in paths_block.items() if isinstance(pv, str)}) + for k, v in paths_block.items() + } + if new == paths_block: + break + paths_block = new + raw["paths"] = paths_block + + # Pass 3: cross-block ({paths.models_dir} -> anywhere else). + flat_map = _flatten_paths({"paths": raw.get("paths", {})}) + raw = _walk_substitute(raw, flat_map) + + # Drop comment keys (e.g. "_comment") before framework/threshold validation. + raw = {k: v for k, v in raw.items() if not k.startswith("_")} + thresholds = {k: v for k, v in thresholds.items() if not k.startswith("_")} + + return raw, thresholds diff --git a/cvs/lib/utils/docs/gpu-metrics.md b/cvs/lib/utils/docs/gpu-metrics.md new file mode 100644 index 000000000..93e11de94 --- /dev/null +++ b/cvs/lib/utils/docs/gpu-metrics.md @@ -0,0 +1,374 @@ +# GPU Metrics Polling — Integration Guide + +`cvs/lib/utils/gpu.py` is a shared library that any CVS suite — inference or training — +can use to collect GPU utilisation data during a run and surface it as rows in the +HTML report. It has no suite-specific logic: it shells out to `amd-smi metric --json` +via an `Orchestrator` and parses/aggregates the result. This document explains what the +library measures and how a suite can wire it in; the exact fixture/parametrize/threshold +plumbing shown below is illustrative reference pseudocode drawn from an inference suite — +adapt it to your suite's own lifecycle-as-tests structure. + +> **Prerequisite**: this guide assumes you have completed (or are familiar with) +> the steps in `cvs/lib/inference/ADDING_A_SUITE.md`. Concepts like `cell_key`, +> `GATED_METRICS`, and `inf_res_dict` structure are defined there. + +--- + +## What it measures + +Five derived metrics are produced per run: + +| Metric key | Unit | Description | +|---|---|---| +| `gpu.peak_gpu_memory_mb` | MB | Highest VRAM used across all GPUs at any single poll during inference. Each poll sums VRAM across all GPUs on the node; this value is the max of those sums. | +| `gpu.model_load_memory_mb` | MB | VRAM delta between a snapshot taken before model load and one taken after. Represents the memory cost of loading the model weights. | +| `gpu.model_load_s` | s | Wall-clock time from server start to the post-load snapshot. | +| `gpu.gpu_bandwidth_util_pct` | % | Mean UMC (unified memory controller) activity across all GPUs, averaged over all polls taken during inference. | +| `gpu.gpu_compute_util_pct` | % | Mean GFX (shader/compute) activity across all GPUs, averaged over all polls taken during inference. | + +Each metric appears as its own row in the HTML report, with value, unit, and a +pass/fail result if a threshold is configured. + +--- + +## How polling works + +1. **Pre-load snapshot** — `capture_gpu_metrics(orch)` is called before the server + starts. Records baseline VRAM. +2. **Server start + post-load snapshot** — after the server is ready, + `capture_gpu_metrics(orch)` is called again. The VRAM delta and elapsed time give + `model_load_memory_mb` and `model_load_s`. +3. **Client phase polling** — `poll_gpu_metrics(...)` is called (either synchronously + with a backgrounded client, or from a thread with a synchronous client) and calls + `amd-smi metric --json` on the head node every `poll_interval_s` seconds + (default 15 s) until `is_done_fn()` returns `True`. +4. **Aggregation** — after the client completes, `agg_readings(readings)` reduces the + poll list to `peak_gpu_memory_mb`, `gpu_compute_util_pct`, and + `gpu_bandwidth_util_pct`. +5. **Results stored** — all five derived metrics are written into `inf_res_dict` under + `gpu.` so `test_gpu_metric` can read them. + +`amd-smi` runs on the host node, not inside the container. Single-node suites use +`orch.exec_on_head("amd-smi metric --json")`; multi-node suites pass a `nodes` list and +`gpu.py` calls `orch.exec("amd-smi metric --json", hosts=hosts)` per node instead. This +is intentional — `amd-smi` is a host-side tool and is not available inside the benchmark +container. + +--- + +## Multi-node polling + +Both `capture_gpu_metrics` and `poll_gpu_metrics` accept an optional `nodes` parameter: +a `list[(label, hosts)]`, where `hosts` is a list of hostnames. When provided, `gpu.py` +calls `orch.exec("amd-smi metric --json", hosts=hosts)` once per `(label, hosts)` pair +per poll, merges every node's GPU entries into a single aggregated snapshot (same shape +as the single-node case), and separately tracks the last successful per-node VRAM +reading for the summary block. + +```python +nodes = [ + ("prefill-0", prefill_node_list), + ("decode-0", decode_node_list), +] +poll_readings = poll_gpu_metrics( + orch, + is_done_fn=, + log_path=str(_gpu_log) if _gpu_log else None, + model_load_s=load_s, + model_load_memory_mb=load_mb, + nodes=nodes, +) +``` + +Any `Orchestrator` subclass works here since `.exec(cmd, hosts=...)` is part of the base +`Orchestrator` contract — no need to construct raw `Pssh`/ssh handles per role. See +`cvs/lib/inference/sglang_disagg_lib.py::sglang_disagg_gpu_counts` for a disaggregated +prefill/decode suite that groups nodes by role this way. + +When `nodes` is provided, log lines are tagged with `[label1+label2]` and the summary +block gains a `--- per-node vram (last reading) ---` section listing each label's most +recent successful VRAM reading (or `-` if every poll failed for that node). + +--- + +## Integrating into a suite + +### 1. Add the GPU polling block to `test__inference` + +The function signature must include `gpu_metrics_snap` (see Step 3). Wrap +`capture_gpu_metrics` in a helper that degrades gracefully if `amd-smi` is unavailable +at snapshot time — unlike `poll_gpu_metrics`, it can raise. + +**Pattern A — client is backgrounded by the framework (synchronous poll):** + +```python +import pathlib +import time +from cvs.lib.utils.gpu import GPU_METRICS, GPU_METRIC_UNITS, agg_readings, capture_gpu_metrics, poll_gpu_metrics + +def test__inference(orch, variant_config, inf_res_dict, gpu_metrics_snap, request, ...): + + def _snap(): + try: + return capture_gpu_metrics(orch) + except Exception: + return {} + + pre_snap = _snap() + t0 = time.monotonic() + # ... start server (returns immediately; framework backgrounds the client) ... + post_snap = _snap() + load_s = time.monotonic() - t0 + load_mb = ((post_snap.get("gpu.used_vram") or 0) - (pre_snap.get("gpu.used_vram") or 0)) or None + + # Write the log into the local report dir so it lands in the zip bundle. + _htmlpath = getattr(request.config.option, "htmlpath", None) + _html_dir = getattr(request.config, "_test_html_dir", "test_html") + _gpu_log = ( + pathlib.Path(_htmlpath).parent / _html_dir / "gpu_poll.log" + if _htmlpath else None + ) + + poll_readings = poll_gpu_metrics( + orch, + is_done_fn=, # e.g. job.is_client_done + log_path=str(_gpu_log) if _gpu_log else None, + model_load_s=load_s, + model_load_memory_mb=load_mb, + ) + + agg = agg_readings(poll_readings) + inf_res_dict["gpu.peak_gpu_memory_mb"] = agg.get("peak_gpu_memory_mb") + inf_res_dict["gpu.model_load_memory_mb"] = load_mb + inf_res_dict["gpu.model_load_s"] = load_s + inf_res_dict["gpu.gpu_bandwidth_util_pct"] = agg.get("gpu_bandwidth_util_pct") + inf_res_dict["gpu.gpu_compute_util_pct"] = agg.get("gpu_compute_util_pct") +``` + +**Pattern B — client runs synchronously in the main thread (thread the poll):** + +```python +import threading + + done_flag = threading.Event() + poll_readings = [] + def _poll(): + poll_readings.extend(poll_gpu_metrics( + orch, done_flag.is_set, + log_path=f"{variant_config.paths.log_dir}/gpu_poll.log", + model_load_s=load_s, + model_load_memory_mb=load_mb, + )) + poll_thread = threading.Thread(target=_poll, daemon=True) + poll_thread.start() + # ... run client synchronously ... + done_flag.set() + poll_thread.join() + # then aggregate as in Pattern A +``` + +### 2. Add `test_gpu_metric` + +`test_gpu_metric` is parametrized via `pytest_generate_tests` (see Step 4), not via a +`@pytest.mark.parametrize` decorator. The fixture parameter name is `gpu_metric` +(singular, matching the `pytest_generate_tests` branch). + +Pass the **full** per-cell actuals dict to `evaluate_all` — not just the single metric +— so that `min_ratio` threshold specs can resolve their reference metric: + +```python +from cvs.lib.utils.gpu import GPU_METRIC_UNITS +from cvs.lib.utils.verdict import ThresholdViolation, evaluate_all + +def test_gpu_metric(gpu_metric, inf_res_dict, variant_config, request): + val = inf_res_dict.get(gpu_metric) + unit = GPU_METRIC_UNITS.get(gpu_metric, "") + + request.node.user_properties.append(("metric_value", val)) + request.node.user_properties.append(("metric_unit", unit)) + + if val is None: + pytest.skip(f"{gpu_metric}: no value recorded (amd-smi unavailable or polling failed)") + + if not variant_config.enforce_thresholds: + return + + cell = variant_config.cell_key(isl, osl, concurrency) # same key used for test_metric + spec = (variant_config.thresholds.get(cell) or {}).get(gpu_metric) + if spec is None: + return # no spec → record-only + + # Pass full cell actuals so min_ratio specs can resolve their reference metric + cell_actuals = {k: inf_res_dict.get(k) for k in inf_res_dict} + try: + evaluate_all(cell_actuals, {gpu_metric: spec}) + except ThresholdViolation as exc: + pytest.fail(str(exc)) +``` + +### 3. Add `gpu_metrics_snap` fixture to `conftest.py` + +```python +@pytest.fixture(scope="module") +def gpu_metrics_snap(): + return {} +``` + +This fixture is a forward-declaration that lets `test_gpu_metric` be collected without +errors even if a future version stores intermediate state in it. + +### 4. Register `test_gpu_metric` in `pytest_collection_modifyitems` and `pytest_generate_tests` + +**Collection sort** — add `test_gpu_metric` at rank 4 alongside `test_metric`: + +```python +rank = { + "test_launch_container": 0, + "test_setup_sshd": 1, + "test_model_fetch": 2, + "test__inference": 3, + "test_metric": 4, + "test_gpu_metric": 4, # must be present; omitting → rank 99 → runs after teardown + "test_print_results_table": 5, + "test_teardown": 6, +} +``` + +**Parametrize** — add an `elif` branch to `pytest_generate_tests` in the test module. +The fixture name is `gpu_metric` (singular): + +```python +from cvs.lib.utils.gpu import GPU_METRICS + +def pytest_generate_tests(metafunc): + if "metric" in metafunc.fixturenames: + # ... your existing metric parametrize branch ... + elif "gpu_metric" in metafunc.fixturenames: + metafunc.parametrize( + "gpu_metric", + [k for k, _ in GPU_METRICS], + ids=[k for k, _ in GPU_METRICS], + ) +``` + +Without this branch `test_gpu_metric` collects zero instances and produces no HTML rows. + +### 5. Add threshold entries and update `GATED_METRICS` + +**Threshold JSON** — threshold keys use the `gpu.` prefix. For each sweep cell: + +```json +"isl1000_osl1000_conc16": { + "client.total_token_throughput": { "kind": "min_tok_s", "value": 1000 }, + "gpu.peak_gpu_memory_mb": { "kind": "max", "value": 200000 }, + "gpu.model_load_memory_mb": { "kind": "max", "value": 150000 }, + "gpu.model_load_s": { "kind": "max", "value": 300 }, + "gpu.gpu_bandwidth_util_pct": { "kind": "min", "value": 10 }, + "gpu.gpu_compute_util_pct": { "kind": "min", "value": 5 } +} +``` + +**`GATED_METRICS`** — if your `VariantConfig` subclass validates that every gated +metric has a threshold entry (the two-axis coverage check in `ADDING_A_SUITE.md` +Step 2), add all five `gpu.*` keys to your `GATED_METRICS` set: + +```python +GATED_METRICS = { + "client.total_token_throughput", + ... + "gpu.peak_gpu_memory_mb", + "gpu.model_load_memory_mb", + "gpu.model_load_s", + "gpu.gpu_bandwidth_util_pct", + "gpu.gpu_compute_util_pct", +} +``` + +Omitting them causes a silent green PASS with no assertions when `enforce_thresholds=True` +and the spec is missing. + +**First run / characterisation** — set `enforce_thresholds: false` in the suite config. +All five metrics will be collected and surfaced as HTML rows but will never cause a +test failure. Use the reported values to populate your threshold JSON, then flip +`enforce_thresholds` to `true`. + +See `docs/threshold-kinds.md` for the full threshold kind reference (`min`, `max`, +`max_ms`, `within`, `min_tok_s`, `min_ratio`). + +--- + +## The `gpu_poll.log` file + +Every run writes `gpu_poll.log` into the local HTML report directory (the same folder +as the per-test HTML files, e.g. `_html/`). Because the zip bundle includes +that directory, the log is always available in the run archive. It is also copied to +the suite's NFS `out_dir` on the head node for cluster-side inspection. + +The file contains one line per poll and a summary block: + +``` +[gpu poll 1/?] used_vram=131072 MB gfx=87% umc=74% mm=0% +[gpu poll 2/?] used_vram=132864 MB gfx=91% umc=78% mm=0% +... +[gpu poll 12/?] used_vram=132480 MB gfx=89% umc=76% mm=0% [done] + +--- summary --- +samples: 12 +peak_gpu_memory_mb: 132864 MB +model_load_memory_mb: 127418 MB +model_load_s: 148.3 s +gpu_compute_util_pct: 89.2 % +gpu_bandwidth_util_pct: 76.1 % +``` + +A poll that fails (e.g. `amd-smi` exits non-zero or returns unparseable JSON) is +logged with a `FAILED [N/max consecutive]` tag and excluded from aggregation. After +`max_consecutive_failures` (default 3) consecutive failures the loop stops early and +logs a warning. + +--- + +## Failure handling and None values + +The library never raises from `poll_gpu_metrics`. Every metric can be `None`: + +| Situation | Result | +|---|---| +| `amd-smi` fails or returns unparseable JSON | snapshot excluded from aggregation; metric may be `None` if all polls fail | +| GPU reports `"N/A"` for a field | that field is `None` in the snapshot | +| Zero valid polls | all three `agg_readings` outputs are `None` | +| Caller passes `model_load_memory_mb=None` | stored as `None`; `test_gpu_metric` should `pytest.skip` rather than fail | + +`test_gpu_metric` should always check for `None` before evaluating thresholds. +`pytest.skip` (not `pytest.fail`) is the correct response when a metric is `None` — +the metric was unavailable for this run, not a regression. + +--- + +## Gotchas + +- **`model_load_memory_mb` should be `None` when VRAM data is unavailable, not `0`.** + Use `... or None` after the subtraction (as shown in Step 1). A zero stored as `0` + gets gated against thresholds and displayed as `"0"` in the report; `None` causes + `test_gpu_metric` to skip instead. +- **`capture_gpu_metrics` can raise; `poll_gpu_metrics` never does.** Always wrap + one-shot snapshot calls in a `try/except` that returns `{}` on failure. +- **`agg_readings` returns 3 keys, not 5.** `model_load_memory_mb` and `model_load_s` + are measured by the caller and stored separately. Do not look for them in + `agg_readings` output. +- **Raw snapshot keys differ from derived metric keys.** The poll loop returns dicts + with keys like `gpu.used_vram`; the stored/threshold-gated keys use names like + `gpu.peak_gpu_memory_mb`. Do not pass raw snapshots to `evaluate_all`. +- **Threshold JSON keys use the `gpu.` prefix** (`"gpu.peak_gpu_memory_mb"`, not + `"peak_gpu_memory_mb"`). A missing prefix means the spec is never found and the + metric silently operates as record-only even when `enforce_thresholds=True`. +- **`amd-smi` runs on the host, not in the container.** Single-node polling requires + `orch.exec_on_head`; multi-node polling (via `nodes=`) requires `orch.exec(cmd, + hosts=...)` instead. Every `Orchestrator` subclass supports both — if yours doesn't, + GPU polling is not available for your suite. +- **Multi-node degrades per label, not globally.** If `orch.exec` raises for one node in + `nodes`, that node's entries are excluded from the merged snapshot and its per-node + VRAM is `None` for that poll; other nodes' data is unaffected. +- **Pass the full cell actuals dict to `evaluate_all`.** `min_ratio` threshold specs + need to resolve a reference metric from `actuals`. Passing only the single metric's + value causes a reference-resolution failure. diff --git a/cvs/lib/utils/docs/placeholder-substitution.md b/cvs/lib/utils/docs/placeholder-substitution.md new file mode 100644 index 000000000..61cb211f7 --- /dev/null +++ b/cvs/lib/utils/docs/placeholder-substitution.md @@ -0,0 +1,279 @@ +# Placeholder Substitution — Three-Pass Walkthrough + +`substitute_config` resolves placeholders in a variant config JSON in exactly three +passes. Each pass has a defined scope: which keys it reads from and which part of the +document it writes to. Understanding the order matters because a value produced by an +earlier pass becomes available as input to a later one. + +Source: `cvs/lib/utils/config_loader.py`, function `substitute_config`. + +--- + +## Worked example + +The example uses `llama31_70b_fp8_config.json` (the `w1_llama31_70b_fp8kv` variant). +The cluster dict supplied at runtime is: + +```json +{ "username": "jsmith" } +``` + +### Starting document (before any substitution) + +Only the fields that contain placeholders or that receive substituted values are shown; +the rest of the config (model, sweep, params, roles) is omitted for brevity. + +```json +{ + "threshold_json": "", + "paths": { + "shared_fs": "/home/{user-id}", + "models_dir": "{shared_fs}/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "" + }, + "container": { + "runtime": { + "args": { + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "{paths.models_dir}:/models" + ] + } + } + } +} +``` + +Note: `threshold_json` and `hf_token_file` are shown here with the sentinel value +`""` that appears in the actual file — both fields must be filled in with +real absolute paths before use. The substitution walkthrough below uses illustrative +values (`/shared/cvs/thresholds/llama31_70b_fp8_threshold.json` and +`/shared/cvs/tokens/{user-id}.token` respectively) to keep the example concrete; those +paths do not come from the real config file. + +Note: `threshold_json` must always be a fully-resolved absolute path with no +placeholders. Using `{user-id}` or any other token there causes a +`FileNotFoundError` at the literal unresolved path (see the `threshold_json` +section below). + +--- + +## Pass 1 — Cluster placeholders (`{user-id}`) everywhere + +**Mapping built:** `_resolve_cluster_mapping(cluster_dict)` reads +`cluster_dict["username"]` (`"jsmith"`), producing `{"user-id": "jsmith"}`. When +`cluster_dict` has no `username` key or the value is falsy (empty string, `None`, +etc.), `getpass.getuser()` is used as the fallback. + +**Scope:** `_walk_substitute` is called on the entire document. Every `{user-id}` token +in any string value is replaced. + +**After Pass 1:** + +```json +{ + "threshold_json": "/shared/cvs/thresholds/llama31_70b_fp8_threshold.json", + "paths": { + "shared_fs": "/home/jsmith", + "models_dir": "{shared_fs}/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "/shared/cvs/tokens/jsmith.token" + }, + "container": { + "runtime": { + "args": { + "volumes": [ + "/home/jsmith:/home/jsmith", + "{paths.models_dir}:/models" + ] + } + } + } +} +``` + +Key observations: + +- `threshold_json` contains no placeholders so it is unchanged by Pass 1. The + threshold file was already read off disk before this pass ran (see the + `threshold_json` section below). +- `paths.shared_fs` resolved to `/home/jsmith`, but `paths.models_dir` still shows + `{shared_fs}/models`. That token is a self-reference within the paths block and is + not in the cluster mapping. It is resolved in Pass 2. +- `volumes[1]` still shows `{paths.models_dir}:/models`. That cross-block reference + is resolved in Pass 3. + +--- + +## Pass 2 — Self-reference within `paths` (`{shared_fs}` inside `paths.*`) + +**Mapping built:** the keys and string values of the `paths` block at their current +state after Pass 1: + +``` +shared_fs -> "/home/jsmith" +models_dir -> "{shared_fs}/models" +log_dir -> "{shared_fs}/LOGS" +hf_token_file -> "/shared/cvs/tokens/jsmith.token" +``` + +**Scope:** `_walk_substitute` is called only on the `paths` block. The mapping is the +paths block's own key-value pairs (string values only). The substitution loop repeats +until the block stops changing, which handles chains: if `models_dir` referenced +`{log_dir}` which in turn referenced `{shared_fs}`, multiple iterations would unwind +the chain completely. The loop is capped at `len(paths_block) + 1` iterations. A cycle +in self-references (e.g. `a = "{b}"`, `b = "{a}"`) exits the cap silently, leaving +both tokens unresolved and verbatim — the same no-error-at-load-time behaviour as any +other unknown token. + +**After Pass 2:** + +```json +{ + "paths": { + "shared_fs": "/home/jsmith", + "models_dir": "/home/jsmith/models", + "log_dir": "/home/jsmith/LOGS", + "hf_token_file": "/shared/cvs/tokens/jsmith.token" + } +} +``` + +Every `paths.*` value is now fully concrete. The rest of the document is unchanged at +this point — `volumes[1]` still holds `{paths.models_dir}:/models`. + +--- + +## Pass 3 — Cross-block (`{paths.models_dir}` into the rest of the document) + +**Mapping built:** `_flatten_paths({"paths": raw.get("paths", {})})` produces dotted keys for +every leaf in the paths block: + +``` +paths.shared_fs -> "/home/jsmith" +paths.models_dir -> "/home/jsmith/models" +paths.log_dir -> "/home/jsmith/LOGS" +paths.hf_token_file -> "/shared/cvs/tokens/jsmith.token" +``` + +**Scope:** `_walk_substitute` is called on the entire document with this mapping. Any +`{paths.}` token anywhere in the document is replaced with the fully-resolved +paths value. + +Note: `{paths.*}` tokens inside the paths block itself are also resolved by Pass 3 — +Pass 2 only recognises bare keys (e.g. `{shared_fs}`), not dotted keys (e.g. +`{paths.shared_fs}`). A dotted self-reference in paths survives Pass 2 verbatim and is +then expanded by Pass 3. + +**After Pass 3 (final document):** + +```json +{ + "threshold_json": "/shared/cvs/thresholds/llama31_70b_fp8_threshold.json", + "paths": { + "shared_fs": "/home/jsmith", + "models_dir": "/home/jsmith/models", + "log_dir": "/home/jsmith/LOGS", + "hf_token_file": "/shared/cvs/tokens/jsmith.token" + }, + "container": { + "runtime": { + "args": { + "volumes": [ + "/home/jsmith:/home/jsmith", + "/home/jsmith/models:/models" + ] + } + } + } +} +``` + +`volumes[1]` is now `/home/jsmith/models:/models`. This is the value the container +orchestrator receives for the volume mount. + +--- + +## Special case: `threshold_json` and substitution + +`threshold_json` is a **literal absolute path**. The threshold file is read using the +raw (pre-substitution) string value before any pass runs: + +```python +raw = json.loads(config_path.read_text()) +threshold_path = Path(raw["threshold_json"]) # raw value, before any pass +thresholds = json.loads(threshold_path.read_text()) +# ... passes run after this point +``` + +This has two consequences: + +1. **`{user-id}` in `threshold_json` causes a FileNotFoundError.** If your + `threshold_json` is `/shared/cvs/thresholds/{user-id}/threshold.json`, the file + open is attempted at that literal string — the substitution that would resolve + `{user-id}` has not run yet. Write `threshold_json` as a fully-resolved absolute + path with no placeholders. + + Pass 1 does rewrite the `threshold_json` string value in the in-memory `raw` dict + (so after substitution the string shows the resolved path), but because the + threshold file was already opened before Pass 1 ran, that rewrite has no effect on + which file was loaded. + +2. **`{paths.*}` tokens in `threshold_json` are not substituted.** Pass 3 rewrites the + entire document, so `{paths.log_dir}` in `threshold_json` would technically be + replaced in the in-memory `raw` dict — but because the threshold file was already + read before that pass, the token in the string value is irrelevant to which file + was loaded. Keep `threshold_json` as a plain absolute path. + +--- + +## What a typo'd placeholder looks like + +If a placeholder token does not match any key in the current pass's mapping, +`_walk_substitute` leaves it verbatim — curly braces and all. No error is raised at +substitution time. + +Example: suppose `volumes` contained a typo in the models dir token: + +```json +"volumes": [ + "/home/{user-id}:/home/{user-id}", + "{paths.modles_dir}:/models" +] +``` + +After all three passes (with `user-id` resolved and `paths.models_dir` in the mapping +but `paths.modles_dir` not), the output is: + +```json +"volumes": [ + "/home/jsmith:/home/jsmith", + "{paths.modles_dir}:/models" +] +``` + +The misspelled token `{paths.modles_dir}` survives all three passes unchanged. The +container launch then receives the literal string `{paths.modles_dir}:/models` as a +volume mount argument, which Docker rejects at runtime — not at config-load time. + +The same applies to `paths` self-references. A typo'd `{sahred_fs}` in +`paths.models_dir` survives Pass 2 and propagates as a brace-wrapped string into the +paths block that feeds Pass 3. + +**Diagnostic rule:** if a mount, log path, or model path looks wrong at runtime, +inspect the `raw` dict returned by `substitute_config`. Any string value containing +a literal `{...}` is a placeholder that failed to resolve, which always indicates a +token name mismatch or a missing mapping key. + +--- + +## Summary table + +| Pass | Mapping source | Document scope | Example tokens resolved | +|------|---------------|----------------|------------------------| +| 1 | `cluster_dict["username"]` (falls back to `getpass.getuser()` when the key is absent **or the value is falsy** — empty string, `None`, etc.) | Entire document | `{user-id}` | +| 2 | `paths` block key-value pairs (string values only), iterated until stable | `paths` block only | `{shared_fs}` (any bare key name in paths can be used as a self-reference token) | +| 3 | Flattened `paths` block as dotted keys | Entire document | `{paths.shared_fs}`, `{paths.models_dir}`, `{paths.log_dir}`, `{paths.hf_token_file}` | + +Unknown tokens survive all passes verbatim. No error is raised at substitution time. diff --git a/cvs/lib/utils/docs/threshold-kinds.md b/cvs/lib/utils/docs/threshold-kinds.md new file mode 100644 index 000000000..b3562c104 --- /dev/null +++ b/cvs/lib/utils/docs/threshold-kinds.md @@ -0,0 +1,356 @@ +# Threshold Kinds Reference + +Full reference for every threshold kind understood by `_check_one` and +`evaluate_all` in `cvs/lib/utils/verdict.py`. + +--- + +## How thresholds are evaluated + +`evaluate_all(actuals, thresholds)` iterates over every key in `thresholds`. +Before calling `_check_one` on a spec, it handles two conditions centrally: + +- **Metric missing from actuals** — appends + `"{metric}: missing from actuals"` as a violation and skips `_check_one`. +- **Metric value is `None`** — appends + `"{metric}: value is None (metric unavailable for this run)"` as a violation + and skips `_check_one`. This prevents a `float(None)` TypeError and surfaces + the problem explicitly. + +> **Note:** only `None` and missing keys are handled centrally. Any other +> non-float-convertible value — for example, a string `"N/A"` or `"error"` +> produced by a metric-extraction bug — passes both guards and reaches +> `_check_one`, where `_to_float` raises a raw `ValueError`, not a +> `ThresholdViolation`. Because `evaluate_all` has no try/except around +> `_check_one`, that `ValueError` propagates uncaught and breaks out of the +> violations-collection loop entirely. Callers that may populate `actuals` from +> untrusted or partially-failed data should coerce or validate metric values to +> `float`-or-`None` before passing to `evaluate_all`. + +After these guards, `evaluate_all` calls `_check_one(metric, actual, spec)`. +If `_check_one` returns a truthy (non-empty, non-`None`) string, that string is +added to the violation list. Note: `_check_one` must return `None` (not `""`) +on the passing path — the collector uses `if v:` rather than +`if v is not None`, so an empty string would be silently dropped. All violations are collected before raising; `ThresholdViolation` +carries the full list in `.violations` and its message is the newline-joined +string of all of them. + +--- + +## Kinds + +### `min` + +**JSON shape** + +```json +{ "kind": "min", "value": } +``` + +| Field | Type | Required | +|---------|--------|----------| +| `kind` | string | yes | +| `value` | number | yes | + +**Comparison** + +Passes when `actual >= value`. Fails when `actual < value`. + +**Failure message** + +``` +{metric}: actual {actual} < min {target} +``` + +**When to use** + +Use for dimensionless or mixed-unit lower bounds where the unit is either +implicit from the metric name or irrelevant to the failure message. Examples: +token counts, request counts, dimensionless ratios that do not already have a +dedicated kind. Prefer `min_tok_s` when the metric is a token-per-second +throughput — it produces a more readable failure message with explicit units. + +--- + +### `max` + +**JSON shape** + +```json +{ "kind": "max", "value": } +``` + +| Field | Type | Required | +|---------|--------|----------| +| `kind` | string | yes | +| `value` | number | yes | + +**Comparison** + +Passes when `actual <= value`. Fails when `actual > value`. + +**Failure message** + +``` +{metric}: actual {actual} > max {target} +``` + +**When to use** + +Use for upper bounds on metrics whose unit is not milliseconds. The canonical +case is `failed` (failed request count): a milliseconds suffix in the message +would be a unit lie. Use `max_ms` when the metric is a latency in milliseconds. +The comparison logic is identical to `max_ms`; only the failure message differs. + +--- + +### `max_ms` + +**JSON shape** + +```json +{ "kind": "max_ms", "value": } +``` + +| Field | Type | Required | +|---------|--------|----------| +| `kind` | string | yes | +| `value` | number | yes | + +**Comparison** + +Passes when `actual <= value`. Fails when `actual > value`. + +**Failure message** + +``` +{metric}: actual {actual} ms > max {target} ms +``` + +**When to use** + +Use for latency upper bounds where the metric is expressed in milliseconds (TTFT, +TPOT, E2EL, ITL). The `ms` suffix in both slots of the failure message makes the +unit explicit. Use `max` instead when the metric is not a time measurement. + +--- + +### `within` + +**JSON shape** + +```json +{ "kind": "within", "value": , "tolerance_pct": } +``` + +| Field | Type | Required | +|-----------------|--------|----------| +| `kind` | string | yes | +| `value` | number | yes | +| `tolerance_pct` | number | yes | + +**Comparison** + +Computes an acceptable band: + +``` +lo = value * (1 - tolerance_pct / 100.0) +hi = value * (1 + tolerance_pct / 100.0) +``` + +Passes when `lo <= actual <= hi`. Fails otherwise. + +> **Gotcha — `value` must be positive.** A negative `value` inverts the band: +> for example, `value = -100` with `tolerance_pct = 10` yields +> `lo = -90` and `hi = -110`, so `lo > hi` and the test `lo <= actual <= hi` +> can never pass. Every check fails with the normal outside-band message, +> giving no hint that the spec itself is the problem. There is no guard in +> `_check_one` or `evaluate_all` against this. +> +> A `value` of `0` collapses the band to a single point (`lo = hi = 0`); only +> `actual == 0` passes. + +**Failure message** + +``` +{metric}: actual {actual} outside {target} ±{pct}% +``` + +**When to use** + +Use when the acceptable range is symmetric around a target value and you want +to express the tolerance as a percentage rather than an absolute bound. Useful +for stability metrics or regressions against a known baseline where ±N% drift +is acceptable. Prefer `min` or `max` when the bound is one-sided. + +--- + +### `min_tok_s` + +**JSON shape** + +```json +{ "kind": "min_tok_s", "value": } +``` + +| Field | Type | Required | +|---------|--------|----------| +| `kind` | string | yes | +| `value` | number | yes | + +**Comparison** + +Passes when `actual >= value`. Fails when `actual < value`. + +**Failure message** + +``` +{metric}: actual {actual} tok/s < min {target} tok/s +``` + +**When to use** + +Use for token throughput lower bounds (`client.total_token_throughput`, +`client.output_throughput`, `client.per_gpu_throughput`). Functionally identical +to `min` but annotates `tok/s` in both slots of the failure message. Use `min` +for lower bounds on metrics that are not token-per-second rates. + +--- + +### `min_ratio` + +**JSON shape** + +```json +{ "kind": "min_ratio", "value": , "reference": "" } +``` + +| Field | Type | Required | +|-------------|--------|----------| +| `kind` | string | yes | +| `value` | number | yes | +| `reference` | string | yes | + +**Comparison** + +Computes `observed = actual / actuals[reference]` and passes when +`observed >= value`. Fails when `observed < value`. + +`reference` names another metric that must also appear in `actuals`. The +observed value of that metric is used as the denominator. + +**Failure message (ratio below minimum)** + +``` +{metric}: observed ratio {observed:.3f} < min {ratio} (vs {ref_metric}) +``` + +**When to use** + +Use when the threshold for one metric is expressed as a fraction of another +metric from the same cell run. Example: asserting that `client.output_throughput` +is at least 0.8 of `client.total_token_throughput`. This avoids hard-coding an +absolute number that would need to be recalibrated every time the reference +metric changes with hardware or model configuration. + +Do not use `min_ratio` when an absolute lower bound suffices — the ratio +interpretation adds coupling between two metrics and the failure message is +harder to read than a plain `min` or `min_tok_s` violation. + +--- + +#### The `_actuals` injection mechanism + +Callers of `evaluate_all` never set `_actuals` in a spec dict. `evaluate_all` +injects it automatically for every `min_ratio` spec: + +```python +spec_with_actuals = dict(spec) +if spec.get("kind") == "min_ratio": + spec_with_actuals["_actuals"] = actuals +v = _check_one(metric, actuals[metric], spec_with_actuals) +``` + +Inside `_check_one`, `_actuals` is read with `.get("_actuals", {})` to look up +the reference metric's value at check time. This means: + +- The threshold JSON never contains `_actuals`. +- The reference metric is resolved at evaluation time from the same `actuals` + dict that holds the primary metric's value. +- Passing all cell actuals to `evaluate_all` (not just the single metric being + asserted) is required for `min_ratio` to work. Inference suite `test_metric` + hooks (e.g. `cvs/tests/inference/vllm/vllm_single.py`) call + `evaluate_all(actuals, {full: spec})` with the full per-host actuals dict for + exactly this reason. + +--- + +#### `min_ratio` failure modes + +There are four failure conditions distinct from the ratio comparison itself. +Each produces its own violation string and short-circuits before the ratio is +computed. + +**Reference metric missing from actuals** + +Triggered when the metric named by `reference` is not a key in the `actuals` +dict at all (e.g., the key was never populated, or has a typo in `reference`). + +``` +{metric}: reference metric '{ref_metric}' missing from actuals +``` + +**Reference metric is `None`** + +Triggered when the reference metric key exists in `actuals` but its value is +`None`. This happens when a derived metric could not be computed for the run +(e.g., a zero-divisor upstream in `to_client_metrics`). + +``` +{metric}: reference '{ref_metric}' is None (metric unavailable for this run) +``` + +**Reference metric is zero** + +Triggered when `float(actuals[ref_metric]) == 0`. Division by zero is blocked +explicitly before the ratio is computed. + +``` +{metric}: reference '{ref_metric}' is 0; cannot compute ratio +``` + +**Reference metric is non-numeric and non-`None`** + +Triggered when `actuals[ref_metric]` is neither `None` nor float-convertible +(e.g., the string `"error"` produced by a metric-extraction bug). `_to_float` +raises `ValueError` uncaught; `evaluate_all` has no `try/except` around +`_check_one`, so the exception propagates and breaks out of the +violations-collection loop. Callers must coerce reference metric values to +`float`-or-`None` before calling `evaluate_all`. + +--- + +## Unknown kind + +If `kind` does not match any of the six strings above, `_check_one` returns: + +``` +{metric}: unknown threshold kind '{kind}' +``` + +This surfaces as a violation collected by `evaluate_all`. There is no exception +at parse time — the error only appears when `evaluate_all` runs against a spec +with an unrecognised kind. + +--- + +## Quick reference + +| Kind | Bound | Unit in message | Required fields beyond `kind` | +|-------------|---------------|-----------------|---------------------------------| +| `min` | lower | none | `value` | +| `max` | upper | none | `value` | +| `max_ms` | upper | `ms` | `value` | +| `within` | lower + upper | none | `value`, `tolerance_pct` | +| `min_tok_s` | lower | `tok/s` | `value` | +| `min_ratio` | lower (ratio) | ratio (3 d.p.) | `value`, `reference` | diff --git a/cvs/lib/utils/gpu.py b/cvs/lib/utils/gpu.py new file mode 100644 index 000000000..cd6f10894 --- /dev/null +++ b/cvs/lib/utils/gpu.py @@ -0,0 +1,533 @@ +'''Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +from __future__ import annotations + +import getpass +import json +import logging +import pathlib +import re +import shlex +from dataclasses import dataclass + +# Sentinel line delimiting per-iteration amd-smi chunks in the remote poller's +# output file (raw amd-smi --json output is multi-line/pretty-printed, not NDJSON). +_RECORD_SEP = "===GPU_POLL_RECORD_SEP===" + +# Human-readable derived metrics exposed as HTML rows (one row per entry per cell). +# These are computed by the calling suite from the raw amd-smi snapshots and stored +# under "gpu." keys in inf_res_dict. +GPU_METRICS: list[tuple[str, str]] = [ + ("peak_gpu_memory_mb", "MB"), + ("model_load_memory_mb", "MB"), + ("model_load_s", "s"), + ("gpu_bandwidth_util_pct", "%"), + ("gpu_compute_util_pct", "%"), +] +GPU_METRIC_UNITS: dict[str, str] = {k: u for k, u in GPU_METRICS} + +# Raw amd-smi field keys emitted by parse_gpu_metrics(). Not used as test rows. +_RAW_GPU_FIELDS: list[tuple[str, str]] = [ + ("gfx_activity", "%"), + ("umc_activity", "%"), + ("mm_activity", "%"), + ("total_vram", "MB"), + ("used_vram", "MB"), + ("free_vram", "MB"), + ("energy_j", "J"), +] +_RAW_GPU_FIELD_UNITS: dict[str, str] = {k: u for k, u in _RAW_GPU_FIELDS} + + +def _safe_get(d, *keys, default=None): + """Navigate nested dicts safely; return default on missing key or 'N/A' value.""" + cur = d + for key in keys: + if not isinstance(cur, dict): + return default + cur = cur.get(key, default) + if cur is default: + return default + if cur == "N/A": + return default + return cur + + +def parse_usage(gpu_entry: dict) -> dict: + """Extract activity metrics from one GPU entry dict. + + Returns: {"gpu.gfx_activity", "gpu.umc_activity", "gpu.mm_activity"} + Values are int or None; never raises. + """ + fields = ("gfx_activity", "umc_activity", "mm_activity") + result = {} + for field in fields: + val = _safe_get(gpu_entry, "usage", field, "value") + result[f"gpu.{field}"] = val + return result + + +def parse_mem_usage(gpu_entry: dict) -> dict: + """Extract memory usage metrics from one GPU entry dict. + + Returns: {"gpu.total_vram", "gpu.used_vram", "gpu.free_vram"} + Values are int or None; never raises. + """ + fields = ("total_vram", "used_vram", "free_vram") + result = {} + for field in fields: + val = _safe_get(gpu_entry, "mem_usage", field, "value") + result[f"gpu.{field}"] = val + return result + + +def parse_energy(gpu_entry: dict) -> dict: + """Extract energy consumption from one GPU entry dict. + + Returns: {"gpu.energy_j"} + Value is float or None; never raises. + """ + val = _safe_get(gpu_entry, "energy", "total_energy_consumption", "value") + if val is not None: + val = float(val) + return {"gpu.energy_j": val} + + +def parse_gpu_metrics(raw: list) -> dict: + """Aggregate all GPU entries from one host's amd-smi --json output. + + raw: the parsed JSON list (one dict per GPU per host). + Activity metrics (%) -> averaged across GPUs (only non-None values counted). + Memory / energy metrics -> summed across GPUs (only non-None values counted). + Empty/missing -> all None. + """ + all_none = {f"gpu.{k}": None for k, _u in _RAW_GPU_FIELDS} + if not raw: + return all_none + + activity_keys = ("gpu.gfx_activity", "gpu.umc_activity", "gpu.mm_activity") + vram_keys = ("gpu.total_vram", "gpu.used_vram", "gpu.free_vram") + energy_key = "gpu.energy_j" + + # Accumulators: sum and count per field (None excluded from both) + activity_sums: dict[str, float] = {k: 0.0 for k in activity_keys} + activity_counts: dict[str, int] = {k: 0 for k in activity_keys} + vram_sums: dict[str, int | None] = {k: None for k in vram_keys} + energy_sum: float | None = None + + for entry in raw: + usage = parse_usage(entry) + mem = parse_mem_usage(entry) + eng = parse_energy(entry) + + for key in activity_keys: + val = usage[key] + if val is not None: + activity_sums[key] += val + activity_counts[key] += 1 + + for key in vram_keys: + val = mem[key] + if val is not None: + if vram_sums[key] is None: + vram_sums[key] = val + else: + vram_sums[key] += val + + e = eng[energy_key] + if e is not None: + if energy_sum is None: + energy_sum = e + else: + energy_sum += e + + result = {} + for key in activity_keys: + count = activity_counts[key] + result[key] = (activity_sums[key] / count) if count > 0 else None + + for key in vram_keys: + result[key] = vram_sums[key] + + result[energy_key] = energy_sum + return result + + +def _try_parse(text: str) -> list: + """Parse JSON text; return [] on empty/None/invalid JSON or non-list result. + + Accepts both bare-list format and the {"gpu_data": [...]} envelope that + amd-smi metric --json emits on ROCm 6.x nodes. + """ + if not text: + return [] + try: + parsed = json.loads(text) + except (json.JSONDecodeError, ValueError, TypeError): + return [] + if isinstance(parsed, dict): + parsed = parsed.get("gpu_data", []) + if not isinstance(parsed, list): + return [] + return parsed + + +def capture_gpu_metrics(orch, nodes=None, timeout_s=None) -> dict: + """One amd-smi exec on the node(s). Returns flat {gpu.* metrics} dict. + + amd-smi runs fine from inside the benchmark container -- GPU device + files (/dev/kfd, /dev/dri) are passed through, so this uses the same + orch.exec_on_head()/orch.exec() calls as every other command in the + suite (server launch, client run, log tailing, etc.), with no special + host-vs-container routing needed. + + Single-node (nodes=None): orch must have .exec_on_head(cmd) -> {host: str}. + Multi-node (nodes provided, incl. []): nodes is a list of (label, hosts) + pairs where hosts is a list of hostnames passed to + orch.exec(cmd, hosts=hosts) -> {host: str}. nodes=[] is a valid + "zero nodes" case: no exec call is made and all fields come back None, the + same no-op result an empty raw list produces. All nodes' GPU entries are + merged before aggregation. Return type is identical in both cases. + + timeout_s: optional timeout (seconds) passed through to orch.exec/ + exec_on_head. None means no timeout (blocks until the remote call + returns), matching this function's historical behavior. + + Exceptions from exec calls (including a timeout firing) propagate to the + caller. + """ + all_entries = [] + if nodes is None: + kwargs = {"timeout": timeout_s} if timeout_s is not None else {} + out = orch.exec_on_head("amd-smi metric --json", print_console=False, **kwargs) + for _host, text in out.items(): + all_entries.extend(_try_parse(text)) + else: + kwargs = {"timeout": timeout_s} if timeout_s is not None else {} + for _label, hosts in nodes: + out = orch.exec("amd-smi metric --json", hosts=hosts, print_console=False, **kwargs) + for _host, text in out.items(): + all_entries.extend(_try_parse(text)) + return parse_gpu_metrics(all_entries) + + +def _mean(values: list) -> "float | None": + vals = [v for v in values if v is not None] + return sum(vals) / len(vals) if vals else None + + +def agg_readings(readings: list) -> dict: + """Aggregate poll readings into derived metrics. + Returns dict with peak_gpu_memory_mb, gpu_compute_util_pct, gpu_bandwidth_util_pct. + Any metric is None if no valid readings exist for it. + + Readings are raw snapshot dicts from capture_gpu_metrics (keys use gpu.* prefix). + """ + used_vrams = [r.get("gpu.used_vram") for r in readings if r.get("gpu.used_vram") is not None] + gfx_vals = [r.get("gpu.gfx_activity") for r in readings if r.get("gpu.gfx_activity") is not None] + umc_vals = [r.get("gpu.umc_activity") for r in readings if r.get("gpu.umc_activity") is not None] + return { + "peak_gpu_memory_mb": max(used_vrams) if used_vrams else None, + "gpu_compute_util_pct": _mean(gfx_vals), + "gpu_bandwidth_util_pct": _mean(umc_vals), + } + + +def _node_label_tag(nodes) -> str: + """Return '+'-joined node labels for log line tagging, or empty string.""" + if not nodes: + return "" + return "[" + "+".join(lbl for lbl, _hosts in nodes) + "] " + + +def _capture_multi_node(orch, nodes, timeout_s=None) -> "tuple[dict, dict[str, int | None]]": + """One amd-smi exec per (label, hosts) pair. + + Returns (merged_snapshot, per_node_vram) computed from a single exec round: + merged_snapshot is parse_gpu_metrics() over every node's GPU entries combined + (same shape as capture_gpu_metrics), per_node_vram is {label: used_vram_mb}. + + Degrades per label: if orch.exec raises (including a timeout_s + firing) for a node, that label's entries are excluded from the merge and + its per-node VRAM is None. + """ + all_entries = [] + per_node_vram: "dict[str, int | None]" = {} + kwargs = {"timeout": timeout_s} if timeout_s is not None else {} + for label, hosts in nodes: + try: + out = orch.exec("amd-smi metric --json", hosts=hosts, print_console=False, **kwargs) + node_entries = [] + for _host, text in out.items(): + node_entries.extend(_try_parse(text)) + all_entries.extend(node_entries) + snap = parse_gpu_metrics(node_entries) + per_node_vram[label] = snap.get("gpu.used_vram") + except Exception: + per_node_vram[label] = None + return parse_gpu_metrics(all_entries), per_node_vram + + +@dataclass +class GpuPollerHandle: + """Handle returned by start_gpu_poller; opaque to callers other than + passing it back into stop_and_collect_gpu_poller.""" + + run_id: str + marker: str + nodes: "list[str] | None" + paths: "str | dict[str, str]" + + +def _sanitize_run_id(run_id: str) -> str: + return re.sub(r"[^A-Za-z0-9_.-]", "_", run_id) + + +def _poller_script(marker: str, poll_interval_s: float, max_iterations: int) -> str: + log_path = f"/tmp/{marker}.log" + return ( + "#!/bin/bash\n" + f"for i in $(seq 1 {max_iterations}); do\n" + f" amd-smi metric --json >> {shlex.quote(log_path)} 2>/dev/null\n" + f" echo {shlex.quote(_RECORD_SEP)} >> {shlex.quote(log_path)}\n" + f" sleep {poll_interval_s}\n" + "done\n" + ) + + +def start_gpu_poller( + orch, + run_id: str, + poll_interval_s: float = 15, + nodes: "list[str] | None" = None, + hard_cap_s: float = 14400, +) -> GpuPollerHandle: + """Launch a detached remote background script that repeatedly snapshots + amd-smi metrics to a file on each node, avoiding a second OS thread + sharing the orchestrator's SSH transport with the main polling thread. + + run_id: arbitrary string (e.g. a pytest node id); sanitized into the + marker/file name. poll_interval_s: seconds between amd-smi calls. + nodes: None for single-node (head only, via orch.exec_on_head); a list + of hostnames for multi-node (one script launched per host via + orch.exec(cmd, hosts=[host])). hard_cap_s: orphan-safety backstop -- the + remote script self-terminates after hard_cap_s // poll_interval_s + iterations even if stop_and_collect_gpu_poller is never called. + + The marker (and therefore the remote /tmp script/log paths) is scoped by + the local SSH user in addition to run_id, so two users polling on the + same shared node under a similarly-named run_id never collide on the + same /tmp path -- see the file-collision note from the Fremont conda + incident. The log file is also truncated at launch (not just appended + to) so a leftover log from a prior crashed run under the same marker + can't leak old readings into this run. + + Raises if the launch exec call(s) raise. + """ + sanitized = _sanitize_run_id(run_id) + marker = f"cvs_gpu_poll_{getpass.getuser()}_{sanitized}" + max_iterations = int(hard_cap_s // poll_interval_s) + script = _poller_script(marker, poll_interval_s, max_iterations) + script_path = f"/tmp/{marker}.sh" + log_path = f"/tmp/{marker}.log" + write_cmd = "bash -c " + shlex.quote( + f"printf '%s' {shlex.quote(script)} > {script_path} && : > {shlex.quote(log_path)}" + ) + launch_cmd = "bash -c " + shlex.quote(f"nohup bash {script_path} > /dev/null 2>&1 &") + + if nodes is None: + orch.exec_on_head(write_cmd) + orch.exec_on_head(launch_cmd) + paths: "str | dict[str, str]" = f"/tmp/{marker}.log" + else: + for host in nodes: + orch.exec(write_cmd, hosts=[host]) + orch.exec(launch_cmd, hosts=[host]) + paths = {host: f"/tmp/{marker}.log" for host in nodes} + + return GpuPollerHandle(run_id=sanitized, marker=marker, nodes=nodes, paths=paths) + + +def _split_chunks(text: str) -> list: + """Split raw poller-log text on _RECORD_SEP, stripping exactly one + well-terminated trailing phantom chunk if the file ends with the + separator.""" + if not text: + return [] + chunks = text.split(_RECORD_SEP) + if chunks and chunks[-1].strip() == "": + chunks = chunks[:-1] + return chunks + + +def stop_and_collect_gpu_poller( + orch, + handle: GpuPollerHandle, + log_path=None, + model_load_s=None, + model_load_memory_mb=None, +) -> list: + """Stop the remote poller launched by start_gpu_poller and collect its + readings. + + Never raises for orch-transport failures (the stop broadcast, the file + read-back, or the remote cleanup) -- those are caught, logged, and + degrade the affected host's contribution rather than propagating, so + this is safe to call from a `finally` block during in-flight exception + handling. Returns a list of raw snapshot dicts in the same shape + capture_gpu_metrics() returns (failed/malformed polls excluded). Writes + a summary block (compatible with agg_readings()) to log_path if given. + + Removes the remote script/log files (under handle.marker) after + reading them back, so no per-run file is left behind in the node's + shared /tmp -- see the file-collision note from the Fremont conda + incident. + """ + log = logging.getLogger(__name__) + pkill_cmd = "bash -c " + shlex.quote(f"pkill -f {handle.marker} || true") + try: + if handle.nodes is None: + orch.exec_on_head(pkill_cmd) + else: + orch.exec(pkill_cmd, hosts=handle.nodes) + except Exception as exc: + log.warning("stop_and_collect_gpu_poller: pkill broadcast failed: %s", exc) + + log_lines: list = [] + node_tag = _node_label_tag([(h, [h]) for h in handle.nodes] if handle.nodes else None) + + if handle.nodes is None: + text = None + try: + out = orch.exec_on_head(f"cat {shlex.quote(handle.paths)}", print_console=False) + text = next(iter(out.values()), "") + except Exception as exc: + log.warning("stop_and_collect_gpu_poller: read-back failed: %s", exc) + text = "" + + script_path = f"/tmp/{handle.marker}.sh" + rm_cmd = "bash -c " + shlex.quote(f"rm -f {shlex.quote(script_path)} {shlex.quote(handle.paths)}") + try: + orch.exec_on_head(rm_cmd) + except Exception as exc: + log.warning("stop_and_collect_gpu_poller: remote cleanup failed: %s", exc) + + chunks = _split_chunks(text) + poll_n = len(chunks) + readings: list = [] + for i, chunk in enumerate(chunks, start=1): + entries = _try_parse(chunk) + if not entries: + log_lines.append(f"[gpu poll {i}/{poll_n}] FAILED: empty/malformed chunk (skipped)") + continue + snap = parse_gpu_metrics(entries) + readings.append(snap) + used = snap.get("gpu.used_vram") + gfx = snap.get("gpu.gfx_activity") + umc = snap.get("gpu.umc_activity") + mm = snap.get("gpu.mm_activity") + log_lines.append(f"[gpu poll {i}/{poll_n}] used_vram={used} MB gfx={gfx}% umc={umc}% mm={mm}%") + n_failed = poll_n - len(readings) + node_last_vram: "dict[str, int | None]" = {} + else: + host_chunks: "dict[str, list]" = {} + script_path = f"/tmp/{handle.marker}.sh" + for host in handle.nodes: + path = handle.paths[host] if isinstance(handle.paths, dict) else handle.paths + text = "" + try: + out = orch.exec(f"cat {shlex.quote(path)}", hosts=[host], print_console=False) + text = next(iter(out.values()), "") + except Exception as exc: + log.warning("stop_and_collect_gpu_poller: read-back failed for %s: %s", host, exc) + text = "" + host_chunks[host] = _split_chunks(text) + + rm_cmd = "bash -c " + shlex.quote(f"rm -f {shlex.quote(script_path)} {shlex.quote(path)}") + try: + orch.exec(rm_cmd, hosts=[host]) + except Exception as exc: + log.warning("stop_and_collect_gpu_poller: remote cleanup failed for %s: %s", host, exc) + + n_rounds = max((len(v) for v in host_chunks.values()), default=0) + readings = [] + node_last_vram = {host: None for host in handle.nodes} + n_failed = 0 + for i in range(n_rounds): + round_entries: list = [] + round_hosts: list = [] + for host in handle.nodes: + chunks = host_chunks[host] + if i >= len(chunks): + continue + entries = _try_parse(chunks[i]) + if entries: + round_entries.extend(entries) + round_hosts.append(host) + if not round_entries: + n_failed += 1 + log_lines.append(f"[gpu poll {i + 1}/{n_rounds}] {node_tag}FAILED: all nodes malformed (skipped)") + continue + snap = parse_gpu_metrics(round_entries) + readings.append(snap) + for host in round_hosts: + host_entries = _try_parse(host_chunks[host][i]) + host_snap = parse_gpu_metrics(host_entries) + vram = host_snap.get("gpu.used_vram") + if vram is not None: + node_last_vram[host] = vram + used = snap.get("gpu.used_vram") + gfx = snap.get("gpu.gfx_activity") + umc = snap.get("gpu.umc_activity") + mm = snap.get("gpu.mm_activity") + log_lines.append( + f"[gpu poll {i + 1}/{n_rounds}] {node_tag}used_vram={used} MB gfx={gfx}% umc={umc}% mm={mm}%" + ) + poll_n = n_rounds + + agg = agg_readings(readings) + failed_note = f" ({n_failed} failed, excluded)" if n_failed else "" + peak = agg.get("peak_gpu_memory_mb") + compute = agg.get("gpu_compute_util_pct") + bw = agg.get("gpu_bandwidth_util_pct") + ml_mem = f"{model_load_memory_mb:.0f}" if model_load_memory_mb is not None else "-" + ml_s = f"{model_load_s:.1f}" if model_load_s is not None else "-" + compute_s = f"{compute:.1f}" if compute is not None else "-" + bw_s = f"{bw:.1f}" if bw is not None else "-" + peak_s = f"{peak:.0f}" if peak is not None else "-" + + summary_lines = [ + "", + "--- summary ---", + f"samples: {poll_n}{failed_note}", + f"peak_gpu_memory_mb: {peak_s} MB", + f"model_load_memory_mb: {ml_mem} MB", + f"model_load_s: {ml_s} s", + f"gpu_compute_util_pct: {compute_s} %", + f"gpu_bandwidth_util_pct: {bw_s} %", + ] + if handle.nodes: + summary_lines.append("--- per-node vram (last reading) ---") + for host in handle.nodes: + vram = node_last_vram.get(host) + vram_s = f"{vram}" if vram is not None else "-" + summary_lines.append(f"node_vram_mb [{host}]: {vram_s} MB") + log_lines.extend(summary_lines) + + if log_path is not None: + try: + pathlib.Path(log_path).write_text("\n".join(log_lines) + "\n") + except Exception as exc: + log.warning("stop_and_collect_gpu_poller: failed to write log %s: %s", log_path, exc) + + log.info( + "stop_and_collect_gpu_poller: %d readings (%d failed) | peak_vram=%s MB compute=%s%% bw=%s%%", + len(readings), + n_failed, + peak_s, + compute_s, + bw_s, + ) + return readings diff --git a/cvs/lib/utils/ib_discovery.py b/cvs/lib/utils/ib_discovery.py new file mode 100644 index 000000000..455a6773e --- /dev/null +++ b/cvs/lib/utils/ib_discovery.py @@ -0,0 +1,231 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +InfiniBand HCA discovery via ibv_devinfo or /sys/class/infiniband fallback. + +Shared across suites (vllm, rccl, inferencemax). Infrastructure step, not +a benchmark step: topology is stable within a run and should be probed once. +''' + +from __future__ import annotations + +import re +import shlex + +from cvs.lib import globals + +log = globals.log + +_IB_HCA_NETDEV_RE = re.compile(r"^mlx5_\d+$", re.I) +_HCA_NAME_RE = re.compile(r"^(mlx5_\d+|rdma\d+|rocep\w+|bnxt_\w+)$", re.I) +_NETDEV_NAME_RE = re.compile(r"^[a-zA-Z0-9_.:-]{1,64}$") +_INVALID_NETDEV_MARKERS = ( + "command not found", + "syntax error", + "bash:", + "no such file", + "cannot open", +) + +_SYSFS_CMD = "ls /sys/class/infiniband/ 2>/dev/null | tr '\\n' ' '" +_IBVDEVINFO_CMD = "ibv_devinfo -l 2>/dev/null" + + +def _topology_exec(orch, cmd, hosts=None): + """Run topology probes on the host OS when the orchestrator is container-backed.""" + host_exec = getattr(orch, "exec_on_host", None) + if callable(host_exec): + return host_exec(cmd, hosts=hosts) + return orch.exec(cmd, hosts=hosts) + + +def _parse_sysfs(output: str) -> list[str]: + return [tok for tok in _parse_ibv_devinfo_list(output)] + + +def _parse_ibv_devinfo_list(output: str) -> list[str]: + """Parse ``ibv_devinfo -l`` output, ignoring banner lines like ``8 HCAs found:``.""" + if not (output or "").strip(): + return [] + tokens: list[str] = [] + for line in (output or "").splitlines(): + line = line.strip() + if not line or line.lower().startswith("warning"): + continue + for tok in line.split(): + if _HCA_NAME_RE.match(tok): + tokens.append(tok) + return tokens + + +def _valid_netdev_name(name: str) -> bool: + name = (name or "").strip() + if not name or not _NETDEV_NAME_RE.match(name): + return False + lower = name.lower() + if any(marker in lower for marker in _INVALID_NETDEV_MARKERS): + return False + if _IB_HCA_NETDEV_RE.match(name): + return False + return True + + +def discover_ib_hca_names(orch) -> dict[str, list[str]]: + """Return {host: [hca_name, ...]} for all hosts in orch. + + Tries ``ibv_devinfo -l`` first; falls back to listing + ``/sys/class/infiniband/`` when ibv_devinfo is absent from the image. + Returns HCA names (e.g. ``rocep28s0``, ``mlx5_0``), correct for + ``NCCL_IB_HCA``. These are NOT Linux netdev names (``ens51f1np1``) -- + those belong in ``ib_netdev`` in the suite config. + + Raises ``RuntimeError`` if: + - any host returns an empty HCA list (indicates missing driver or no IB + hardware on that node), or + - the HCA lists are asymmetric across nodes (a hardware/driver mismatch + must surface loudly, not be silently papered over by intersection). + """ + raw = _topology_exec(orch, _IBVDEVINFO_CMD) + result: dict[str, list[str]] = {} + use_sysfs = False + for host, output in (raw or {}).items(): + hcas = _parse_ibv_devinfo_list(output or "") + if not hcas: + use_sysfs = True + break + result[host] = hcas + + if use_sysfs: + log.info("ib_discovery: ibv_devinfo unavailable or empty; falling back to /sys/class/infiniband") + raw = _topology_exec(orch, _SYSFS_CMD) + result = {} + for host, output in (raw or {}).items(): + hcas = _parse_sysfs(output) + log.info("ib_discovery (sysfs): %s -> %s", host, hcas) + result[host] = hcas + else: + for host, hcas in result.items(): + log.info("ib_discovery: %s -> %s", host, hcas) + + empty = [h for h, devs in result.items() if not devs] + if empty: + raise RuntimeError( + f"ib_discovery: no IB HCA devices found on {empty}. " + "Check that ibv_devinfo is installed and IB drivers are loaded." + ) + + lists = [tuple(sorted(devs)) for devs in result.values()] + if len(set(lists)) > 1: + detail = "; ".join(f"{h}={devs}" for h, devs in result.items()) + raise RuntimeError( + f"ib_discovery: asymmetric HCA device lists across nodes ({detail}). " + "Investigate hardware/driver mismatch before running." + ) + + return result + + +def validate_ib_hca_preflight(discovered: dict[str, list[str]], requested: list[str]) -> None: + """Raise if any requested HCA name is absent from any node's discovered list. + + Called when the config provides an explicit ``ib_hca_devices`` list (not + absent/``"auto"``). Fails loudly naming the missing devices and the node, + so the operator knows exactly which device is wrong rather than getting a + cryptic NCCL error later. + """ + for host, devs in discovered.items(): + missing = [d for d in requested if d not in devs] + if missing: + raise RuntimeError( + f"ib_discovery preflight: requested HCA devices {missing} not found on {host}. Available: {devs}" + ) + + +def _netdev_for_ip_cmd(ip: str) -> str: + inner = ( + f"IF=$( (ip -4 -o addr show 2>/dev/null || /sbin/ip -4 -o addr show 2>/dev/null) | " + f"awk -v ip={shlex.quote(ip)} '{{split($4,a,\"/\"); if(a[1]==ip) {{print $2; exit}}}}'); " + 'echo "${IF}"' + ) + return f"bash -c {shlex.quote(inner)}" + + +def _netdev_via_route_cmd(dest_ip: str) -> str: + inner = ( + f"IF=$( (ip route get {shlex.quote(dest_ip)} 2>/dev/null || " + f"/sbin/ip route get {shlex.quote(dest_ip)} 2>/dev/null) | awk " + "'{{for(i=1;i<=NF;i++) if($i==\"dev\") {{print $(i+1); exit}}}}'); " + 'echo "${IF}"' + ) + return f"bash -c {shlex.quote(inner)}" + + +def discover_socket_netdev_name(orch, master_addr: str | None = None) -> str: + """Return the Linux netdev for NCCL/GLOO socket traffic on a homogeneous cluster. + + On each host, prefers the interface that owns that host's cluster IP (the key + in ``orch.hosts``). Falls back to the egress interface toward ``master_addr``. + Requires the same netdev **name** on every node because the suite broadcasts + one env script to all ranks. + + These are IP netdevs (``ens51f1np1``), not IB HCA names (``mlx5_0``). + """ + hosts = list(getattr(orch, "hosts", []) or []) + if not hosts: + raise RuntimeError("socket_netdev discovery: orchestrator has no hosts") + + master = (master_addr or "").strip() or hosts[0] + per_host: dict[str, str] = {} + for host in hosts: + host_ip = str(host).strip() + out = _topology_exec(orch, _netdev_for_ip_cmd(host_ip), hosts=[host]) + netdev = (out or {}).get(host, "").strip() + if not _valid_netdev_name(netdev): + out = _topology_exec(orch, _netdev_via_route_cmd(master), hosts=[host]) + netdev = (out or {}).get(host, "").strip() + if not _valid_netdev_name(netdev): + raise RuntimeError( + f"socket_netdev discovery: no IPv4 netdev on {host} " + f"(host_ip={host_ip!r}, master_addr={master!r}, last_output={netdev!r})" + ) + per_host[host] = netdev + log.info("socket_netdev discovery: %s -> %s", host, netdev) + + unique = set(per_host.values()) + if len(unique) > 1: + detail = "; ".join(f"{h}={d}" for h, d in per_host.items()) + raise RuntimeError( + f"socket_netdev discovery: asymmetric netdev names across nodes ({detail}). " + "Set roles.server.ib_netdev explicitly when node interface names differ." + ) + return next(iter(unique)) + + +def resolve_multinode_fabric( + orch, + *, + ib_hca_devices=None, + ib_netdev=None, + master_addr=None, +) -> tuple[list[str], str]: + """Resolve ``NCCL_IB_HCA`` devices and the socket netdev for a multinode run. + + Used by ``test_discover_topology`` (once per lifecycle) and lazily by + ``AtomJob.build_server_cmd`` when a partial ``-k`` filter skips + the topology test. + """ + discovered = discover_ib_hca_names(orch) + if ib_hca_devices and ib_hca_devices != "auto": + validate_ib_hca_preflight(discovered, ib_hca_devices) + hcas = list(ib_hca_devices) + else: + hcas = list(next(iter(discovered.values()))) + + configured = (ib_netdev or "").strip() + master = (master_addr or "").strip() or orch.hosts[0] + if configured and configured.lower() != "auto": + netdev = configured + else: + netdev = discover_socket_netdev_name(orch, master_addr=master) + return hcas, netdev diff --git a/cvs/lib/utils/model_query_lib.py b/cvs/lib/utils/model_query_lib.py new file mode 100644 index 000000000..f84a79ed1 --- /dev/null +++ b/cvs/lib/utils/model_query_lib.py @@ -0,0 +1,765 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +from __future__ import annotations + +import json +import re +import shlex +from typing import Any, Mapping, Optional + +JsonDict = dict[str, Any] + +# Keys passed from scoring_config into LmEvalBenchmark.check_results(). +LM_EVAL_CHECK_RESULT_KEYS = ( + "task_name", + "parse_metric", + "expected", + "metric_key", + "tasks", + "tolerance_frac", + "log_path", + "label", +) + + +class OpenAIProbe: + """OpenAI-compatible HTTP smoke tests (stdlib probe script + validation).""" + + TIMEOUT_S = 120.0 + CHAT_MAX_TOKENS = 8 + COMPLETION_MAX_TOKENS = 50 + STRUCTURED_BOOK_MAX_TOKENS = 256 + + CHAT_USER = "Reply with exactly one word: OK." + COMPLETION_PROMPT = "The capital of France is" + STRUCTURED_BOOK_SYSTEM = "Respond with a single JSON object only. No markdown or text outside the JSON." + STRUCTURED_BOOK_USER = ( + "Return one book as a JSON object with keys: title (string), author (string), year (integer), genre (string)." + ) + + STEP_TITLES: dict[str, str] = { + "model_endpoint": "Model endpoint — GET /v1/models", + "chat_completion_endpoint": "Chat completion endpoint — POST /v1/chat/completions", + "completion_endpoint": "Completion endpoint — POST /v1/completions", + "structured_output_book": ( + "Structured output (book) — POST /v1/chat/completions (response_format: json_object)" + ), + } + + _FAILURE_MARKER = "OpenAI-compatible probe failed port=" + + @classmethod + def probe_script( + cls, + port: int, + model: str, + *, + host: str = "0.0.0.0", + timeout_s: float = TIMEOUT_S, + chat_max_tokens: int = CHAT_MAX_TOKENS, + completion_max_tokens: int = COMPLETION_MAX_TOKENS, + structured_book_max_tokens: int = STRUCTURED_BOOK_MAX_TOKENS, + ) -> str: + """Stdlib-only Python source for docker exec / orch.exec.""" + chat_messages = [{"role": "user", "content": cls.CHAT_USER}] + book_messages = [ + {"role": "system", "content": cls.STRUCTURED_BOOK_SYSTEM}, + {"role": "user", "content": cls.STRUCTURED_BOOK_USER}, + ] + return "\n".join( + [ + "import json, urllib.request, urllib.error", + f"PORT = {int(port)}", + f"TIMEOUT = {float(timeout_s)}", + f"CHAT_MAX = {int(chat_max_tokens)}", + f"COMP_MAX = {int(completion_max_tokens)}", + f"BOOK_MAX = {int(structured_book_max_tokens)}", + f"MODEL = {json.dumps(model)}", + f'BASE = "http://{host}:{int(port)}"', + "", + "def req(method, path, body=None):", + " url = BASE + path", + " data = None if body is None else json.dumps(body).encode('utf-8')", + " hdrs = {}", + " if body is not None:", + ' hdrs["Content-Type"] = "application/json"', + " r = urllib.request.Request(url, data=data, headers=hdrs, method=method)", + " try:", + " with urllib.request.urlopen(r, timeout=TIMEOUT) as resp:", + ' raw = resp.read().decode("utf-8", errors="replace")', + " code = int(getattr(resp, 'status', 200))", + " try:", + " return code, json.loads(raw) if raw else {}", + " except json.JSONDecodeError:", + " return code, raw", + " except urllib.error.HTTPError as e:", + ' raw = e.read().decode("utf-8", errors="replace")', + " try:", + " return int(e.code), json.loads(raw) if raw else {}", + " except json.JSONDecodeError:", + " return int(e.code), raw", + "", + "out = {}", + 'out["model_endpoint"] = req("GET", "/v1/models")', + 'out["chat_completion_endpoint"] = req("POST", "/v1/chat/completions", {', + ' "model": MODEL,', + f' "messages": {json.dumps(chat_messages)},', + ' "max_tokens": CHAT_MAX,', + ' "temperature": 0.0,', + "})", + 'out["completion_endpoint"] = req("POST", "/v1/completions", {', + ' "model": MODEL,', + f' "prompt": {json.dumps(cls.COMPLETION_PROMPT)},', + ' "max_tokens": COMP_MAX,', + ' "temperature": 0.0,', + "})", + 'out["structured_output_book"] = req("POST", "/v1/chat/completions", {', + ' "model": MODEL,', + f' "messages": {json.dumps(book_messages)},', + ' "max_tokens": BOOK_MAX,', + ' "temperature": 0.0,', + ' "response_format": {"type": "json_object"},', + "})", + "print(json.dumps(out, default=str))", + ] + ) + + @classmethod + def log_results(cls, results: dict[str, tuple[int, Any]], logger: Any) -> None: + """One headed block per endpoint for logs / HTML capture.""" + for step, (code, body) in results.items(): + title = cls.STEP_TITLES.get(step, step) + logger.info("#================ %s ================#", title) + logger.info("HTTP status: %s", code) + if isinstance(body, (dict, list)): + try: + logger.info("Body:\n%s", json.dumps(body, indent=2, default=str)) + except (TypeError, ValueError): + logger.info("Body: %s", body) + else: + logger.info("Body: %s", body) + + @classmethod + def check_results( + cls, + results: dict[str, tuple[int, Any]], + *, + port: Any, + logger: Optional[Any] = None, + ) -> tuple[bool, Optional[str]]: + """ + Each step: HTTP 200, JSON object body, and minimal useful content. + """ + failures: list[str] = [] + + def _fail(detail: str) -> None: + failures.append(detail) + if logger is not None: + logger.info( + "OpenAI-compatible probe check FAILED: %s port=%r", + detail, + port, + ) + + for step, (code, body) in results.items(): + title = cls.STEP_TITLES.get(step, step) + if code != 200: + _fail(f"{title} (step={step!r}): HTTP status={code} body={body!r}") + continue + if body is None: + _fail(f"{title}: response body is None") + continue + if not isinstance(body, dict): + _fail(f"{title}: body is not a JSON object ({type(body).__name__})") + continue + + if step == "model_endpoint": + data = body.get("data") + if not isinstance(data, list) or not data: + _fail(f"{title}: missing or empty models list") + continue + first = data[0] + if not isinstance(first, dict) or not str(first.get("id") or first.get("model") or "").strip(): + _fail(f"{title}: no model id in models response") + continue + + if not str(body.get("model") or "").strip(): + _fail(f"{title}: response has no model field") + continue + + choices = body.get("choices") + if not isinstance(choices, list) or not choices: + _fail(f"{title}: missing or empty choices") + continue + ch0 = choices[0] + if not isinstance(ch0, dict): + _fail(f"{title}: choices[0] is not an object") + continue + + if step == "completion_endpoint": + if not str(ch0.get("text") or "").strip(): + _fail(f"{title}: empty completion text") + continue + + if step in ("chat_completion_endpoint", "structured_output_book"): + msg = ch0.get("message") + if not isinstance(msg, dict): + _fail(f"{title}: choices[0].message is not an object") + continue + content = msg.get("content") + if content is None or not str(content).strip(): + _fail(f"{title}: empty assistant content") + continue + if step == "structured_output_book": + try: + obj = json.loads(str(content)) + except json.JSONDecodeError as e: + _fail(f"{title}: assistant content is not JSON ({e!r})") + continue + if not isinstance(obj, dict): + _fail(f"{title}: parsed content is not a JSON object") + continue + for key in ("title", "author", "year", "genre"): + if key not in obj: + _fail(f"{title}: JSON missing {key!r}") + break + v = obj[key] + if key == "year": + if isinstance(v, (int, float)): + continue + if not str(v).strip(): + _fail(f"{title}: JSON year empty") + break + elif not str(v).strip(): + _fail(f"{title}: JSON field {key!r} empty") + break + continue + + if failures: + return False, f"{cls._FAILURE_MARKER}{port!r}: " + " | ".join(failures) + return True, None + + @classmethod + def summarize_results( + cls, + results: dict[str, tuple[int, Any]], + ok: bool, + err: Optional[str], + ) -> list[str]: + """Human-readable per-step pass/fail lines for logs and inf_res_dict.""" + failure_parts: list[str] = [] + if not ok and err and err.startswith(cls._FAILURE_MARKER): + rest = err[len(cls._FAILURE_MARKER) :] + colon_idx = rest.find(": ") + if colon_idx != -1: + failure_parts = [p.strip() for p in rest[colon_idx + 2 :].split("|")] + + summary: list[str] = [] + for step, (status, _content) in results.items(): + title = cls.STEP_TITLES.get(step, step) + if ok: + outcome = "Pass" if status == 200 else "Fail" + elif status != 200: + outcome = "Fail" + elif any(p.startswith(title) or p.startswith(f"{title} (step=") for p in failure_parts): + outcome = "Fail" + else: + outcome = "Pass" + summary.append(f"{title} -> {outcome} ({status})") + return summary + + +class LmEvalBenchmark: + """lm-eval helpers for OpenAI-compatible local-* adapters.""" + + DEFAULT_NUM_CONCURRENT = "4" + DEFAULT_NUM_FEWSHOT = "5" + DEFAULT_BATCH_SIZE = "auto" + DEFAULT_LM_EVAL_MODEL = "local-completions" + DEFAULT_EXEC_TIMEOUT_SEC = 7200 + DEFAULT_TOLERANCE_FRAC = 0.05 + + @staticmethod + def parse_metric_value(text: str, task: str, metric: str) -> float | None: + """Parse a metric value from lm-eval's ASCII results table.""" + m = re.search( + rf"{re.escape(metric)}\s*\|\s*[^|\n]+\|\s*([0-9]+(?:\.[0-9]+)?)", + text, + flags=re.I, + ) + return float(m.group(1)) if m else None + + @staticmethod + def openai_base_url(port: int, lm_eval_model: str, host: str = "0.0.0.0") -> str: + """Build base_url for lm-eval's local-completions / local-chat-completions.""" + path = "/v1/chat/completions" if "chat" in lm_eval_model.lower() else "/v1/completions" + return f"http://{host}:{int(port)}{path}" + + @classmethod + def build_model_args( + cls, + *, + model_id: str, + base_url: str, + num_concurrent: str, + extra_model_args: str = "", + ) -> str: + model_args = f"model={model_id},base_url={base_url},num_concurrent={num_concurrent},tokenized_requests=False" + extra = str(extra_model_args or "").strip() + if extra: + model_args = f"{model_args},{extra}" + return model_args + + @classmethod + def build_command( + cls, + *, + lm_eval_model: str, + model_id: str, + base_url: str, + tasks: str, + num_fewshot: str, + batch_size: str, + num_concurrent: str, + limit: str = "", + extra_model_args: str = "", + log_path: str, + pip_install: bool = True, + ) -> str: + """ + Inner shell fragment to run lm-eval (no docker/ssh). + Caller handles mkdir, source env, docker exec, etc. + """ + limit_s = str(limit or "").strip() + limit_arg = f" --limit {limit_s}" if limit_s else "" + model_args_q = shlex.quote( + cls.build_model_args( + model_id=model_id, + base_url=base_url, + num_concurrent=num_concurrent, + extra_model_args=extra_model_args, + ) + ) + parts: list[str] = [] + if pip_install: + parts.append("pip install -q 'lm-eval[api]'") + parts.append( + "python3 -m lm_eval " + f"--model {lm_eval_model} " + f"--model_args {model_args_q} " + f"--tasks {tasks} " + f"--num_fewshot {num_fewshot} " + f"--batch_size {batch_size}" + f"{limit_arg} " + f"2>&1 | tee {shlex.quote(log_path)}" + ) + return " && ".join(parts) + + @classmethod + def check_results( + cls, + text: str, + *, + task_name: str, + parse_metric: str, + expected: float, + metric_key: str, + tasks: str = "", + tolerance_frac: float = DEFAULT_TOLERANCE_FRAC, + log_path: str = "", + label: str = "", + ) -> tuple[bool, dict[str, Any] | None, str | None]: + """ + Validate lm-eval stdout after exec. + + Returns ``(ok, summary, err)`` where summary is:: + + {"task", "metric_key", "actual", "expected"} + """ + display = label or task_name + log_hint = f" (see {log_path})" if log_path else "" + + if not text or not str(text).strip(): + return False, None, f"lm-eval {display} produced no output" + if re.search(r"Traceback \(most recent call last\)", text): + return False, None, f"lm-eval {display} failed: Python traceback in output" + if not re.search(re.escape(task_name), text, re.I): + return ( + False, + None, + f"lm-eval {display} output missing {task_name!r} results{log_hint}", + ) + + actual = cls.parse_metric_value(text, task_name, parse_metric) + if actual is None: + return ( + False, + None, + f"could not parse lm-eval table score for {task_name} / {parse_metric}", + ) + + expected_f = float(expected) + if abs(actual - expected_f) > tolerance_frac * abs(expected_f): + short_metric = "flexible-extract" if "flexible" in metric_key.lower() else parse_metric + err = ( + f"{task_name} {short_metric} {actual:.4f} not within " + f"{tolerance_frac * 100:.0f}% of expected {expected_f:.4f}" + ) + summary = { + "task": str(tasks or task_name), + "metric_key": metric_key, + "actual": float(actual), + "expected": expected_f, + "passed": False, + } + return False, summary, err + summary = { + "task": str(tasks or task_name), + "metric_key": metric_key, + "actual": float(actual), + "expected": expected_f, + "passed": True, + } + return True, summary, None + + @classmethod + def prepare( + cls, + i_dict: Mapping[str, Any], + *, + port: int, + host: str = "0.0.0.0", + model_id: str, + task_name: str, + default_tasks: str, + default_metric: str, + default_metric_key: str, + log_dir: str, + log_basename: str, + default_num_concurrent: str = DEFAULT_NUM_CONCURRENT, + ) -> tuple[str, dict[str, Any]]: + """ + Resolve config and build the inner lm-eval shell command. + + Returns ``(inner_cmd, scoring_config)``. + """ + lm_eval_model = str(i_dict.get("lm_eval_model", cls.DEFAULT_LM_EVAL_MODEL)) + base_url = cls.openai_base_url(port, lm_eval_model, host=host) + num_concurrent = str(i_dict.get("num_concurrent", default_num_concurrent)) + tasks = str(i_dict.get("tasks", default_tasks)) + num_fewshot = str(i_dict.get("num_fewshot", cls.DEFAULT_NUM_FEWSHOT)) + batch_size = str(i_dict.get("batch_size", cls.DEFAULT_BATCH_SIZE)) + limit = str(i_dict.get("limit", "")).strip() + extra_model_args = str(i_dict.get("extra_model_args", "")).strip() + exec_timeout_sec = int(i_dict.get("exec_timeout_sec", cls.DEFAULT_EXEC_TIMEOUT_SEC)) + tolerance_frac = float(i_dict.get("tolerance_frac", cls.DEFAULT_TOLERANCE_FRAC)) + log_path = f"{log_dir.rstrip('/')}/benchmark_node/{log_basename}" + + expected_block = i_dict.get("expected_results") or {} + if not isinstance(expected_block, Mapping): + raise ValueError("expected_results must be a mapping") + task_expected = expected_block.get(task_name) or {} + if not isinstance(task_expected, Mapping): + raise ValueError(f"expected_results[{task_name!r}] must be a mapping") + if default_metric_key not in task_expected: + raise KeyError(f"expected_results[{task_name!r}][{default_metric_key!r}] missing") + expected = float(task_expected[default_metric_key]) + + inner_cmd = cls.build_command( + lm_eval_model=lm_eval_model, + model_id=model_id, + base_url=base_url, + tasks=tasks, + num_fewshot=num_fewshot, + batch_size=batch_size, + num_concurrent=num_concurrent, + limit=limit, + extra_model_args=extra_model_args, + log_path=log_path, + pip_install=bool(i_dict.get("pip_install", True)), + ) + + scoring_config: dict[str, Any] = { + "task_name": task_name, + "parse_metric": default_metric, + "metric_key": default_metric_key, + "expected": expected, + "tasks": tasks, + "tolerance_frac": tolerance_frac, + "log_path": log_path, + "exec_timeout_sec": exec_timeout_sec, + "label": str(i_dict.get("label", task_name)), + "lm_eval_model": lm_eval_model, + "base_url": base_url, + } + return inner_cmd, scoring_config + + @classmethod + def check_kwargs_from_scoring(cls, scoring: Mapping[str, Any]) -> dict[str, Any]: + """Extract kwargs for check_results() from a scoring_config dict.""" + return {k: scoring[k] for k in LM_EVAL_CHECK_RESULT_KEYS} + + @classmethod + def fallback_summary( + cls, + scoring: Mapping[str, Any], + *, + actual: float | None = None, + error: str | None = None, + ) -> dict[str, Any]: + return { + "task": str(scoring.get("tasks") or scoring["task_name"]), + "metric_key": scoring["metric_key"], + "actual": actual, + "expected": float(scoring["expected"]), + "passed": False, + "error": error, + } + + +LONG_CTX_NIAH_CHECK_RESULT_KEYS = ( + "task_name", + "metric_key", + "expected", + "tolerance_frac", + "log_path", + "label", +) + + +class LongContextNiahBenchmark: + """Needle-in-a-haystack long-context accuracy via POST /v1/chat/completions.""" + + DEFAULT_METRIC_KEY = "pass_rate" + DEFAULT_TASK_NAME = "long_ctx_niah" + DEFAULT_TOLERANCE_FRAC = 0.05 + DEFAULT_EXEC_TIMEOUT_SEC = 21600 + DEFAULT_REQUEST_TIMEOUT_SEC = 7200 + + @classmethod + def probe_script( + cls, + *, + port: int, + model: str, + isl: int, + osl: int, + num_prompts: int, + seed: int, + host: str = "0.0.0.0", + request_timeout_sec: int = DEFAULT_REQUEST_TIMEOUT_SEC, + ) -> str: + """Python source run inside benchmark container (docker exec).""" + return "\n".join( + [ + "import json, random, re, string, sys, urllib.error, urllib.request", + "", + "def _pip_transformers():", + " import subprocess", + " subprocess.check_call([sys.executable, '-m', 'pip', 'install', '-q', 'transformers'])", + "", + "try:", + " from transformers import AutoTokenizer", + "except Exception:", + " _pip_transformers()", + " from transformers import AutoTokenizer", + "", + f"PORT = {int(port)}", + f"MODEL = {json.dumps(model)}", + f"TARGET_ISL = {int(isl)}", + f"MAX_TOKENS = {int(osl)}", + f"NUM_PROMPTS = {int(num_prompts)}", + f"SEED = {int(seed)}", + f"REQ_TIMEOUT = {float(request_timeout_sec)}", + f'BASE = "http://{host}:{int(port)}"', + 'URL = BASE + "/v1/chat/completions"', + "", + "def norm(s):", + " return re.sub(r'\\s+', '', str(s or '').lower())", + "", + "def chat(prompt):", + " body = {", + ' "model": MODEL,', + ' "messages": [{"role": "user", "content": prompt}],', + ' "max_tokens": MAX_TOKENS,', + ' "temperature": 0.0,', + " }", + " data = json.dumps(body).encode('utf-8')", + ' req = urllib.request.Request(URL, data=data, headers={"Content-Type": "application/json"}, method="POST")', + " with urllib.request.urlopen(req, timeout=REQ_TIMEOUT) as resp:", + " raw = resp.read().decode('utf-8', errors='replace')", + " obj = json.loads(raw)", + ' choices = obj.get("choices") or []', + " if not choices:", + ' raise RuntimeError("empty choices")', + ' msg = choices[0].get("message") or {}', + ' return str(msg.get("content") or "")', + "", + "def build_prompt(tok, needle):", + ' prefix = "The passkey is %s. " % needle', + ' suffix = "\\n\\nWhat is the passkey? Reply with only the passkey."', + " prefix_ids = tok.encode(prefix, add_special_tokens=False)", + " suffix_ids = tok.encode(suffix, add_special_tokens=False)", + " filler_budget = TARGET_ISL - len(prefix_ids) - len(suffix_ids)", + " if filler_budget < 1:", + ' raise RuntimeError("TARGET_ISL too small for needle prompt skeleton")', + ' word = "foo "', + " word_ids = tok.encode(word, add_special_tokens=False)", + " if not word_ids:", + ' raise RuntimeError("tokenizer produced empty filler word")', + " reps = (filler_budget // len(word_ids)) + 1", + " filler_ids = (word_ids * reps)[:filler_budget]", + " prompt_ids = prefix_ids + filler_ids + suffix_ids", + " if len(prompt_ids) != TARGET_ISL:", + " raise RuntimeError('prompt token len %d != TARGET_ISL %d' % (len(prompt_ids), TARGET_ISL))", + " return tok.decode(prompt_ids, skip_special_tokens=True), needle", + "", + "def main():", + " random.seed(SEED)", + " tok = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)", + " results = []", + " correct = 0", + " for i in range(NUM_PROMPTS):", + ' needle = "NEEDLE-" + "".join(random.choices(string.ascii_uppercase + string.digits, k=8))', + " prompt, expected = build_prompt(tok, needle)", + " try:", + " actual = chat(prompt)", + " ok = norm(expected) in norm(actual)", + " except Exception as e:", + " actual = 'ERROR: %s' % e", + " ok = False", + " correct += int(ok)", + " results.append({", + ' "sample": i,', + ' "expected": expected,', + ' "actual": actual,', + ' "passed": bool(ok),', + " })", + " total = len(results)", + " out = {", + ' "task": "long_ctx_niah",', + ' "metric_key": "pass_rate",', + ' "isl": TARGET_ISL,', + ' "osl": MAX_TOKENS,', + ' "correct": correct,', + ' "total": total,', + ' "pass_rate": (float(correct) / total) if total else 0.0,', + ' "results": results,', + " }", + " print(json.dumps(out))", + "", + 'if __name__ == "__main__":', + " main()", + ] + ) + + @classmethod + def check_results( + cls, + text: str, + *, + task_name: str, + metric_key: str, + expected: float, + tolerance_frac: float = DEFAULT_TOLERANCE_FRAC, + log_path: str = "", + label: str = "", + ) -> tuple[bool, dict[str, Any] | None, str | None]: + display = label or task_name + log_hint = f" (see {log_path})" if log_path else "" + + if not text or not str(text).strip(): + return False, None, f"{display} produced no output" + + payload = None + for line in reversed(str(text).splitlines()): + line = line.strip() + if line.startswith("{") and line.endswith("}"): + try: + payload = json.loads(line) + break + except json.JSONDecodeError: + continue + if not isinstance(payload, dict): + return False, None, f"{display} output missing JSON summary{log_hint}" + + actual = payload.get(metric_key) + if actual is None: + return False, None, f"{display} JSON missing {metric_key!r}{log_hint}" + + actual_f = float(actual) + expected_f = float(expected) + passed = actual_f + tolerance_frac * abs(expected_f) >= expected_f + summary = { + "task": str(payload.get("task") or task_name), + "metric_key": metric_key, + "actual": actual_f, + "expected": expected_f, + "passed": passed, + "isl": payload.get("isl"), + "osl": payload.get("osl"), + "correct": payload.get("correct"), + "total": payload.get("total"), + } + if passed: + return True, summary, None + return ( + False, + summary, + f"{display} pass_rate {actual_f:.4f} below expected {expected_f:.4f} (tol={tolerance_frac * 100:.0f}%)", + ) + + @classmethod + def prepare( + cls, + i_dict: Mapping[str, Any], + *, + port: int, + host: str = "0.0.0.0", + model_id: str, + isl: int, + osl: int, + log_dir: str, + log_basename: str, + ) -> tuple[str, dict[str, Any]]: + num_prompts = int(i_dict.get("num_prompts", 16)) + seed = int(i_dict.get("seed", 42)) + exec_timeout_sec = int(i_dict.get("exec_timeout_sec", cls.DEFAULT_EXEC_TIMEOUT_SEC)) + request_timeout_sec = int(i_dict.get("request_timeout_sec", cls.DEFAULT_REQUEST_TIMEOUT_SEC)) + tolerance_frac = float(i_dict.get("tolerance_frac", cls.DEFAULT_TOLERANCE_FRAC)) + log_path = f"{log_dir.rstrip('/')}/benchmark_node/{log_basename}" + + expected_block = i_dict.get("expected_results") or {} + auto_expected = expected_block.get("auto") or {} + if cls.DEFAULT_METRIC_KEY not in auto_expected: + raise KeyError(f"expected_results.auto[{cls.DEFAULT_METRIC_KEY!r}] missing") + expected = float(auto_expected[cls.DEFAULT_METRIC_KEY]) + + inner_cmd = f"python3 /tmp/long_ctx_niah_probe.py 2>&1 | tee {shlex.quote(log_path)}" + scoring = { + "task_name": cls.DEFAULT_TASK_NAME, + "metric_key": cls.DEFAULT_METRIC_KEY, + "expected": expected, + "tolerance_frac": tolerance_frac, + "log_path": log_path, + "exec_timeout_sec": exec_timeout_sec, + "label": f"long_ctx_niah isl={isl}", + "probe_kwargs": { + "port": int(port), + "host": host, + "model": model_id, + "isl": int(isl), + "osl": int(osl), + "num_prompts": num_prompts, + "seed": seed, + "request_timeout_sec": request_timeout_sec, + }, + } + return inner_cmd, scoring + + @classmethod + def check_kwargs_from_scoring(cls, scoring: Mapping[str, Any]) -> dict[str, Any]: + return {k: scoring[k] for k in LONG_CTX_NIAH_CHECK_RESULT_KEYS} diff --git a/cvs/lib/utils/unittests/__init__.py b/cvs/lib/utils/unittests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cvs/lib/utils/unittests/test_config_loader.py b/cvs/lib/utils/unittests/test_config_loader.py new file mode 100644 index 000000000..e769c1873 --- /dev/null +++ b/cvs/lib/utils/unittests/test_config_loader.py @@ -0,0 +1,459 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.utils.config_loader. + +Covers the public contract described in the spec: + - ModelSpec accepts an optional ``precision`` field (defaults to empty string). + - BaseVariantConfig accepts an optional ``threshold_json`` field (defaults to ""). + - substitute_config reads the threshold file from ``threshold_json`` when set + (literal absolute path), or discovers a sole sibling ``*threshold.json``. + - substitute_config raises FileNotFoundError when ``threshold_json`` names a + non-existent file, or when no sibling threshold exists and the field is empty. + - Multiple sibling ``*threshold.json`` files raise ValueError (ambiguous). + +Framework: unittest.TestCase + self.subTest + unittest.mock (no pytest). +''' + +import json +import tempfile +import unittest +from pathlib import Path +from pydantic import ValidationError + +from cvs.lib.utils.config_loader import ( + BaseVariantConfig, + ContainerSpec, + ModelSpec, + Paths, + substitute_config, +) + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + + +def _paths_dict(): + return { + "shared_fs": "/data/shared", + "models_dir": "/data/models", + "log_dir": "/data/logs", + "hf_token_file": "/data/.hf_token", + } + + +def _model_dict(): + """A valid ModelSpec payload.""" + return {"id": "amd/Llama-3.1-70B-Instruct", "remote": 0} + + +def _container_dict(): + return { + "name": "my-container", + "image": "rocm/vllm-dev:nightly", + "runtime": {"name": "docker"}, + } + + +def _base_config_dict(threshold_json: str = ""): + """Minimal dict that satisfies BaseVariantConfig.""" + d = { + "schema_version": 1, + "paths": _paths_dict(), + "model": _model_dict(), + "container": _container_dict(), + } + if threshold_json: + d["threshold_json"] = threshold_json + return d + + +# --------------------------------------------------------------------------- +# ModelSpec — stateful pydantic model +# --------------------------------------------------------------------------- + + +class TestModelSpecLifecycle(unittest.TestCase): + """ModelSpec is a _Forbid model: valid construction, unknown fields rejected.""" + + # --- legal transitions --- + + def test_valid_construction_remote_zero(self): + spec = ModelSpec(id="some/model", remote=0) + self.assertEqual(spec.id, "some/model") + self.assertEqual(spec.remote, 0) + + def test_valid_construction_remote_one(self): + # remote=1 is accepted by the schema (validator rejects it at + # BaseVariantConfig level, not here) + spec = ModelSpec(id="some/model", remote=1) + self.assertEqual(spec.remote, 1) + + # --- illegal transitions (extra fields forbidden) --- + + def test_precision_field_is_optional(self): + """ModelSpec may carry an optional precision label (e.g. fp8 in filenames).""" + spec = ModelSpec(id="amd/llama", remote=0, precision="fp8") + self.assertEqual(spec.precision, "fp8") + + def test_precision_defaults_empty(self): + spec = ModelSpec(id="amd/llama", remote=0) + self.assertEqual(spec.precision, "") + + def test_arbitrary_extra_field_is_rejected(self): + with self.assertRaises(ValidationError): + ModelSpec(id="amd/llama", remote=0, unknown_key="value") + + def test_missing_id_is_rejected(self): + with self.assertRaises(ValidationError): + ModelSpec(remote=0) + + def test_missing_remote_is_rejected(self): + with self.assertRaises(ValidationError): + ModelSpec(id="amd/llama") + + def test_invalid_remote_value_is_rejected(self): + """remote must be Literal[0, 1] — other ints are invalid.""" + with self.assertRaises(ValidationError): + ModelSpec(id="amd/llama", remote=2) + + # --- idempotent re-entry --- + + def test_model_dict_roundtrip(self): + """model_dump / model_validate are idempotent.""" + spec = ModelSpec(id="amd/llama", remote=0) + dumped = spec.model_dump() + spec2 = ModelSpec.model_validate(dumped) + self.assertEqual(spec.id, spec2.id) + self.assertEqual(spec.remote, spec2.remote) + + +# --------------------------------------------------------------------------- +# BaseVariantConfig — stateful pydantic model +# --------------------------------------------------------------------------- + + +class TestBaseVariantConfigLifecycle(unittest.TestCase): + """BaseVariantConfig optional threshold_json and forbid extra fields.""" + + # --- legal transitions --- + + def test_valid_construction_with_threshold_json(self): + cfg = BaseVariantConfig(**_base_config_dict("/abs/path/threshold.json")) + self.assertEqual(cfg.threshold_json, "/abs/path/threshold.json") + + def test_threshold_json_defaults_empty(self): + cfg = BaseVariantConfig(**_base_config_dict()) + self.assertEqual(cfg.threshold_json, "") + + def test_threshold_json_preserved_verbatim(self): + """threshold_json is stored as-is; no substitution or normalization.""" + path = "/some/deeply/nested/threshold.json" + cfg = BaseVariantConfig(**_base_config_dict(path)) + self.assertEqual(cfg.threshold_json, path) + + def test_enforce_thresholds_defaults_true(self): + cfg = BaseVariantConfig(**_base_config_dict()) + self.assertTrue(cfg.enforce_thresholds) + + def test_enforce_thresholds_can_be_false(self): + d = _base_config_dict() + d["enforce_thresholds"] = False + cfg = BaseVariantConfig(**d) + self.assertFalse(cfg.enforce_thresholds) + + def test_thresholds_defaults_to_empty_dict(self): + cfg = BaseVariantConfig(**_base_config_dict()) + self.assertEqual(cfg.thresholds, {}) + + # --- illegal transitions --- + + def test_missing_threshold_json_is_optional(self): + """threshold_json defaults to empty when omitted.""" + d = _base_config_dict() + self.assertNotIn("threshold_json", d) + cfg = BaseVariantConfig(**d) + self.assertEqual(cfg.threshold_json, "") + + def test_extra_field_is_rejected(self): + d = _base_config_dict() + d["unexpected_key"] = "oops" + with self.assertRaises(ValidationError): + BaseVariantConfig(**d) + + def test_missing_schema_version_raises(self): + d = _base_config_dict() + del d["schema_version"] + with self.assertRaises(ValidationError): + BaseVariantConfig(**d) + + def test_invalid_schema_version_raises(self): + d = _base_config_dict() + d["schema_version"] = 2 + with self.assertRaises(ValidationError): + BaseVariantConfig(**d) + + def test_remote_one_raises_not_implemented(self): + """model.remote==1 triggers the _check_remote_not_implemented guard.""" + d = _base_config_dict() + d["model"] = {"id": "some/model", "remote": 1} + with self.assertRaises((ValidationError, NotImplementedError)): + BaseVariantConfig(**d) + + # --- idempotent re-entry --- + + def test_model_validate_roundtrip(self): + cfg = BaseVariantConfig(**_base_config_dict("/t.json")) + dumped = cfg.model_dump() + cfg2 = BaseVariantConfig.model_validate(dumped) + self.assertEqual(cfg.threshold_json, cfg2.threshold_json) + self.assertEqual(cfg.schema_version, cfg2.schema_version) + + +# --------------------------------------------------------------------------- +# Paths — stateful pydantic model +# --------------------------------------------------------------------------- + + +class TestPathsLifecycle(unittest.TestCase): + """Paths is _Forbid — required fields must be present; extras rejected.""" + + def test_valid_construction(self): + p = Paths(**_paths_dict()) + self.assertEqual(p.shared_fs, "/data/shared") + + def test_extra_field_rejected(self): + d = dict(_paths_dict()) + d["extra"] = "x" + with self.assertRaises(ValidationError): + Paths(**d) + + def test_missing_field_rejected(self): + for key in ["shared_fs", "models_dir", "log_dir", "hf_token_file"]: + with self.subTest(missing=key): + d = dict(_paths_dict()) + del d[key] + with self.assertRaises(ValidationError): + Paths(**d) + + +# --------------------------------------------------------------------------- +# ContainerSpec — stateful pydantic model +# --------------------------------------------------------------------------- + + +class TestContainerSpecLifecycle(unittest.TestCase): + """ContainerSpec is _Forbid; lifetime has a default.""" + + def test_valid_construction_defaults(self): + spec = ContainerSpec(**_container_dict()) + self.assertEqual(spec.lifetime, "per_run") + + def test_valid_lifetime_values(self): + for lifetime in ["no_launch", "per_run", "persistent"]: + with self.subTest(lifetime=lifetime): + d = dict(_container_dict()) + d["lifetime"] = lifetime + spec = ContainerSpec(**d) + self.assertEqual(spec.lifetime, lifetime) + + def test_invalid_lifetime_rejected(self): + d = dict(_container_dict()) + d["lifetime"] = "never" + with self.assertRaises(ValidationError): + ContainerSpec(**d) + + def test_extra_field_rejected(self): + d = dict(_container_dict()) + d["bogus"] = "x" + with self.assertRaises(ValidationError): + ContainerSpec(**d) + + def test_missing_image_rejected(self): + d = dict(_container_dict()) + del d["image"] + with self.assertRaises(ValidationError): + ContainerSpec(**d) + + +# --------------------------------------------------------------------------- +# substitute_config — pure function (with I/O side-effects via filesystem) +# --------------------------------------------------------------------------- + + +class TestSubstituteConfigThresholdJsonField(unittest.TestCase): + """substitute_config threshold discovery: explicit path or sibling glob.""" + + def _write_config(self, tmp_dir: Path, config_dict: dict) -> Path: + config_path = tmp_dir / "variant_config.json" + config_path.write_text(json.dumps(config_dict)) + return config_path + + def _write_threshold(self, tmp_dir: Path, name: str = "threshold.json") -> Path: + threshold_path = tmp_dir / name + threshold_path.write_text(json.dumps({"ISL=128,OSL=2048,TP=8,CONC=16": {}})) + return threshold_path + + def test_happy_path_reads_from_threshold_json_field(self): + """Config with valid threshold_json pointing to a real file must load.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + threshold_path = self._write_threshold(tmp_dir) + cfg = _base_config_dict(str(threshold_path)) + config_path = self._write_config(tmp_dir, cfg) + cluster_dict = {} + raw, thresholds = substitute_config(config_path, cluster_dict) + self.assertEqual(raw["schema_version"], 1) + + def test_sibling_threshold_discovered_when_threshold_json_empty(self): + """When threshold_json is omitted, a sole sibling *threshold.json is used.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + self._write_threshold(tmp_dir, "mi300x_variant_threshold.json") + cfg = _base_config_dict() + config_path = self._write_config(tmp_dir, cfg) + raw, thresholds = substitute_config(config_path, {}) + self.assertIsInstance(thresholds, dict) + self.assertIn("ISL=128,OSL=2048,TP=8,CONC=16", thresholds) + + def test_sibling_threshold_ambiguous_raises_value_error(self): + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + self._write_threshold(tmp_dir, "a_threshold.json") + self._write_threshold(tmp_dir, "b_threshold.json") + config_path = self._write_config(tmp_dir, _base_config_dict()) + with self.assertRaises(ValueError): + substitute_config(config_path, {}) + + def test_no_sibling_and_empty_threshold_json_raises_file_not_found(self): + with tempfile.TemporaryDirectory() as tmp: + config_path = self._write_config(Path(tmp), _base_config_dict()) + with self.assertRaises(FileNotFoundError): + substitute_config(config_path, {}) + + def test_threshold_json_as_absolute_path_no_sibling_glob(self): + """The threshold file must be read from the explicit path, not discovered + via glob. Even when no sibling *threshold.json exists, a valid explicit + path must succeed.""" + with tempfile.TemporaryDirectory() as config_dir, tempfile.TemporaryDirectory() as threshold_dir: + # Put threshold file in a DIFFERENT directory than the config + threshold_path = Path(threshold_dir) / "my_threshold.json" + threshold_path.write_text(json.dumps({"cell": {}})) + + cfg = dict(_base_config_dict(str(threshold_path))) + config_path = self._write_config(Path(config_dir), cfg) + cluster_dict = {} + # Must succeed even though no *threshold.json exists in config_dir + raw, thresholds = substitute_config(config_path, cluster_dict) + self.assertIsNotNone(raw) + + def test_missing_threshold_file_raises_file_not_found(self): + """If threshold_json names a non-existent file, FileNotFoundError is raised.""" + with tempfile.TemporaryDirectory() as tmp: + cfg = _base_config_dict("/nonexistent/path/threshold.json") + config_path = self._write_config(Path(tmp), cfg) + cluster_dict = {} + with self.assertRaises(FileNotFoundError): + substitute_config(config_path, cluster_dict) + + def test_explicit_threshold_json_ignores_sibling_when_path_missing(self): + """An explicit but missing threshold_json path fails even if a sibling exists.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + self._write_threshold(tmp_dir, "sibling_threshold.json") + cfg = _base_config_dict("/does/not/exist/threshold.json") + config_path = self._write_config(tmp_dir, cfg) + with self.assertRaises(FileNotFoundError): + substitute_config(config_path, {}) + + def test_threshold_json_value_not_placeholder_substituted(self): + """The threshold_json value is a literal absolute path; placeholders in it + must NOT be substituted against the cluster dict.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + # If placeholder substitution happened, {user-id} would be replaced + # and the path would change, potentially finding a different file. + # Write the file at the literal path (no substitution expected). + literal_path = tmp_dir / "threshold.json" + literal_path.write_text(json.dumps({})) + # Use a path that contains no placeholders — the real contract is + # the path is used verbatim. The test confirms successful load. + cfg = _base_config_dict(str(literal_path)) + config_path = self._write_config(tmp_dir, cfg) + raw, thresholds = substitute_config(config_path, {}) + self.assertIsInstance(thresholds, dict) + + def test_returns_tuple_of_raw_and_thresholds(self): + """substitute_config must return a (raw_dict, thresholds_dict) tuple.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + threshold_data = {"ISL=128,OSL=2048,TP=8,CONC=16": {"metric": "v"}} + threshold_path = tmp_dir / "threshold.json" + threshold_path.write_text(json.dumps(threshold_data)) + cfg = dict(_base_config_dict(str(threshold_path))) + config_path = self._write_config(tmp_dir, cfg) + result = substitute_config(config_path, {}) + self.assertIsInstance(result, tuple) + self.assertEqual(len(result), 2) + raw, thresholds = result + self.assertIsInstance(raw, dict) + self.assertIsInstance(thresholds, dict) + + def test_threshold_comment_keys_stripped(self): + """Keys starting with '_' (comment keys) in the threshold file must be + stripped from the returned thresholds dict.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + threshold_data = { + "_comment": "this is documentation", + "cell1": {"metric": "v"}, + } + threshold_path = tmp_dir / "threshold.json" + threshold_path.write_text(json.dumps(threshold_data)) + cfg = dict(_base_config_dict(str(threshold_path))) + config_path = self._write_config(tmp_dir, cfg) + _, thresholds = substitute_config(config_path, {}) + self.assertNotIn("_comment", thresholds) + self.assertIn("cell1", thresholds) + + +class TestSubstituteConfigPlaceholders(unittest.TestCase): + """Placeholder substitution behavior preserved from old behavior.""" + + def _write_files(self, tmp_dir: Path, config_dict: dict, threshold_dict: dict = None): + if threshold_dict is None: + threshold_dict = {} + threshold_path = tmp_dir / "threshold.json" + threshold_path.write_text(json.dumps(threshold_dict)) + cfg = dict(config_dict) + cfg["threshold_json"] = str(threshold_path) + config_path = tmp_dir / "variant_config.json" + config_path.write_text(json.dumps(cfg)) + return config_path + + def test_cluster_placeholder_substituted_in_paths(self): + """Cluster dict values are substituted for {key} placeholders.""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + cfg_base = _base_config_dict() + cfg_base["paths"]["log_dir"] = "/logs/{user-id}/run" + config_path = self._write_files(tmp_dir, cfg_base) + cluster = {"username": "jdoe"} + raw, _ = substitute_config(config_path, cluster) + self.assertEqual(raw["paths"]["log_dir"], "/logs/jdoe/run") + + def test_unknown_placeholder_left_verbatim(self): + """An unknown {token} that has no cluster mapping is left as-is (no error).""" + with tempfile.TemporaryDirectory() as tmp: + tmp_dir = Path(tmp) + cfg_base = _base_config_dict() + cfg_base["paths"]["log_dir"] = "/logs/{unknown-token}/run" + config_path = self._write_files(tmp_dir, cfg_base) + raw, _ = substitute_config(config_path, {}) + self.assertEqual(raw["paths"]["log_dir"], "/logs/{unknown-token}/run") + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/utils/unittests/test_gpu.py b/cvs/lib/utils/unittests/test_gpu.py new file mode 100644 index 000000000..a114a9638 --- /dev/null +++ b/cvs/lib/utils/unittests/test_gpu.py @@ -0,0 +1,1264 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unit tests for cvs.lib.utils.gpu. + +Black-box tests authored from the behavioral spec only (impl-blind). The module +contains pure parsers for `amd-smi metric --json` output: no I/O, no hardware, +pure dict transformations. + +Contract under test (from spec): + parse_usage(gpu_entry) -> {"gpu.gfx_activity", "gpu.umc_activity", + "gpu.mm_activity"}; int|None each. Degrades to + None for any missing key or "N/A" value; never raises. + parse_mem_usage(gpu_entry) -> {"gpu.total_vram", "gpu.used_vram", + "gpu.free_vram"}; int|None each. Degrades; never raises. + parse_energy(gpu_entry) -> {"gpu.energy_j"}; float|None. Degrades; never raises. + parse_gpu_metrics(raw) -> single dict with all 7 gpu.* keys. + activity fields averaged across GPUs; + vram + energy_j summed across GPUs. + [] -> all 7 keys present, all None. Never raises. + GPU_METRICS / GPU_METRIC_UNITS: every metric short_name has a matching unit; + parse_gpu_metrics([full]) emits "gpu." for every k. + +Framework: unittest.TestCase + self.subTest + unittest.mock (no pytest). +''' + +import pathlib +import unittest +from unittest.mock import MagicMock, patch + +from cvs.lib.utils.gpu import ( + GPU_METRICS, + GPU_METRIC_UNITS, + _RAW_GPU_FIELDS, + _RAW_GPU_FIELD_UNITS, + _RECORD_SEP, + GpuPollerHandle, + _mean, + agg_readings, + capture_gpu_metrics, + start_gpu_poller, + stop_and_collect_gpu_poller, + parse_usage, + parse_mem_usage, + parse_energy, + parse_gpu_metrics, +) + +# --------------------------------------------------------------------------- +# Shared fixtures — amd-smi JSON schema (one GPU entry) +# --------------------------------------------------------------------------- + +# The seven spec'd metrics, each as the bare "gpu." key produced by +# the parsers / aggregator. +ACTIVITY_KEYS = ["gpu.gfx_activity", "gpu.umc_activity", "gpu.mm_activity"] +VRAM_KEYS = ["gpu.total_vram", "gpu.used_vram", "gpu.free_vram"] +ENERGY_KEY = "gpu.energy_j" +ALL_KEYS = ACTIVITY_KEYS + VRAM_KEYS + [ENERGY_KEY] + + +def _full_gpu_entry(gfx=30, umc=20, mm=10, total=196608, used=4096, free=192512, energy=12345.5): + """A complete amd-smi entry for one GPU with all seven fields present.""" + return { + "usage": { + "gfx_activity": {"value": gfx}, + "umc_activity": {"value": umc}, + "mm_activity": {"value": mm}, + }, + "mem_usage": { + "total_vram": {"value": total}, + "used_vram": {"value": used}, + "free_vram": {"value": free}, + }, + "energy": { + "total_energy_consumption": {"value": energy}, + }, + } + + +# --------------------------------------------------------------------------- +# parse_usage — pure function (dict -> dict) +# --------------------------------------------------------------------------- + + +class TestParseUsage(unittest.TestCase): + """parse_usage extracts ["usage"]; degrades to None; never raises.""" + + def test_full_entry_extracts_all_three(self): + out = parse_usage(_full_gpu_entry(gfx=55, umc=44, mm=33)) + self.assertEqual( + out, + { + "gpu.gfx_activity": 55, + "gpu.umc_activity": 44, + "gpu.mm_activity": 33, + }, + ) + + def test_returns_exactly_the_three_activity_keys(self): + out = parse_usage(_full_gpu_entry()) + self.assertEqual(set(out.keys()), set(ACTIVITY_KEYS)) + + def test_value_types_are_int(self): + out = parse_usage(_full_gpu_entry(gfx=1, umc=2, mm=3)) + for k in ACTIVITY_KEYS: + with self.subTest(key=k): + self.assertIsInstance(out[k], int) + + def test_degradation_table(self): + """Each degraded-input shape maps every activity field to None. + + Boundary classes: empty dict, missing "usage", "N/A" string value. + """ + na = "N/A" + cases = [ + # (description, gpu_entry) + ("empty entry", {}), + ("missing usage key", {"mem_usage": {}}), + ( + "all fields N/A", + { + "usage": { + "gfx_activity": {"value": na}, + "umc_activity": {"value": na}, + "mm_activity": {"value": na}, + } + }, + ), + ] + expected = {k: None for k in ACTIVITY_KEYS} + for desc, entry in cases: + with self.subTest(case=desc): + self.assertEqual(parse_usage(entry), expected) + + def test_partial_entry_degrades_only_missing_field(self): + """One field missing/N/A -> None for that field; others extracted.""" + entry = { + "usage": { + "gfx_activity": {"value": 77}, + "umc_activity": {"value": "N/A"}, + # mm_activity entirely absent + } + } + out = parse_usage(entry) + self.assertEqual(out["gpu.gfx_activity"], 77) + self.assertIsNone(out["gpu.umc_activity"]) + self.assertIsNone(out["gpu.mm_activity"]) + + def test_zero_values_not_coerced_to_none(self): + """0 is a valid reading (fully idle GPU); must not degrade to None.""" + out = parse_usage(_full_gpu_entry(gfx=0, umc=0, mm=0)) + self.assertEqual(out["gpu.gfx_activity"], 0) + self.assertEqual(out["gpu.umc_activity"], 0) + self.assertEqual(out["gpu.mm_activity"], 0) + + def test_never_raises_on_malformed_shapes(self): + """Contract: degrades, never raises. Always returns all three keys as None.""" + malformed = [ + {}, + {"usage": {}}, + {"usage": {"gfx_activity": {}}}, + ] + for entry in malformed: + with self.subTest(entry=entry): + out = parse_usage(entry) + self.assertEqual(set(out.keys()), set(ACTIVITY_KEYS)) + for k in ACTIVITY_KEYS: + self.assertIsNone(out[k]) + + +# --------------------------------------------------------------------------- +# parse_mem_usage — pure function (dict -> dict) +# --------------------------------------------------------------------------- + + +class TestParseMemUsage(unittest.TestCase): + """parse_mem_usage extracts ["mem_usage"]; degrades; never raises.""" + + def test_full_entry_extracts_all_three(self): + out = parse_mem_usage(_full_gpu_entry(total=196608, used=4096, free=192512)) + self.assertEqual( + out, + { + "gpu.total_vram": 196608, + "gpu.used_vram": 4096, + "gpu.free_vram": 192512, + }, + ) + + def test_returns_exactly_the_three_vram_keys(self): + out = parse_mem_usage(_full_gpu_entry()) + self.assertEqual(set(out.keys()), set(VRAM_KEYS)) + + def test_value_types_are_int(self): + out = parse_mem_usage(_full_gpu_entry(total=10, used=3, free=7)) + for k in VRAM_KEYS: + with self.subTest(key=k): + self.assertIsInstance(out[k], int) + + def test_degradation_table(self): + na = "N/A" + cases = [ + ("empty entry", {}), + ("missing mem_usage", {"usage": {}}), + ( + "all N/A", + { + "mem_usage": { + "total_vram": {"value": na}, + "used_vram": {"value": na}, + "free_vram": {"value": na}, + } + }, + ), + ] + expected = {k: None for k in VRAM_KEYS} + for desc, entry in cases: + with self.subTest(case=desc): + self.assertEqual(parse_mem_usage(entry), expected) + + def test_partial_entry_degrades_only_missing_field(self): + entry = { + "mem_usage": { + "total_vram": {"value": 1000}, + "used_vram": {"value": "N/A"}, + # free_vram absent + } + } + out = parse_mem_usage(entry) + self.assertEqual(out["gpu.total_vram"], 1000) + self.assertIsNone(out["gpu.used_vram"]) + self.assertIsNone(out["gpu.free_vram"]) + + def test_zero_values_not_coerced_to_none(self): + """0 is a valid reading (idle GPU); must not degrade to None.""" + out = parse_mem_usage(_full_gpu_entry(total=0, used=0, free=0)) + self.assertEqual(out["gpu.total_vram"], 0) + self.assertEqual(out["gpu.used_vram"], 0) + self.assertEqual(out["gpu.free_vram"], 0) + + def test_never_raises_on_malformed_shapes(self): + malformed = [ + {}, + {"mem_usage": {}}, + {"mem_usage": {"used_vram": {}}}, + ] + for entry in malformed: + with self.subTest(entry=entry): + out = parse_mem_usage(entry) + self.assertEqual(set(out.keys()), set(VRAM_KEYS)) + for k in VRAM_KEYS: + self.assertIsNone(out[k]) + + +# --------------------------------------------------------------------------- +# parse_energy — pure function (dict -> dict) +# --------------------------------------------------------------------------- + + +class TestParseEnergy(unittest.TestCase): + """parse_energy extracts total_energy_consumption; degrades; never raises.""" + + def test_full_entry_extracts_energy(self): + out = parse_energy(_full_gpu_entry(energy=99999.25)) + self.assertEqual(out, {"gpu.energy_j": 99999.25}) + + def test_returns_exactly_the_energy_key(self): + out = parse_energy(_full_gpu_entry()) + self.assertEqual(set(out.keys()), {ENERGY_KEY}) + + def test_value_type_is_float(self): + out = parse_energy(_full_gpu_entry(energy=1.5)) + self.assertIsInstance(out[ENERGY_KEY], float) + + def test_degradation_table(self): + na = "N/A" + cases = [ + ("empty entry", {}), + ("missing energy", {"usage": {}}), + ("missing total_energy_consumption", {"energy": {}}), + ( + "N/A value", + {"energy": {"total_energy_consumption": {"value": na}}}, + ), + ] + for desc, entry in cases: + with self.subTest(case=desc): + self.assertEqual(parse_energy(entry), {ENERGY_KEY: None}) + + def test_never_raises_on_malformed_shapes(self): + malformed = [ + {}, + {"energy": {}}, + {"energy": {"total_energy_consumption": {}}}, + ] + for entry in malformed: + with self.subTest(entry=entry): + out = parse_energy(entry) + self.assertEqual(set(out.keys()), {ENERGY_KEY}) + self.assertIsNone(out[ENERGY_KEY]) + + def test_zero_energy_not_coerced_to_none(self): + """0.0 is a valid reading (GPU powered but idle); must not degrade to None.""" + out = parse_energy(_full_gpu_entry(energy=0.0)) + self.assertEqual(out[ENERGY_KEY], 0.0) + self.assertIsInstance(out[ENERGY_KEY], float) + + def test_int_energy_coerced_to_float(self): + """parse_energy must return float even when the raw value is a Python int.""" + out = parse_energy(_full_gpu_entry(energy=100)) + self.assertIsInstance(out[ENERGY_KEY], float) + + +# --------------------------------------------------------------------------- +# parse_gpu_metrics — pure aggregator (list -> dict) +# --------------------------------------------------------------------------- + + +class TestParseGpuMetrics(unittest.TestCase): + """Aggregates per-GPU entries: activity averaged, vram + energy summed.""" + + # --- key-presence contract --- + + def test_all_seven_keys_present_for_full_entry(self): + out = parse_gpu_metrics([_full_gpu_entry()]) + self.assertIsInstance(out, dict) + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + + def test_empty_list_yields_all_keys_none(self): + """[] -> all 7 keys present, every value None. Never raises.""" + out = parse_gpu_metrics([]) + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + for k in ALL_KEYS: + with self.subTest(key=k): + self.assertIsNone(out[k]) + + # --- single-GPU identity invariant --- + + def test_single_gpu_equals_that_gpus_values(self): + """Single GPU: averaged/summed result equals that GPU's values exactly.""" + entry = _full_gpu_entry(gfx=30, umc=20, mm=10, total=196608, used=4096, free=192512, energy=500.0) + out = parse_gpu_metrics([entry]) + self.assertEqual(out["gpu.gfx_activity"], 30) + self.assertEqual(out["gpu.umc_activity"], 20) + self.assertEqual(out["gpu.mm_activity"], 10) + self.assertEqual(out["gpu.total_vram"], 196608) + self.assertEqual(out["gpu.used_vram"], 4096) + self.assertEqual(out["gpu.free_vram"], 192512) + self.assertEqual(out["gpu.energy_j"], 500.0) + + # --- aggregation semantics: average vs sum --- + + def test_activity_fields_averaged_across_gpus(self): + """gfx/umc/mm averaged. Odd-sum pair verifies true division, not floor.""" + g0 = _full_gpu_entry(gfx=10, umc=40, mm=60) + g1 = _full_gpu_entry(gfx=21, umc=80, mm=20) + out = parse_gpu_metrics([g0, g1]) + self.assertEqual(out["gpu.gfx_activity"], 15.5) # (10+21)/2 — not 15 + self.assertEqual(out["gpu.umc_activity"], 60) # (40+80)/2 + self.assertEqual(out["gpu.mm_activity"], 40) # (60+20)/2 + + def test_vram_and_energy_summed_across_gpus(self): + """total/used/free_vram and energy_j summed across GPUs.""" + g0 = _full_gpu_entry(total=100, used=30, free=70, energy=1.5) + g1 = _full_gpu_entry(total=200, used=50, free=150, energy=2.5) + out = parse_gpu_metrics([g0, g1]) + self.assertEqual(out["gpu.total_vram"], 300) + self.assertEqual(out["gpu.used_vram"], 80) + self.assertEqual(out["gpu.free_vram"], 220) + self.assertEqual(out["gpu.energy_j"], 4.0) + + def test_activity_aggregation_is_average_not_sum(self): + """Guards against an impl that sums activity instead of averaging: + two equal nonzero GPUs must yield the per-GPU value, not double it.""" + g = _full_gpu_entry(gfx=50, umc=50, mm=50) + out = parse_gpu_metrics([g, _full_gpu_entry(gfx=50, umc=50, mm=50)]) + self.assertEqual(out["gpu.gfx_activity"], 50) + self.assertNotEqual(out["gpu.gfx_activity"], 100) + + def test_vram_aggregation_is_sum_not_average(self): + """Guards against an impl that averages vram/energy instead of summing: + two equal GPUs must total double, not stay equal.""" + g0 = _full_gpu_entry(total=100, used=40, free=60, energy=10.0) + g1 = _full_gpu_entry(total=100, used=40, free=60, energy=10.0) + out = parse_gpu_metrics([g0, g1]) + self.assertEqual(out["gpu.total_vram"], 200) + self.assertEqual(out["gpu.energy_j"], 20.0) + self.assertNotEqual(out["gpu.total_vram"], 100) + + # --- partial-entry aggregation --- + + def test_partial_entry_field_excluded_others_aggregated(self): + """A field missing on one GPU -> aggregate the remaining GPUs for it; + other fields still aggregate across all GPUs that have them.""" + g0 = _full_gpu_entry(gfx=20, total=100, used=40, free=60, energy=5.0) + # g1 has no usage block at all -> gfx None for g1 + g1 = { + "mem_usage": { + "total_vram": {"value": 200}, + "used_vram": {"value": 60}, + "free_vram": {"value": 140}, + }, + "energy": {"total_energy_consumption": {"value": 7.0}}, + } + out = parse_gpu_metrics([g0, g1]) + # activity only present on g0 -> aggregate is just g0's values + self.assertEqual(out["gpu.gfx_activity"], 20) + self.assertEqual(out["gpu.umc_activity"], 20) # g0 fixture default + self.assertEqual(out["gpu.mm_activity"], 10) # g0 fixture default + # vram present on both -> summed + self.assertEqual(out["gpu.total_vram"], 300) + self.assertEqual(out["gpu.used_vram"], 100) + self.assertEqual(out["gpu.free_vram"], 200) + # energy present on both -> summed + self.assertEqual(out["gpu.energy_j"], 12.0) + + def test_field_absent_on_all_gpus_yields_none(self): + """If no GPU supplies a field, the aggregate for that field is None, + while present fields still aggregate.""" + no_energy = { + "usage": { + "gfx_activity": {"value": 10}, + "umc_activity": {"value": 10}, + "mm_activity": {"value": 10}, + }, + "mem_usage": { + "total_vram": {"value": 100}, + "used_vram": {"value": 50}, + "free_vram": {"value": 50}, + }, + } + out = parse_gpu_metrics([no_energy, dict(no_energy)]) + self.assertIsNone(out["gpu.energy_j"]) + self.assertEqual(out["gpu.gfx_activity"], 10) + self.assertEqual(out["gpu.total_vram"], 200) + + def test_single_gpu_aggregated_field_types(self): + """Activity and vram fields from a single full entry must be int (or float for energy).""" + out = parse_gpu_metrics([_full_gpu_entry(gfx=10, umc=20, mm=30, total=1000, used=200, free=800, energy=5.0)]) + for k in ACTIVITY_KEYS: + with self.subTest(key=k): + self.assertIsInstance(out[k], (int, float)) + for k in VRAM_KEYS: + with self.subTest(key=k): + self.assertIsInstance(out[k], int) + self.assertIsInstance(out[ENERGY_KEY], float) + + def test_partial_vram_none_excluded_from_sum(self): + """GPU with no mem_usage block: its vram fields are None and excluded; + only the GPU that has vram contributes to the sum.""" + g0 = _full_gpu_entry(total=100, used=40, free=60, energy=2.0) + g1 = { + "usage": {"gfx_activity": {"value": 10}, "umc_activity": {"value": 10}, "mm_activity": {"value": 10}}, + "energy": {"total_energy_consumption": {"value": 3.0}}, + } + out = parse_gpu_metrics([g0, g1]) + self.assertEqual(out["gpu.total_vram"], 100) + self.assertEqual(out["gpu.used_vram"], 40) + self.assertEqual(out["gpu.free_vram"], 60) + self.assertEqual(out["gpu.energy_j"], 5.0) + + def test_partial_energy_none_excluded_from_sum(self): + """GPU with no energy block: its energy is None and excluded; + only the GPU that has energy contributes to the sum.""" + g0 = _full_gpu_entry(energy=500.0) + g1 = { + "usage": {"gfx_activity": {"value": 5}, "umc_activity": {"value": 5}, "mm_activity": {"value": 5}}, + "mem_usage": {"total_vram": {"value": 50}, "used_vram": {"value": 10}, "free_vram": {"value": 40}}, + } + out = parse_gpu_metrics([g0, g1]) + self.assertEqual(out["gpu.energy_j"], 500.0) + + def test_zero_vram_not_excluded_from_aggregation(self): + """total_vram=0 is valid; a falsy-zero aggregation bug (if val: acc += val) + would skip 0 and return None instead of 0. Single-GPU with all-zero VRAM.""" + out = parse_gpu_metrics([_full_gpu_entry(total=0, used=0, free=0)]) + self.assertEqual(out["gpu.total_vram"], 0) + self.assertEqual(out["gpu.used_vram"], 0) + self.assertEqual(out["gpu.free_vram"], 0) + + def test_zero_energy_not_excluded_from_aggregation(self): + """energy=0.0 is valid; a falsy-zero aggregation bug (if energy: skip) would + incorrectly exclude it. Two GPUs each with energy=0.0 must sum to 0.0.""" + g0 = _full_gpu_entry(energy=0.0) + g1 = _full_gpu_entry(energy=0.0) + out = parse_gpu_metrics([g0, g1]) + self.assertEqual(out["gpu.energy_j"], 0.0) + self.assertIsInstance(out["gpu.energy_j"], float) + + def test_zero_activity_not_excluded_from_average(self): + """gfx_activity=0 is valid (GPU idle). A falsy-zero bug in the aggregator + would exclude it from the average, giving wrong denominator+numerator.""" + g0 = _full_gpu_entry(gfx=0) + g1 = _full_gpu_entry(gfx=20) + out = parse_gpu_metrics([g0, g1]) + self.assertEqual(out["gpu.gfx_activity"], 10.0) # (0+20)/2, not 20/1=20 + + def test_partial_none_activity_averaging_three_gpus(self): + """With N=3 where one has no activity, mean of non-None values is: + (30 + 60) / 2 = 45.0 — not sum=90, not divide-by-3=30.""" + g_no_usage = { + "mem_usage": {"total_vram": {"value": 50}, "used_vram": {"value": 20}, "free_vram": {"value": 30}}, + "energy": {"total_energy_consumption": {"value": 1.0}}, + } + g0 = _full_gpu_entry(gfx=30) + g1 = _full_gpu_entry(gfx=60) + out = parse_gpu_metrics([g0, g_no_usage, g1]) + self.assertEqual(out["gpu.gfx_activity"], 45.0) + + def test_activity_averaging_three_full_gpus(self): + """N=3 averaging: (10+20+30)/3=20.0. Guards against hardcoded denominator=2.""" + g0 = _full_gpu_entry(gfx=10, umc=0, mm=5) + g1 = _full_gpu_entry(gfx=20, umc=60, mm=5) + g2 = _full_gpu_entry(gfx=30, umc=120, mm=5) + out = parse_gpu_metrics([g0, g1, g2]) + self.assertEqual(out["gpu.gfx_activity"], 20.0) # (10+20+30)/3 + self.assertEqual(out["gpu.umc_activity"], 60.0) # (0+60+120)/3 + self.assertEqual(out["gpu.mm_activity"], 5.0) # (5+5+5)/3 + + def test_vram_and_energy_summed_three_gpus(self): + """N=3 sum: guards against loop body that caps at 2 entries or re-inits acc.""" + g0 = _full_gpu_entry(total=100, used=10, free=90, energy=1.0) + g1 = _full_gpu_entry(total=200, used=20, free=180, energy=2.0) + g2 = _full_gpu_entry(total=300, used=30, free=270, energy=3.0) + out = parse_gpu_metrics([g0, g1, g2]) + self.assertEqual(out["gpu.total_vram"], 600) + self.assertEqual(out["gpu.used_vram"], 60) + self.assertEqual(out["gpu.free_vram"], 540) + self.assertEqual(out["gpu.energy_j"], 6.0) + + def test_never_raises_on_list_of_empty_entries(self): + """Contract: never raises. All-empty entries -> all keys present, None.""" + out = parse_gpu_metrics([{}, {}, {}]) + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + for k in ALL_KEYS: + with self.subTest(key=k): + self.assertIsNone(out[k]) + + +# --------------------------------------------------------------------------- +# capture_gpu_metrics — I/O subsystem (orch-delegating, not a pure parser) +# Classification: integration boundary; tested only at the mock seam. +# Contract: calls orch to run amd-smi, passes the JSON list to parse_gpu_metrics, +# returns whatever parse_gpu_metrics returns. Never raises on malformed output. +# --------------------------------------------------------------------------- + + +class TestCaptureGpuMetrics(unittest.TestCase): + """capture_gpu_metrics delegates to parse_gpu_metrics and wraps the orch call. + + The function requires a live ContainerOrchestrator to invoke amd-smi, so + unit tests mock the orch dependency and verify delegation semantics only. + They never assert on hardware-specific values. + """ + + def _make_orch(self, raw_gpu_list): + """Return a mock orchestrator whose exec_on_head result decodes to raw_gpu_list. + + The real ContainerOrchestrator.exec_on_head(cmd) returns {host: str}; + we mock the same shape so tests are grounded in the actual interface contract. + """ + import json + + orch = MagicMock() + orch.exec_on_head.return_value = {"node0": json.dumps(raw_gpu_list)} + return orch + + def test_happy_path_key_set_matches_all_keys(self): + """Given a valid amd-smi JSON list, capture_gpu_metrics returns all 7 keys, + delegates to parse_gpu_metrics, and passes the parsed values through.""" + orch = self._make_orch([_full_gpu_entry()]) + with patch("cvs.lib.utils.gpu.parse_gpu_metrics", wraps=parse_gpu_metrics) as mock_parse: + out = capture_gpu_metrics(orch) + self.assertIsInstance(out, dict) + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + mock_parse.assert_called_once_with([_full_gpu_entry()]) + # Pin the exact command string sent to amd-smi (host-side, no sudo needed). + orch.exec_on_head.assert_called_once_with("amd-smi metric --json", print_console=False) + # Verify parse result is actually returned, not silently discarded. + self.assertEqual(out["gpu.gfx_activity"], 30) + self.assertIsNotNone(out["gpu.total_vram"]) + + def test_multi_host_entries_aggregated_together(self): + """All hosts' GPU entries must be pooled before aggregation. + + A mutant that reads only the first host's data would yield gfx=10 + (average of one entry), not 15.0 (average across both hosts' entries). + """ + import json + + orch = MagicMock() + orch.exec_on_head.return_value = { + "node0": json.dumps([_full_gpu_entry(gfx=10)]), + "node1": json.dumps([_full_gpu_entry(gfx=20)]), + } + out = capture_gpu_metrics(orch) + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + self.assertAlmostEqual(out["gpu.gfx_activity"], 15.0) + + def test_no_raise_on_empty_gpu_list(self): + """Empty GPU list -> all 7 keys, all None. Must not raise.""" + orch = self._make_orch([]) + out = capture_gpu_metrics(orch) + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + for k in ALL_KEYS: + with self.subTest(key=k): + self.assertIsNone(out[k]) + + def test_no_raise_on_malformed_orch_output(self): + """If orch returns non-JSON text, capture_gpu_metrics degrades; never raises.""" + orch = MagicMock() + orch.exec_on_head.return_value = {"node0": "not valid json at all"} + try: + out = capture_gpu_metrics(orch) + except Exception as exc: # noqa: BLE001 + self.fail(f"capture_gpu_metrics raised unexpectedly: {exc!r}") + else: + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + for k in ALL_KEYS: + with self.subTest(key=k): + self.assertIsNone(out[k]) + + def test_gpu_data_envelope_unwrapped(self): + """ROCm 6.x amd-smi wraps the GPU list as {"gpu_data": [...]}; must be unwrapped.""" + import json + + orch = MagicMock() + orch.exec_on_head.return_value = {"node0": json.dumps({"gpu_data": [_full_gpu_entry(gfx=42)]})} + out = capture_gpu_metrics(orch) + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + self.assertEqual(out["gpu.gfx_activity"], 42) + + def test_no_raise_on_valid_json_wrong_type(self): + """Valid JSON that decodes to a non-list (dict, null, scalar, string) + must degrade gracefully — never raises, returns all-None.""" + import json + + non_list_values = [{}, None, 42, "string"] + for val in non_list_values: + with self.subTest(decoded_type=type(val).__name__): + orch = MagicMock() + orch.exec_on_head.return_value = {"node0": json.dumps(val)} + try: + out = capture_gpu_metrics(orch) + except Exception as exc: # noqa: BLE001 + self.fail(f"capture_gpu_metrics raised on decoded {val!r}: {exc!r}") + else: + self.assertEqual(set(out.keys()), set(ALL_KEYS)) + for k in ALL_KEYS: + self.assertIsNone(out[k]) + + +# --------------------------------------------------------------------------- +# GPU_METRICS / GPU_METRIC_UNITS — module constants (invariants) +# --------------------------------------------------------------------------- + + +class TestGpuMetricsConstants(unittest.TestCase): + """Invariants tying GPU_METRICS, GPU_METRIC_UNITS, _RAW_GPU_FIELDS, and parser output keys.""" + + # Raw amd-smi parser output fields (internal; not surfaced as HTML rows). + EXPECTED_RAW_NAMES = { + "gfx_activity", + "umc_activity", + "mm_activity", + "total_vram", + "used_vram", + "free_vram", + "energy_j", + } + + # --- GPU_METRICS (derived, human-readable) --- + + def test_derived_unit_strings_match_spec(self): + """Unit strings pinned to spec values.""" + EXPECTED_UNITS = { + "peak_gpu_memory_mb": "MB", + "model_load_memory_mb": "MB", + "model_load_s": "s", + "gpu_bandwidth_util_pct": "%", + "gpu_compute_util_pct": "%", + } + self.assertEqual(GPU_METRIC_UNITS, EXPECTED_UNITS) + + # --- _RAW_GPU_FIELDS (amd-smi parser output) --- + + def test_raw_fields_covers_all_seven_amd_smi_fields(self): + raw_names = {short for short, _unit in _RAW_GPU_FIELDS} + self.assertEqual(raw_names, self.EXPECTED_RAW_NAMES) + + def test_raw_unit_strings_match_spec(self): + EXPECTED_RAW_UNITS = { + "gfx_activity": "%", + "umc_activity": "%", + "mm_activity": "%", + "total_vram": "MB", + "used_vram": "MB", + "free_vram": "MB", + "energy_j": "J", + } + self.assertEqual(_RAW_GPU_FIELD_UNITS, EXPECTED_RAW_UNITS) + + def test_parse_gpu_metrics_emits_key_for_every_raw_field(self): + """parse_gpu_metrics([full]) produces "gpu." for every k in _RAW_GPU_FIELDS.""" + self.assertGreater(len(_RAW_GPU_FIELDS), 0, "_RAW_GPU_FIELDS must not be empty") + out = parse_gpu_metrics([_full_gpu_entry()]) + for short, _unit in _RAW_GPU_FIELDS: + with self.subTest(metric=short): + self.assertIn(f"gpu.{short}", out) + + def test_derived_metrics_not_emitted_by_parser(self): + """GPU_METRICS (derived) are computed by the calling suite, not by the + parser. parse_gpu_metrics must NOT emit keys for derived short names.""" + out = parse_gpu_metrics([_full_gpu_entry()]) + for short, _unit in GPU_METRICS: + with self.subTest(metric=short): + self.assertNotIn(f"gpu.{short}", out) + + +class TestMean(unittest.TestCase): + def test_empty(self): + self.assertIsNone(_mean([])) + + def test_all_none(self): + self.assertIsNone(_mean([None, None])) + + def test_normal(self): + self.assertAlmostEqual(_mean([1.0, 3.0]), 2.0) + + def test_skips_none(self): + self.assertAlmostEqual(_mean([None, 4.0, None, 2.0]), 3.0) + + +class TestAggReadings(unittest.TestCase): + def test_empty(self): + result = agg_readings([]) + self.assertIsNone(result["peak_gpu_memory_mb"]) + self.assertIsNone(result["gpu_compute_util_pct"]) + self.assertIsNone(result["gpu_bandwidth_util_pct"]) + + def test_all_none_values(self): + readings = [{"gpu.used_vram": None, "gpu.gfx_activity": None, "gpu.umc_activity": None}] + result = agg_readings(readings) + self.assertIsNone(result["peak_gpu_memory_mb"]) + + def test_normal(self): + readings = [ + {"gpu.used_vram": 1000, "gpu.gfx_activity": 80.0, "gpu.umc_activity": 60.0}, + {"gpu.used_vram": 2000, "gpu.gfx_activity": 90.0, "gpu.umc_activity": 70.0}, + ] + result = agg_readings(readings) + self.assertEqual(result["peak_gpu_memory_mb"], 2000) + self.assertAlmostEqual(result["gpu_compute_util_pct"], 85.0) + self.assertAlmostEqual(result["gpu_bandwidth_util_pct"], 65.0) + + +class TestCaptureGpuMetricsMultiNode(unittest.TestCase): + """Tests for capture_gpu_metrics with the nodes= parameter (orch.exec(hosts=...)).""" + + def _make_gpu_json(self, used_vram: int, gfx: float = 80.0) -> str: + import json + + return json.dumps( + [ + { + "usage": { + "gfx_activity": {"value": gfx}, + "umc_activity": {"value": 10.0}, + "mm_activity": {"value": "N/A"}, + }, + "mem_usage": { + "used_vram": {"value": used_vram}, + "total_vram": {"value": used_vram + 1000}, + "free_vram": {"value": 1000}, + }, + "energy": {"total_energy_consumption": {"value": 50.0}}, + } + ] + ) + + def _make_exec_by_hosts(self, host_to_vram: dict, gfx: float = 80.0): + """Build an orch.exec side_effect keyed by the hosts= kwarg.""" + + def _exec(cmd, hosts=None, print_console=True): + return {h: self._make_gpu_json(host_to_vram[h], gfx) for h in hosts} + + return _exec + + def test_nodes_none_calls_exec_on_head(self): + """nodes=None must call orch.exec_on_head (regression guard).""" + orch = MagicMock() + orch.exec_on_head.return_value = {"host0": self._make_gpu_json(1000)} + from cvs.lib.utils.gpu import capture_gpu_metrics + + result = capture_gpu_metrics(orch, nodes=None) + orch.exec_on_head.assert_called_once_with("amd-smi metric --json", print_console=False) + self.assertEqual(result["gpu.used_vram"], 1000) + + def test_nodes_provided_calls_orch_exec_with_hosts_not_exec_on_head(self): + """nodes provided: orch.exec(cmd, hosts=...) is called, orch.exec_on_head is NOT.""" + orch = MagicMock() + orch.exec.side_effect = self._make_exec_by_hosts({"prefill-host": 2000, "decode-host": 3000}) + from cvs.lib.utils.gpu import capture_gpu_metrics + + capture_gpu_metrics( + orch, + nodes=[("prefill-0", ["prefill-host"]), ("decode-0", ["decode-host"])], + ) + orch.exec_on_head.assert_not_called() + orch.exec.assert_any_call("amd-smi metric --json", hosts=["prefill-host"], print_console=False) + orch.exec.assert_any_call("amd-smi metric --json", hosts=["decode-host"], print_console=False) + + def test_nodes_vram_summed_across_nodes(self): + """VRAM from all nodes is summed in the merged result.""" + orch = MagicMock() + orch.exec.side_effect = self._make_exec_by_hosts({"p": 2000, "d": 3000}) + from cvs.lib.utils.gpu import capture_gpu_metrics + + result = capture_gpu_metrics(orch, nodes=[("prefill-0", ["p"]), ("decode-0", ["d"])]) + self.assertEqual(result["gpu.used_vram"], 5000) + + def test_nodes_activity_averaged_across_nodes(self): + """GFX activity from all nodes is averaged.""" + orch = MagicMock() + + def _exec(cmd, hosts=None, print_console=True): + gfx = 60.0 if hosts == ["p"] else 100.0 + return {hosts[0]: self._make_gpu_json(1000, gfx)} + + orch.exec.side_effect = _exec + from cvs.lib.utils.gpu import capture_gpu_metrics + + result = capture_gpu_metrics(orch, nodes=[("prefill-0", ["p"]), ("decode-0", ["d"])]) + self.assertAlmostEqual(result["gpu.gfx_activity"], 80.0) + + def test_nodes_exception_propagates(self): + """Exception from orch.exec propagates (not swallowed).""" + orch = MagicMock() + orch.exec.side_effect = RuntimeError("ssh failed") + from cvs.lib.utils.gpu import capture_gpu_metrics + + with self.assertRaises(RuntimeError): + capture_gpu_metrics(orch, nodes=[("prefill-0", ["p"])]) + + def test_nodes_gpu_data_envelope_unwrapped_and_merged(self): + """Each node may independently use the {"gpu_data": [...]} envelope.""" + import json + + orch = MagicMock() + + def _exec(cmd, hosts=None, print_console=True): + vram = {"p": 1000, "d": 2000}[hosts[0]] + return { + hosts[0]: json.dumps( + { + "gpu_data": [ + { + "usage": { + "gfx_activity": {"value": 50.0}, + "umc_activity": {"value": 10.0}, + "mm_activity": {"value": "N/A"}, + }, + "mem_usage": { + "used_vram": {"value": vram}, + "total_vram": {"value": vram + 100}, + "free_vram": {"value": 100}, + }, + "energy": {"total_energy_consumption": {"value": 1.0}}, + } + ] + } + ) + } + + orch.exec.side_effect = _exec + from cvs.lib.utils.gpu import capture_gpu_metrics + + result = capture_gpu_metrics(orch, nodes=[("prefill-0", ["p"]), ("decode-0", ["d"])]) + self.assertEqual(result["gpu.used_vram"], 3000) + + +# --------------------------------------------------------------------------- +# start_gpu_poller / stop_and_collect_gpu_poller / GpuPollerHandle +# +# poll_gpu_metrics (thread + shared-SSH polling) was replaced by a detached +# remote background script per node, read back via ordinary sequential +# exec/exec_on_head calls -- avoids a second OS thread sharing the +# orchestrator's SSH transport with the main log-tail polling thread (real +# HW-observed SessionError(OutOfBoundaryError()) race). +# --------------------------------------------------------------------------- + + +def _gpu_chunk_text(used_vram: int = 1000, gfx: float = 80.0, umc: float = 60.0) -> str: + """One well-formed amd-smi --json chunk (single GPU entry).""" + import json + + return json.dumps([_full_gpu_entry(gfx=gfx, umc=umc, used=used_vram)]) + + +def _poller_file_text(*chunks: str) -> str: + """Join chunks with _RECORD_SEP, well-terminated (trailing separator).""" + return "".join(c + _RECORD_SEP + "\n" for c in chunks) + + +class TestGpuPollerHandle(unittest.TestCase): + def test_fields(self): + handle = GpuPollerHandle(run_id="r1", marker="cvs_gpu_poll_r1", nodes=None, paths="/tmp/cvs_gpu_poll_r1.log") + self.assertEqual(handle.run_id, "r1") + self.assertEqual(handle.marker, "cvs_gpu_poll_r1") + self.assertIsNone(handle.nodes) + self.assertEqual(handle.paths, "/tmp/cvs_gpu_poll_r1.log") + + +class TestStartGpuPollerSingleNode(unittest.TestCase): + """nodes=None: one write+launch pair via orch.exec_on_head.""" + + def test_run_id_sanitized_in_marker_and_paths(self): + """A raw pytest node id (::, [, ], /) must not leak into marker/paths. + + handle.paths is a legitimate filesystem path (e.g. /tmp/.log), + so it's the basename -- not the whole path string -- that must be + free of the sanitized-away characters. + """ + orch = MagicMock() + orch.exec_on_head.return_value = {"head": ""} + handle = start_gpu_poller(orch, run_id="test_foo.py::test_bar[isl-osl-4]") + basename = pathlib.Path(handle.paths).name + for bad_char in ("::", "[", "]", "/"): + with self.subTest(bad_char=bad_char): + self.assertNotIn(bad_char, handle.marker) + self.assertNotIn(bad_char, basename) + self.assertTrue(handle.marker.startswith("cvs_gpu_poll_")) + + def test_launch_uses_exec_on_head_not_exec(self): + orch = MagicMock() + orch.exec_on_head.return_value = {"head": ""} + start_gpu_poller(orch, run_id="r1") + orch.exec.assert_not_called() + self.assertTrue(orch.exec_on_head.called) + + def test_marker_scoped_by_local_user(self): + """Two users polling with the same run_id on a shared node must not + collide on the same /tmp path (Fremont-conda incident: shared /tmp, + second user's write hits a permission error on the first user's + leftover file).""" + orch = MagicMock() + orch.exec_on_head.return_value = {"head": ""} + with patch("cvs.lib.utils.gpu.getpass.getuser", return_value="alice"): + handle_alice = start_gpu_poller(orch, run_id="r1") + with patch("cvs.lib.utils.gpu.getpass.getuser", return_value="bob"): + handle_bob = start_gpu_poller(orch, run_id="r1") + self.assertIn("alice", handle_alice.marker) + self.assertIn("bob", handle_bob.marker) + self.assertNotEqual(handle_alice.marker, handle_bob.marker) + self.assertNotEqual(handle_alice.paths, handle_bob.paths) + + def test_write_cmd_truncates_stale_log(self): + """A leftover log from a prior crashed run under the same marker + must not leak old readings into this run's file (poller appends + via >>, so launch must truncate first).""" + orch = MagicMock() + orch.exec_on_head.return_value = {"head": ""} + start_gpu_poller(orch, run_id="r1") + write_cmd = orch.exec_on_head.call_args_list[0].args[0] + self.assertIn(": >", write_cmd) + + def test_launch_command_contains_nohup_and_record_sep(self): + orch = MagicMock() + orch.exec_on_head.return_value = {"head": ""} + start_gpu_poller(orch, run_id="r1") + all_cmds = " ".join(c.args[0] for c in orch.exec_on_head.call_args_list) + self.assertIn("nohup", all_cmds) + self.assertIn(_RECORD_SEP, all_cmds) + + def test_default_hard_cap_yields_960_iterations(self): + """hard_cap_s=14400, poll_interval_s=15 -> 14400 // 15 == 960.""" + orch = MagicMock() + orch.exec_on_head.return_value = {"head": ""} + start_gpu_poller(orch, run_id="r1", poll_interval_s=15, hard_cap_s=14400) + all_cmds = " ".join(c.args[0] for c in orch.exec_on_head.call_args_list) + self.assertIn("960", all_cmds) + + def test_returns_handle_with_nodes_none_and_str_path(self): + orch = MagicMock() + orch.exec_on_head.return_value = {"head": ""} + handle = start_gpu_poller(orch, run_id="r1") + self.assertIsNone(handle.nodes) + self.assertIsInstance(handle.paths, str) + + def test_launch_failure_propagates(self): + orch = MagicMock() + orch.exec_on_head.side_effect = RuntimeError("ssh failed") + with self.assertRaises(RuntimeError): + start_gpu_poller(orch, run_id="r1") + + +class TestStartGpuPollerMultiNode(unittest.TestCase): + """nodes=[hosts]: one write+launch pair per host via orch.exec(hosts=[host]).""" + + def test_one_launch_call_per_host(self): + orch = MagicMock() + orch.exec.return_value = {"h1": ""} + start_gpu_poller(orch, run_id="r1", nodes=["h1", "h2"]) + orch.exec_on_head.assert_not_called() + hosts_seen = [c.kwargs.get("hosts") for c in orch.exec.call_args_list] + self.assertIn(["h1"], hosts_seen) + self.assertIn(["h2"], hosts_seen) + + def test_returns_handle_with_per_host_paths(self): + orch = MagicMock() + orch.exec.return_value = {"h1": ""} + handle = start_gpu_poller(orch, run_id="r1", nodes=["h1", "h2"]) + self.assertEqual(handle.nodes, ["h1", "h2"]) + self.assertIsInstance(handle.paths, dict) + self.assertEqual(set(handle.paths), {"h1", "h2"}) + + +class TestStopAndCollectGpuPollerSingleNode(unittest.TestCase): + """nodes=None: pkill via exec_on_head, read-back via exec_on_head cat.""" + + def _handle(self): + return GpuPollerHandle(run_id="r1", marker="cvs_gpu_poll_r1", nodes=None, paths="/tmp/cvs_gpu_poll_r1.log") + + def test_pkill_contains_marker(self): + orch = MagicMock() + orch.exec_on_head.return_value = {"head": ""} + stop_and_collect_gpu_poller(orch, self._handle()) + all_cmds = " ".join(c.args[0] for c in orch.exec_on_head.call_args_list) + self.assertIn("cvs_gpu_poll_r1", all_cmds) + + def test_well_terminated_chunks_all_parse(self): + """3 well-formed chunks + correct trailing separator -> 3 readings, 0 failed.""" + text = _poller_file_text(_gpu_chunk_text(1000), _gpu_chunk_text(2000), _gpu_chunk_text(3000)) + orch = MagicMock() + orch.exec_on_head.side_effect = [{"head": ""}, {"head": text}, {"head": ""}] + readings = stop_and_collect_gpu_poller(orch, self._handle()) + self.assertEqual(len(readings), 3) + self.assertEqual(readings[-1]["gpu.used_vram"], 3000) + + def test_trailing_phantom_not_counted_as_failure_but_mid_malformed_is(self): + """Trailing separator's phantom empty chunk must not count as failed; + a genuinely empty/malformed chunk mixed in the middle must.""" + text = ( + _gpu_chunk_text(1000) + + _RECORD_SEP + + "\n" + + "" + + _RECORD_SEP + + "\n" + + _gpu_chunk_text(3000) + + _RECORD_SEP + + "\n" + ) + orch = MagicMock() + orch.exec_on_head.side_effect = [{"head": ""}, {"head": text}, {"head": ""}] + log_path = str(pathlib.Path(__file__).parent / "_tmp_test_gpu_poller_log.txt") + try: + readings = stop_and_collect_gpu_poller(orch, self._handle(), log_path=log_path) + self.assertEqual(len(readings), 2) + content = pathlib.Path(log_path).read_text() + self.assertIn("samples: 3 (1 failed, excluded)", content) + finally: + pathlib.Path(log_path).unlink(missing_ok=True) + + def test_read_failure_degrades_not_raises(self): + orch = MagicMock() + orch.exec_on_head.side_effect = [{"head": ""}, RuntimeError("ssh failed"), {"head": ""}] + try: + readings = stop_and_collect_gpu_poller(orch, self._handle()) + except Exception as exc: # noqa: BLE001 + self.fail(f"stop_and_collect_gpu_poller raised unexpectedly: {exc!r}") + else: + self.assertEqual(readings, []) + + def test_pkill_failure_degrades_not_raises_and_readback_still_happens(self): + text = _poller_file_text(_gpu_chunk_text(1000)) + orch = MagicMock() + orch.exec_on_head.side_effect = [RuntimeError("pkill failed"), {"head": text}, {"head": ""}] + try: + readings = stop_and_collect_gpu_poller(orch, self._handle()) + except Exception as exc: # noqa: BLE001 + self.fail(f"stop_and_collect_gpu_poller raised unexpectedly: {exc!r}") + else: + self.assertEqual(len(readings), 1) + + def test_cleanup_failure_degrades_not_raises(self): + text = _poller_file_text(_gpu_chunk_text(1000)) + orch = MagicMock() + orch.exec_on_head.side_effect = [{"head": ""}, {"head": text}, RuntimeError("rm failed")] + try: + readings = stop_and_collect_gpu_poller(orch, self._handle()) + except Exception as exc: # noqa: BLE001 + self.fail(f"stop_and_collect_gpu_poller raised unexpectedly: {exc!r}") + else: + self.assertEqual(len(readings), 1) + + def test_remote_files_removed_after_readback_single_node(self): + """Fremont-conda incident: shared-node /tmp files must not be left + behind after a run, or the next user hits a permission error on the + stale file. Confirms both script and log paths are rm'd on the node + the poller ran on.""" + text = _poller_file_text(_gpu_chunk_text(1000)) + orch = MagicMock() + orch.exec_on_head.side_effect = [{"head": ""}, {"head": text}, {"head": ""}] + stop_and_collect_gpu_poller(orch, self._handle()) + rm_calls = [c.args[0] for c in orch.exec_on_head.call_args_list if "rm -f" in c.args[0]] + self.assertEqual(len(rm_calls), 1) + self.assertIn("cvs_gpu_poll_r1.sh", rm_calls[0]) + self.assertIn("cvs_gpu_poll_r1.log", rm_calls[0]) + + def test_summary_log_matches_agg_readings(self): + text = _poller_file_text(_gpu_chunk_text(1000, gfx=80.0, umc=60.0), _gpu_chunk_text(2000, gfx=90.0, umc=70.0)) + orch = MagicMock() + orch.exec_on_head.side_effect = [{"head": ""}, {"head": text}, {"head": ""}] + log_path = str(pathlib.Path(__file__).parent / "_tmp_test_gpu_poller_summary.txt") + try: + readings = stop_and_collect_gpu_poller( + orch, self._handle(), log_path=log_path, model_load_s=12.3, model_load_memory_mb=456 + ) + agg = agg_readings(readings) + content = pathlib.Path(log_path).read_text() + self.assertIn("--- summary ---", content) + self.assertIn(f"peak_gpu_memory_mb: {agg['peak_gpu_memory_mb']:.0f} MB", content) + self.assertIn("model_load_memory_mb: 456 MB", content) + self.assertIn("model_load_s: 12.3 s", content) + self.assertIn(f"gpu_compute_util_pct: {agg['gpu_compute_util_pct']:.1f} %", content) + self.assertIn(f"gpu_bandwidth_util_pct: {agg['gpu_bandwidth_util_pct']:.1f} %", content) + finally: + pathlib.Path(log_path).unlink(missing_ok=True) + + +class TestStopAndCollectGpuPollerMultiNode(unittest.TestCase): + """nodes=[hosts]: pkill via exec(hosts=...), round-aligned merge across hosts.""" + + def _handle(self, nodes): + return GpuPollerHandle( + run_id="r1", + marker="cvs_gpu_poll_r1", + nodes=nodes, + paths={h: "/tmp/cvs_gpu_poll_r1.log" for h in nodes}, + ) + + def test_round_alignment_longer_host_extends_not_truncates(self): + """Host A: 3 chunks, host B: 2 chunks -> 3 rounds, round 3 uses only A.""" + text_a = _poller_file_text(_gpu_chunk_text(1000), _gpu_chunk_text(2000), _gpu_chunk_text(3000)) + text_b = _poller_file_text(_gpu_chunk_text(500), _gpu_chunk_text(600)) + orch = MagicMock() + + def _exec(cmd, hosts=None, print_console=True): + if "pkill" in cmd: + return {h: "" for h in hosts} + host = hosts[0] + return {host: text_a if host == "A" else text_b} + + orch.exec.side_effect = _exec + readings = stop_and_collect_gpu_poller(orch, self._handle(["A", "B"])) + self.assertEqual(len(readings), 3) + # round 3 (index 2) merges only host A's 3000 vram entry. + self.assertEqual(readings[2]["gpu.used_vram"], 3000) + + def test_round_alignment_not_counted_as_failed_when_one_host_has_data(self): + text_a = _poller_file_text(_gpu_chunk_text(1000), _gpu_chunk_text(2000), _gpu_chunk_text(3000)) + text_b = _poller_file_text(_gpu_chunk_text(500), _gpu_chunk_text(600)) + orch = MagicMock() + + def _exec(cmd, hosts=None, print_console=True): + if "pkill" in cmd: + return {h: "" for h in hosts} + host = hosts[0] + return {host: text_a if host == "A" else text_b} + + orch.exec.side_effect = _exec + log_path = str(pathlib.Path(__file__).parent / "_tmp_test_gpu_poller_multinode.txt") + try: + stop_and_collect_gpu_poller(orch, self._handle(["A", "B"]), log_path=log_path) + content = pathlib.Path(log_path).read_text() + self.assertIn("samples: 3", content) + self.assertNotIn("failed", content) + finally: + pathlib.Path(log_path).unlink(missing_ok=True) + + def test_round_failed_when_every_contributing_host_malformed(self): + """Round where every host's chunk at that index is malformed/empty -> counted as failed.""" + text_a = _poller_file_text(_gpu_chunk_text(1000), "") + text_b = _poller_file_text(_gpu_chunk_text(500), "") + orch = MagicMock() + + def _exec(cmd, hosts=None, print_console=True): + if "pkill" in cmd: + return {h: "" for h in hosts} + host = hosts[0] + return {host: text_a if host == "A" else text_b} + + orch.exec.side_effect = _exec + readings = stop_and_collect_gpu_poller(orch, self._handle(["A", "B"])) + self.assertEqual(len(readings), 1) + + def test_per_node_vram_summary_reflects_last_successful_round(self): + text_a = _poller_file_text(_gpu_chunk_text(1000), _gpu_chunk_text(2000), _gpu_chunk_text(3000)) + text_b = _poller_file_text(_gpu_chunk_text(500), _gpu_chunk_text(600)) + orch = MagicMock() + + def _exec(cmd, hosts=None, print_console=True): + if "pkill" in cmd: + return {h: "" for h in hosts} + host = hosts[0] + return {host: text_a if host == "A" else text_b} + + orch.exec.side_effect = _exec + log_path = str(pathlib.Path(__file__).parent / "_tmp_test_gpu_poller_pernode.txt") + try: + stop_and_collect_gpu_poller(orch, self._handle(["A", "B"]), log_path=log_path) + content = pathlib.Path(log_path).read_text() + self.assertIn("node_vram_mb [A]: 3000 MB", content) + self.assertIn("node_vram_mb [B]: 600 MB", content) + finally: + pathlib.Path(log_path).unlink(missing_ok=True) + + def test_pkill_broadcasts_to_all_nodes(self): + orch = MagicMock() + orch.exec.return_value = {"A": "", "B": ""} + stop_and_collect_gpu_poller(orch, self._handle(["A", "B"])) + pkill_calls = [c for c in orch.exec.call_args_list if "pkill" in c.args[0]] + self.assertTrue(any(c.kwargs.get("hosts") == ["A", "B"] for c in pkill_calls)) + + def test_remote_files_removed_per_host(self): + """Each host's script/log files must be individually rm'd -- a + shared-node file left behind on any single host is enough to hit + the next user's run with a permission error.""" + orch = MagicMock() + orch.exec.return_value = {"A": "", "B": ""} + stop_and_collect_gpu_poller(orch, self._handle(["A", "B"])) + rm_calls = [c for c in orch.exec.call_args_list if "rm -f" in c.args[0]] + rm_hosts = {c.kwargs.get("hosts", [None])[0] for c in rm_calls} + self.assertEqual(rm_hosts, {"A", "B"}) + for c in rm_calls: + self.assertIn("cvs_gpu_poll_r1.sh", c.args[0]) + self.assertIn("cvs_gpu_poll_r1.log", c.args[0]) + + def test_read_failure_for_one_host_degrades_not_raises(self): + text_a = _poller_file_text(_gpu_chunk_text(1000)) + orch = MagicMock() + + def _exec(cmd, hosts=None, print_console=True): + if "pkill" in cmd: + return {h: "" for h in hosts} + host = hosts[0] + if host == "B": + raise RuntimeError("ssh failed") + return {host: text_a} + + orch.exec.side_effect = _exec + try: + readings = stop_and_collect_gpu_poller(orch, self._handle(["A", "B"])) + except Exception as exc: # noqa: BLE001 + self.fail(f"stop_and_collect_gpu_poller raised unexpectedly: {exc!r}") + else: + self.assertEqual(len(readings), 1) + self.assertEqual(readings[0]["gpu.used_vram"], 1000) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/utils/unittests/test_ib_discovery.py b/cvs/lib/utils/unittests/test_ib_discovery.py new file mode 100644 index 000000000..3d10e7edd --- /dev/null +++ b/cvs/lib/utils/unittests/test_ib_discovery.py @@ -0,0 +1,180 @@ +''' +Copyright 2025 Advanced Micro Devices Inc. +All rights reserved. +''' + +import unittest + +from cvs.lib.utils.ib_discovery import ( + _parse_ibv_devinfo_list, + discover_ib_hca_names, + discover_socket_netdev_name, + resolve_multinode_fabric, +) + + +class _NetdevOrch: + def __init__(self, hosts, responses): + self.hosts = list(hosts) + self._responses = dict(responses) + + def exec(self, cmd, hosts=None, **kwargs): + target_hosts = list(hosts) if hosts is not None else self.hosts + return {h: self._responses.get((h, cmd), "") for h in target_hosts} + + def exec_on_host(self, cmd, hosts=None, **kwargs): + return self.exec(cmd, hosts=hosts, **kwargs) + + +class _ContainerOrch(_NetdevOrch): + """Simulates container exec (broken/minimal) vs host exec (full OS tools).""" + + def exec(self, cmd, hosts=None, **kwargs): + target_hosts = list(hosts) if hosts is not None else self.hosts + return {h: "bash: line 1: ip: command not found\n" for h in target_hosts} + + def exec_on_host(self, cmd, hosts=None, **kwargs): + return _NetdevOrch.exec(self, cmd, hosts=hosts, **kwargs) + + +class TestParseIbvDevinfoList(unittest.TestCase): + def test_parses_newline_and_space_separated_names(self): + self.assertEqual(_parse_ibv_devinfo_list("mlx5_0\nmlx5_1\n"), ["mlx5_0", "mlx5_1"]) + self.assertEqual(_parse_ibv_devinfo_list("mlx5_0 mlx5_1"), ["mlx5_0", "mlx5_1"]) + + def test_ignores_ibv_banner_lines(self): + raw = """8 HCAs found: + rdma3 + rdma0 + rdma2 + rdma1 +""" + self.assertEqual(_parse_ibv_devinfo_list(raw), ["rdma3", "rdma0", "rdma2", "rdma1"]) + + +class TestDiscoverSocketNetdev(unittest.TestCase): + def test_resolves_common_netdev_from_cluster_ips(self): + h0, h1 = "10.32.80.112", "10.32.80.113" + orch = _NetdevOrch( + [h0, h1], + { + (h0, _cmd_for_ip(h0)): "ens51f1np1\n", + (h1, _cmd_for_ip(h1)): "ens51f1np1\n", + }, + ) + self.assertEqual(discover_socket_netdev_name(orch, master_addr=h0), "ens51f1np1") + + def test_raises_on_asymmetric_netdev_names(self): + h0, h1 = "10.32.80.112", "10.32.80.113" + orch = _NetdevOrch( + [h0, h1], + { + (h0, _cmd_for_ip(h0)): "ens51f1np1\n", + (h1, _cmd_for_ip(h1)): "ens51f1np2\n", + }, + ) + with self.assertRaisesRegex(RuntimeError, "asymmetric netdev"): + discover_socket_netdev_name(orch, master_addr=h0) + + def test_rejects_mlx5_hca_name(self): + h0 = "10.32.80.112" + orch = _NetdevOrch([h0], {(h0, _cmd_for_ip(h0)): "mlx5_0\n"}) + with self.assertRaisesRegex(RuntimeError, "no IPv4 netdev"): + discover_socket_netdev_name(orch, master_addr=h0) + + def test_rejects_shell_error_output(self): + h0 = "10.32.80.112" + orch = _NetdevOrch( + [h0], + { + (h0, _cmd_for_ip(h0)): "bash: line 1: ip: command not found\n", + }, + ) + with self.assertRaisesRegex(RuntimeError, "no IPv4 netdev"): + discover_socket_netdev_name(orch, master_addr=h0) + + def test_container_orch_uses_host_exec_not_container_exec(self): + from cvs.lib.utils.ib_discovery import _IBVDEVINFO_CMD + + h0, h1 = "10.32.80.112", "10.32.80.113" + orch = _ContainerOrch( + [h0, h1], + { + (h0, _IBVDEVINFO_CMD): "mlx5_0\nmlx5_1\n", + (h1, _IBVDEVINFO_CMD): "mlx5_0\nmlx5_1\n", + (h0, _cmd_for_ip(h0)): "ens51f1np1\n", + (h1, _cmd_for_ip(h1)): "ens51f1np1\n", + }, + ) + hcas, netdev = resolve_multinode_fabric( + orch, + ib_hca_devices="auto", + ib_netdev="auto", + master_addr=h0, + ) + self.assertEqual(hcas, ["mlx5_0", "mlx5_1"]) + self.assertEqual(netdev, "ens51f1np1") + + +class TestDiscoverIbHcaNames(unittest.TestCase): + def test_parses_ibv_devinfo_list_output(self): + from cvs.lib.utils.ib_discovery import _IBVDEVINFO_CMD + + h0, h1 = "10.32.80.112", "10.32.80.113" + orch = _NetdevOrch( + [h0, h1], + { + (h0, _IBVDEVINFO_CMD): "mlx5_0\nmlx5_1\n", + (h1, _IBVDEVINFO_CMD): "mlx5_0\nmlx5_1\n", + }, + ) + discovered = discover_ib_hca_names(orch) + self.assertEqual(discovered[h0], ["mlx5_0", "mlx5_1"]) + + +class TestResolveMultinodeFabric(unittest.TestCase): + def test_resolves_hcas_and_netdev_from_auto_config(self): + from cvs.lib.utils.ib_discovery import _IBVDEVINFO_CMD + + h0, h1 = "10.32.80.112", "10.32.80.113" + ibv_out = "8 HCAs found:\n rdma0\n rdma1\n" + orch = _NetdevOrch( + [h0, h1], + { + (h0, _IBVDEVINFO_CMD): ibv_out, + (h1, _IBVDEVINFO_CMD): ibv_out, + (h0, _cmd_for_ip(h0)): "ens51f1np1\n", + (h1, _cmd_for_ip(h1)): "ens51f1np1\n", + }, + ) + hcas, netdev = resolve_multinode_fabric( + orch, + ib_hca_devices="auto", + ib_netdev="auto", + master_addr=h0, + ) + self.assertEqual(hcas, ["rdma0", "rdma1"]) + self.assertEqual(netdev, "ens51f1np1") + + +def _cmd_for_ip(ip: str) -> str: + from cvs.lib.utils.ib_discovery import _netdev_for_ip_cmd + + return _netdev_for_ip_cmd(ip) + + +class TestNetdevShellSyntax(unittest.TestCase): + def test_netdev_cmds_use_valid_bash_subshell(self): + ip_cmd = _cmd_for_ip("10.32.80.112") + from cvs.lib.utils.ib_discovery import _netdev_via_route_cmd + + route_cmd = _netdev_via_route_cmd("10.32.80.112") + self.assertIn("IF=$( (ip", ip_cmd) + self.assertNotIn("IF=$((", ip_cmd) + self.assertNotIn("$({ip", ip_cmd) + self.assertIn("IF=$( (ip route", route_cmd) + self.assertNotIn("$({ip route", route_cmd) + + +if __name__ == "__main__": + unittest.main() diff --git a/cvs/lib/utils/verdict.py b/cvs/lib/utils/verdict.py new file mode 100644 index 000000000..4217f3a09 --- /dev/null +++ b/cvs/lib/utils/verdict.py @@ -0,0 +1,88 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +from __future__ import annotations + + +class ThresholdViolation(Exception): + def __init__(self, violations): + self.violations = list(violations) + super().__init__("\n".join(self.violations)) + + +def _to_float(x): + return float(x) + + +def _check_one(metric, actual_raw, spec): + kind = spec["kind"] + # "info": record-only, never gates -- always passes. Used for metrics that + # should appear with a PASS status but carry no threshold to enforce. + if kind == "info": + return None + actual = _to_float(actual_raw) + if kind == "min": + target = _to_float(spec["value"]) + if actual < target: + return f"{metric}: actual {actual} < min {target}" + elif kind == "max_ms": + target = _to_float(spec["value"]) + if actual > target: + return f"{metric}: actual {actual} ms > max {target} ms" + elif kind == "max": + # Unit-agnostic upper bound, for counts like `failed` where `max_ms` + # would be a unit lie. Same comparison, honest message. + target = _to_float(spec["value"]) + if actual > target: + return f"{metric}: actual {actual} > max {target}" + elif kind == "within": + target = _to_float(spec["value"]) + pct = _to_float(spec["tolerance_pct"]) + lo, hi = target * (1 - pct / 100.0), target * (1 + pct / 100.0) + if not (lo <= actual <= hi): + return f"{metric}: actual {actual} outside {target} ±{pct}%" + elif kind == "min_tok_s": + target = _to_float(spec["value"]) + if actual < target: + return f"{metric}: actual {actual} tok/s < min {target} tok/s" + elif kind == "min_ratio": + ref_metric = spec["reference"] + ratio = _to_float(spec["value"]) + actuals = spec.get("_actuals", {}) + if ref_metric not in actuals: + return f"{metric}: reference metric '{ref_metric}' missing from actuals" + if actuals[ref_metric] is None: + return f"{metric}: reference '{ref_metric}' is None (metric unavailable for this run)" + ref_actual = _to_float(actuals[ref_metric]) + if ref_actual == 0: + return f"{metric}: reference '{ref_metric}' is 0; cannot compute ratio" + observed = actual / ref_actual + if observed < ratio: + return f"{metric}: observed ratio {observed:.3f} < min {ratio} (vs {ref_metric})" + else: + return f"{metric}: unknown threshold kind '{kind}'" + return None + + +def evaluate_all(actuals, thresholds): + violations = [] + for metric, spec in thresholds.items(): + if metric not in actuals: + violations.append(f"{metric}: missing from actuals") + continue + if actuals[metric] is None: + # A derived/goodput metric that could not be computed (e.g. + # decode_latency_ratio with no p50, or goodput with no SLO run). + # Loud violation, not a float(None) TypeError. + violations.append(f"{metric}: value is None (metric unavailable for this run)") + continue + spec_with_actuals = dict(spec) + if spec.get("kind") == "min_ratio": + spec_with_actuals["_actuals"] = actuals + v = _check_one(metric, actuals[metric], spec_with_actuals) + if v: + violations.append(v) + if violations: + raise ThresholdViolation(violations) diff --git a/cvs/lib/utils_lib.py b/cvs/lib/utils_lib.py index c6414b795..8208a07f2 100644 --- a/cvs/lib/utils_lib.py +++ b/cvs/lib/utils_lib.py @@ -364,14 +364,12 @@ def resolve_cluster_config_placeholders(cluster_dict): # Get username from environment (fallback chain: USER -> LOGNAME -> USERNAME -> 'root') username = os.getenv('USER') or os.getenv('LOGNAME') or os.getenv('USERNAME') or 'root' - log.info(f'Resolving cluster path placeholders with system username: {username}') - # Define replacement mapping - only resolve {user-id} in cluster config replacements = { '{user-id}': username, } - resolved_cluster = _resolve_placeholders_in_dict(cluster_dict, replacements, context_name="cluster config") + resolved_cluster = _resolve_placeholders_in_dict(cluster_dict, replacements) return resolved_cluster diff --git a/cvs/parsers/schemas.py b/cvs/parsers/schemas.py index 83711f452..ccc0f0ea6 100644 --- a/cvs/parsers/schemas.py +++ b/cvs/parsers/schemas.py @@ -1139,6 +1139,146 @@ class PreflightConnectivityCheckConfig(BaseModel): ifoe: PreflightIfoeConfig = Field(default_factory=PreflightIfoeConfig, description="IFoE connectivity settings") +class PreflightNodeSmokeConfig(BaseModel): + """Primus node_smoke settings (primus-cli direct -- node_smoke).""" + + model_config = ConfigDict(extra="allow") + + connectivity_mode: str = Field( + default="skip", + description="Primus node_smoke mode: 'run' (host/GPU/RDMA roll-call) or 'skip' (default)", + ) + auto_setup: bool = Field( + default=True, + description="Clone/update Primus and prepare venv on each node before node_smoke", + ) + setup_timeout: int = Field(default=600, ge=60, description="SSH timeout in seconds for Primus auto_setup") + force_reclone: bool = Field( + default=False, + description="Remove primus_dir and clone fresh on every run (destructive)", + ) + shared_install: bool = Field( + default=True, + description=( + "When true (default), clone and venv setup run only on the first reachable node; " + "other nodes wait for the shared NFS home install. Set false only if each node has " + "a local primus_dir/venv_activate path." + ), + ) + pip_install_mode: str = Field( + default="minimal", + description="Venv deps: minimal (torch only), requirements, or skip", + ) + torch_pip_index_url: str = Field( + default="https://download.pytorch.org/whl/rocm6.2", + description="PyTorch ROCm wheel index URL for minimal pip_install_mode", + ) + primus_git_url: str = Field( + default="https://github.com/AMD-AIG-AIMA/Primus.git", + description="Primus repository URL for auto_setup clone", + ) + primus_git_branch: str = Field( + default="dev/preflight-direct-test", + description="Git branch to checkout during auto_setup", + ) + primus_git_recurse_submodules: bool = Field( + default=False, + description="Clone git submodules during auto_setup (not required for node_smoke)", + ) + primus_dir: str = Field( + default="/home/{user-id}/INSTALL/Primus", + description="Path to cloned Primus repo under the user's home directory (required when connectivity_mode is 'run')", + ) + venv_activate: str = Field( + default="/home/{user-id}/envs/preflight/.venv/bin/activate", + description="Path to Python venv activate script on each node (required when connectivity_mode is 'run')", + ) + gpus_per_node: int = Field(default=8, ge=1, description="GPUs per node for node_smoke") + master_port: int = Field(default=1234, ge=1024, le=65535, description="Distributed master port for node_smoke") + dump_path: str = Field( + default="", + description="Per-node dump directory for smoke JSON (default: /node_smoke)", + ) + expected_rdma_nics: Optional[int] = Field( + default=None, + ge=1, + description="Hard-fail when training RDMA NIC count differs (default: len(node_check.rdma_interfaces))", + ) + ulimit_l_min_gb: float = Field(default=32.0, ge=0, description="Minimum RLIMIT_MEMLOCK in GiB (0 disables)") + shm_min_gb: float = Field(default=8.0, ge=0, description="Minimum /dev/shm size in GiB (0 disables)") + skip_dmesg: bool = Field(default=False, description="Skip dmesg error scan (e.g. unprivileged containers)") + allow_foreign_procs: bool = Field( + default=False, + description="Do not FAIL nodes with foreign GPU processes (still reported)", + ) + allowed_procs: str = Field( + default="gpuagent,rocm-smi-daemon,amd-smi,dcgm-exporter", + description="Comma-separated process names allowed to hold GPUs", + ) + require_tools: str = Field( + default="", + description="Comma-separated CLI tools that must exist in PATH (empty = warn only)", + ) + nccl_socket_ifname: str = Field(default="", description="NCCL_SOCKET_IFNAME override for node_smoke") + gloo_socket_ifname: str = Field( + default="", description="GLOO_SOCKET_IFNAME override (defaults to nccl_socket_ifname)" + ) + nccl_ib_hca: str = Field(default="", description="NCCL_IB_HCA override (defaults to node_check.rdma_interfaces)") + nccl_ib_gid_index: Optional[int] = Field( + default=None, + description="NCCL_IB_GID_INDEX override (defaults to node_check.gid_index)", + ) + rdma_nic_allowlist: str = Field( + default="", + description="Training NIC allowlist for node_smoke (defaults to node_check.rdma_interfaces)", + ) + ssh_timeout: int = Field(default=300, ge=30, description="SSH timeout in seconds for each node_smoke run") + tier2_perf: bool = Field( + default=False, + description=( + "Enable Primus node_smoke Tier 2 perf sanity (--tier2-perf): " + "8192³ GEMM TFLOPS floor, HBM D2D bandwidth, local multi-GPU RCCL all-reduce" + ), + ) + gemm_tflops_min: float = Field( + default=600.0, + ge=0, + description="Tier 2 large GEMM TFLOPS floor (--gemm-tflops-min); used when tier2_perf is true", + ) + hbm_gbs_min: float = Field( + default=2000.0, + ge=0, + description="Tier 2 HBM device-to-device bandwidth floor in GB/s (--hbm-gbs-min)", + ) + rccl_gbs_min: float = Field( + default=100.0, + ge=0, + description="Tier 2 local multi-GPU RCCL all-reduce bandwidth floor in GB/s (--rccl-gbs-min)", + ) + rccl_size_mb: int = Field( + default=64, + ge=1, + description="Tier 2 local RCCL all-reduce message size in MB (--rccl-size-mb)", + ) + rccl_timeout_sec: int = Field( + default=120, + ge=30, + description="Tier 2 local RCCL all-reduce hard timeout in seconds (--rccl-timeout-sec)", + ) + extra_args: List[str] = Field( + default_factory=list, + description="Additional node_smoke CLI flags forwarded to primus-cli", + ) + + @field_validator("connectivity_mode") + @classmethod + def validate_node_smoke_mode(cls, v: str) -> str: + valid_modes = ["run", "skip"] + if v not in valid_modes: + raise ValueError(f"node_smoke.connectivity_mode must be one of: {', '.join(valid_modes)}") + return v + + class PreflightReportingConfig(BaseModel): """Report generation and output settings.""" @@ -1179,6 +1319,9 @@ class PreflightConfigFile(BaseModel): connectivity_check: PreflightConnectivityCheckConfig = Field( default_factory=PreflightConnectivityCheckConfig, description="Inter-node connectivity tests" ) + node_smoke: PreflightNodeSmokeConfig = Field( + default_factory=PreflightNodeSmokeConfig, description="Primus node_smoke checks" + ) reporting: PreflightReportingConfig = Field( default_factory=PreflightReportingConfig, description="Report generation and output settings" ) diff --git a/cvs/tests/inference/atom/README.md b/cvs/tests/inference/atom/README.md new file mode 100644 index 000000000..326a86f2c --- /dev/null +++ b/cvs/tests/inference/atom/README.md @@ -0,0 +1,200 @@ +# ATOM Inference Suite (single-node and multinode) + +Cluster validation suite that runs **ATOM** serving benchmarks on AMD Instinct +GPUs and gates each sweep cell on tiered performance and health metrics with a +PASS/FAIL HTML report. + +## Overview + +The suite drives a serving + benchmark-serving job inside a container on one or +more cluster nodes, then parses the benchmark artifact to produce `client.*` +metrics and verdicts. It provides: + +1. **One unified suite** — `atom` handles single-node and multinode PP from the + same entry point; topology and driver behaviour come from the variant config. +2. **Execution drivers** — `params.driver` is `atom` on single-node variants + (native `openai_server`) or `vllm_atom` on shipped multinode PP stems. +3. **Parameter sweeps** — one benchmark run per sweep cell (ISL/OSL shape × + concurrency), each with its own result rows in the report. +4. **Tiered metric gating** — one pytest row per **metric tier** per cell + (throughput, TTFT, TPOT, health, scaling, record) against threshold specs. +5. **Server reuse** — optional reuse of a warm server across sweep cells when + `reuse_server_across_sweep: true` and the session key matches. +6. **Multinode fabric discovery** — `test_discover_topology` resolves IB HCAs + and socket netdev before the sweep when `params.nnodes > 1`. +7. **HTML report + Run Deck** — pytest HTML rows, console results tables, and + (when `--html` is set) an ATOM Run Deck bundle for interactive charts. + +Single-node vs multinode is determined by `params.nnodes` and the cluster file +host count, not by a separate suite name. + +## Quick Start + +Single-node W1 run (MI300X, `driver=atom`): + +```bash +cvs run atom \ + --cluster_file ~/input/cluster_file/atom_cluster.json \ + --config_file ~/input/config_file/inference/atom/single/mi300x_atom_deepseek-r1_fp8_single.json \ + --html ~/cvs_results/atom-w1-single.html --self-contained-html -vvv +``` + +Multinode PP run (2-node, `driver=vllm_atom`): + +```bash +cvs run atom \ + --cluster_file ~/input/cluster_file/atom_cluster.json \ + --config_file ~/input/config_file/inference/atom/distributed/mi300x_atom_deepseek-r1_fp8_distributed.json \ + --html ~/cvs_results/atom-w1-distributed.html --self-contained-html -vvv +``` + +- `--cluster_file` — JSON describing the node(s); `len(node_dict)` must match + `params.nnodes` in the config. +- `--config_file` — a variant JSON under + `cvs/input/config_file/inference/atom/` (see that folder's README for the + variable-by-variable reference, copy-config flow, and lab prerequisites). +- `--html` / `--self-contained-html` — write the pytest report; a sibling + bundle directory holds per-test logs and, when the report engine is enabled, + the ATOM Run Deck artifacts. + +> Use a **single-host** cluster file with `nnodes=1` variants and a **two-host** +> cluster file with multinode PP variants (`driver=vllm_atom`). The config's +> `params.driver` and `params.nnodes` must match the intended topology. + +For smoke runs, filter with `-k`, for example `-k "w1_1k_1k-conc128"`. + +## Suite layout + +| Item | File | Role | +|---|---|---| +| Suite (`cvs run atom`) | `atom.py` | Lifecycle tests, sweep inference, tiered metric gates | +| Fixtures / ordering | `conftest.py` | Cluster + variant load, orchestrator, lifecycle rank | +| Results table | `_shared.py` | Console + HTML results table (`test_print_results_table`) | + +`conftest.py` and `_shared.py` are helpers, not runnable suites. + +## Test lifecycle (report rows) + +Tests run in this pinned order. `[cell]` = one row per sweep cell; +`[cell-tier]` = one row per metric tier per cell. + +| Order | Test | Runs on | Purpose | +|---|---|---|---| +| 1 | `test_launch_container` | once | Launch and verify the container | +| 2 | `test_setup_sshd` | multinode | SSH daemon setup across nodes | +| 3 | `test_discover_topology` | once | Resolve IB HCAs + socket netdev (skipped work on single-node) | +| 4 | `test_model_fetch` | once | Verify / fetch the model cache | +| 5 | `test_atom_inference[cell]` | per cell | Build server env, start server, run bench client, parse results | +| 6 | `test_cell_metrics[cell-tier]` | per cell × tier | Threshold PASS/FAIL for that tier's metrics | +| 7 | `test_print_results_table` | once | Console tables + consolidated results | +| 8 | `test_teardown` | once | Tear the container down | + +On inference failure, `lifecycle.failed` is set so downstream cells and metric +rows are skipped. Server reuse skips restart when the session key +(`server_session_key`) matches the prior cell. + +## Sweeps + +A **sweep cell** is one `(sequence shape, concurrency)` pair declared under +`sweep.sequence_combinations` and `sweep.runs`. Parametrize IDs look like +`w1_1k_1k-conc128` or, when metric tiers are collected, +`w1_1k_1k-conc128-throughput`. + +Each cell's **threshold key** is built by `cell_key()`, for example: + +- Single-node: `ISL=1024,OSL=1024,TP=8,CONC=128` +- Multinode PP: `ISL=1024,OSL=1024,TP=8,PP=2,NNODES=2,CONC=128` + +That key must exist in the sibling threshold file referenced by +`threshold_json`. + +## Metrics and PASS/FAIL + +Each `test_cell_metrics[cell-tier]` evaluates metrics for one tier against the +cell's threshold specs and reports one of: + +| Status | Meaning | +|---|---| +| PASS | value satisfies the threshold | +| FAIL | value violates the threshold (row is red) | +| skip | prior stage failed, cell did not run, or tier not applicable (e.g. scaling on single-node) | +| RECORD | `enforce_thresholds: false` or `record` tier — value logged, not gated | + +**Metric tiers** (namespace `client.*` unless noted): + +| Tier | Example metrics | +|---|---| +| `throughput` | `total_token_throughput`, `output_throughput`, `per_gpu_throughput`, `output_tput_per_gpu` | +| `ttft` | `mean_ttft_ms`, `p99_ttft_ms` | +| `tpot` | `mean_tpot_ms`, `p99_tpot_ms` | +| `health` | `success_rate`, `failed` | +| `scaling` | `scaling.efficiency_pct` (multinode) | +| `record` | remaining client metrics not in a gate tier | + +Gating requires `enforce_thresholds: true` in the config. ATOM may omit some +tail percentiles even when `metric_percentiles` requests them; the suite only +gates metrics present in the benchmark artifact. See +`cvs/input/config_file/inference/atom/README.md` for threshold kinds and +variant-specific enforcement policy. + +## Reports and logs + +- **Results table** — one row per test; metric tier rows reflect threshold + verdicts. +- **Full Log** — each test row links to its captured log when HTML reporting is + enabled. +- **Console summary** — `test_print_results_table` prints per-cell tables with + throughput, latency, and health columns. +- **Run Deck** — when `--html` is set, the inference report engine emits + `atom_run_deck.html`, `.json`, and `_viewer.html` at session end (bundled + into the pytest zip). See `cvs/lib/report/README.md`. + +Server and client logs for each cell are written under the variant +`paths.log_dir` on the cluster nodes. + +## Config and threshold files + +Variant configs and thresholds live in `cvs/input/config_file/inference/atom/` +as flat sibling pairs: + +```text +{gpu}_atom_{model}_{precision}[_{mode}].json +{gpu}_atom_{model}_{precision}[_{mode}]_threshold.json +``` + +On a lab machine, copy each variant into its **own subdirectory** so threshold +discovery is unambiguous (see the input-config README). + +| Example config | Mode | Driver | +|---|---|---| +| `mi300x_atom_deepseek-r1_fp8_single` | single-node W1 | `atom` | +| `mi300x_atom_deepseek-r1_fp8_baseline_sweep` | DTNI baseline matrix | `atom` | +| `mi300x_atom_deepseek-r1_fp8_distributed` | 2-node PP W1 | `vllm_atom` | +| `mi300x_atom_deepseek-r1_fp8_mtp3` | single-node MTP3 | `atom` | + +See `cvs/input/config_file/inference/atom/README.md` for the full variant +catalog, copy-config commands, cluster-file editing, and step-by-step lab run +recipes. + +## Prerequisites + +- Passwordless SSH from the control host to each cluster node (key in the + cluster file), and Docker available on the GPU nodes. +- A container image with ATOM; shipped multinode configs use `` until + pinned for your lab. +- A Hugging Face token file at `paths.hf_token_file` when fetching models. +- Model cache at `paths.models_dir` on GPU nodes when `model.remote: 0`. +- For multinode runs: a shared or reachable log path, matching host count in + the cluster file, `params.master_addr` set to the head VPC IP, and IB/socket + interfaces discoverable (or explicit `roles.server.ib_hca_devices` / + `roles.server.ib_netdev` overrides). + +## Related code + +| Module | Purpose | +|---|---| +| `cvs/lib/inference/atom/atom_orch.py` | `AtomJob` — server/client lifecycle, result parsing | +| `cvs/lib/inference/atom/atom_config_loader.py` | Typed variant load, sweep expansion, session keys | +| `cvs/lib/inference/atom/atom_parsing.py` | Metric tiers, `client.*` mapping, scaling efficiency | +| `cvs/lib/inference/utils/inference_suite_lifecycle.py` | Shared lifecycle stages (`test_launch_container`, …) | +| `cvs/lib/utils/ib_discovery.py` | Multinode IB HCA and socket netdev discovery | diff --git a/cvs/tests/inference/atom/_shared.py b/cvs/tests/inference/atom/_shared.py new file mode 100644 index 000000000..38d504c97 --- /dev/null +++ b/cvs/tests/inference/atom/_shared.py @@ -0,0 +1,13 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +from cvs.lib.inference.utils.inference_suite_results_table import ( + ATOM_RESULTS_COLUMNS, + make_print_results_table, +) + +test_print_results_table = make_print_results_table(ATOM_RESULTS_COLUMNS) + +__all__ = ["test_print_results_table"] diff --git a/cvs/tests/inference/atom/atom.py b/cvs/tests/inference/atom/atom.py new file mode 100644 index 000000000..6707c9c5a --- /dev/null +++ b/cvs/tests/inference/atom/atom.py @@ -0,0 +1,205 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +import json +import os +import time + +import pytest + +from cvs.lib import globals +from cvs.lib.inference.utils.inference_suite_lifecycle import ( + sweep_cell_result_key, + test_launch_container, # noqa: F401 + test_model_fetch, # noqa: F401 + test_setup_sshd, # noqa: F401 + test_teardown, # noqa: F401 +) +from cvs.lib.inference.atom.atom_orch import AtomJob +from cvs.lib.inference.atom.atom_config_loader import ( + expand_sweep_parametrize, + reuse_server_flag, + server_session_key, +) +from cvs.lib.inference.atom.atom_parsing import ( + CLIENT_METRIC_UNITS as _METRIC_UNITS, + METRIC_TIERS, + RECORD_METRICS, + SCALING_METRIC_UNITS, + tier_metric_specs, +) +from cvs.lib.utils.verdict import evaluate_all +from cvs.tests.inference.atom._shared import test_print_results_table # noqa: F401 + +log = globals.log + + +def _tier_display_metric(tier): + if tier == "record": + return RECORD_METRICS[0] if RECORD_METRICS else "output_throughput" + names = METRIC_TIERS.get(tier, ()) + return names[0] if names else tier + + +def test_discover_topology(orch, variant_config, lifecycle, request): + """Discover IB HCAs and socket netdev on all nodes before the benchmark sweep.""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + nn = int(variant_config.params.nnodes) + if nn == 1: + lifecycle.ib_hcas = [] + lifecycle.ib_netdev = "" + return + + from cvs.lib.utils.ib_discovery import resolve_multinode_fabric + + t = time.monotonic() + master_addr = (variant_config.params.master_addr or "").strip() or orch.hosts[0] + try: + resolved_hcas, resolved_netdev = resolve_multinode_fabric( + orch, + ib_hca_devices=variant_config.roles.server.ib_hca_devices, + ib_netdev=variant_config.roles.server.ib_netdev, + master_addr=master_addr, + ) + except RuntimeError as e: + lifecycle.failed = True + lifecycle.record(request.node.nodeid, "topology_discovery", time.monotonic() - t) + pytest.fail(str(e)) + + lifecycle.ib_hcas = resolved_hcas + lifecycle.ib_netdev = resolved_netdev + lifecycle.record(request.node.nodeid, "topology_discovery", time.monotonic() - t) + log.info( + "test_discover_topology: resolved netdev=%s HCAs=%s", + resolved_netdev, + resolved_hcas, + ) + + +def pytest_generate_tests(metafunc): + config_file = metafunc.config.getoption("config_file") + if not config_file or not os.path.isfile(config_file): + raise pytest.UsageError(f"--config_file not found or not specified: {config_file!r}") + with open(config_file) as fp: + raw = json.load(fp) + spec = expand_sweep_parametrize(raw.get("sweep", {}), metafunc.fixturenames) + if spec: + argnames, argvalues, ids = spec + metafunc.parametrize(argnames, argvalues, ids=ids) + + +def test_atom_inference( + orch, + variant_config, + hf_token, + seq_combo, + concurrency, + inf_res_dict, + server_session, + lifecycle, + request, +): + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + isl = seq_combo["isl"] + osl = seq_combo["osl"] + p = variant_config.params + job = AtomJob.from_variant( + orch=orch, + variant=variant_config, + hf_token=hf_token, + isl=isl, + osl=osl, + concurrency=concurrency, + ib_hcas=getattr(lifecycle, "ib_hcas", []), + ib_netdev=getattr(lifecycle, "ib_netdev", None), + ) + + session_key = server_session_key(variant_config, isl, osl) + reuse = reuse_server_flag(p) and server_session.get("key") == session_key + + try: + if not reuse: + job.stop_server() + job.build_server_cmd() + t = time.monotonic() + job.start_server() + job.wait_ready() + lifecycle.record(request.node.nodeid, "server_ready", time.monotonic() - t) + if reuse_server_flag(p): + server_session["key"] = session_key + else: + log.info("reusing ATOM server across sweep cell (key=%s)", session_key) + job.prepare_cell_out_dir() + t_client = time.monotonic() + job.run_client() + job.wait_client_complete() + results = job.parse_results() + except Exception: + lifecycle.failed = True + raise + + inf_res_dict[sweep_cell_result_key(variant_config, seq_combo, isl, osl, concurrency)] = results + lifecycle.record(request.node.nodeid, "client_complete", time.monotonic() - t_client) + + +def test_cell_metrics( + seq_combo, + concurrency, + metric_tier, + inf_res_dict, + variant_config, + lifecycle, + request, +): + """One pytest row per metric tier per sweep cell (W1 gate batches). + + Fails when ``enforce_thresholds`` is on and either (1) the cell has no + threshold specs for the requested tier, or (2) specs exist but every gated + metric for that tier is missing from the benchmark artifact (ATOM may omit + tail percentiles even when ``metric_percentiles`` requests them). + """ + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + isl = seq_combo["isl"] + osl = seq_combo["osl"] + key = sweep_cell_result_key(variant_config, seq_combo, isl, osl, concurrency) + if key not in inf_res_dict: + pytest.skip(f"no recorded results for cell {key!r} (inference did not run)") + host_dict = inf_res_dict[key] + _host, actuals = next(iter(host_dict.items())) + cell = variant_config.cell_key(isl, osl, concurrency) + thresholds_cell = variant_config.thresholds.get(cell) or {} + specs = tier_metric_specs(thresholds_cell, metric_tier) + + display = _tier_display_metric(metric_tier) + if metric_tier == "scaling": + full = f"scaling.{display}" + unit = SCALING_METRIC_UNITS.get(display, "%") + else: + full = f"client.{display}" + unit = _METRIC_UNITS.get(display, metric_tier) + value = actuals.get(full) + request.node.user_properties.append(("metric_value", value)) + request.node.user_properties.append(("metric_unit", unit)) + + if not variant_config.enforce_thresholds or metric_tier == "record": + return + if not specs: + if metric_tier == "scaling" and int(variant_config.params.nnodes) <= 1: + pytest.skip("scaling tier not configured for single-node runs") + pytest.fail(f"no threshold specs for tier {metric_tier!r} in cell {cell!r}") + # ATOM benchmark_serving may omit some tail percentiles even when + # metric_percentiles requests them; only gate metrics present in actuals. + specs = {k: v for k, v in specs.items() if k in actuals and actuals[k] is not None} + if not specs: + pytest.fail( + f"no assertable threshold specs for tier {metric_tier!r} in cell {cell!r} " + f"(metrics missing from benchmark artifact)" + ) + evaluate_all(actuals, specs) diff --git a/cvs/tests/inference/atom/conftest.py b/cvs/tests/inference/atom/conftest.py new file mode 100644 index 000000000..68d17b418 --- /dev/null +++ b/cvs/tests/inference/atom/conftest.py @@ -0,0 +1,149 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +import json +import os + +import pytest + +from cvs.core.orchestrators.factory import OrchestratorConfig, OrchestratorFactory +from cvs.lib import globals +from cvs.lib.inference.utils.inference_suite_lifecycle import ( + InferenceLifecycle, + # html_metric_table_header, + # html_metric_table_row, + sort_lifecycle_items, +) +from cvs.lib.inference.atom.atom_config_loader import ( + load_variant, + orchestrator_container_from_variant, +) +from cvs.lib.utils_lib import resolve_cluster_config_placeholders + +log = globals.log + + +def _log_variant_run_card(variant_config): + rc = variant_config.run_card + parts = [ + f"gpu_arch={variant_config.gpu_arch}", + f"driver={variant_config.params.driver}", + f"model={variant_config.model.id}", + ] + atom_args = variant_config.roles.server.atom_args + if atom_args: + parts.append(f"atom_args={len(atom_args)} tokens") + if rc.atom_image_pin: + parts.append(f"image_pin={rc.atom_image_pin}") + if rc.upstream_run_url: + parts.append(f"upstream_run={rc.upstream_run_url}") + if rc.notes: + parts.append(f"notes={rc.notes}") + if int(variant_config.params.nnodes) > 1: + parts.append(f"nnodes={variant_config.params.nnodes}") + parts.append(f"pp={variant_config.params.pipeline_parallel_size}") + log.info("ATOM run card: %s", "; ".join(parts)) + + +@pytest.fixture(scope="module", autouse=True) +def _emit_variant_run_card(variant_config): + """Log the variant run card once per module (not once per sweep cell).""" + _log_variant_run_card(variant_config) + + +LIFECYCLE_RANK = { + "test_launch_container": 0, + "test_setup_sshd": 1, + "test_discover_topology": 2, + "test_model_fetch": 3, + "test_atom_inference": 4, + "test_cell_metrics": 5, + "test_print_results_table": 6, + "test_teardown": 7, +} + + +def _deep_merge(base, override): + """Recursively merge ``override`` onto ``base`` (dicts merged key-wise; scalars/lists replaced).""" + if not (isinstance(base, dict) and isinstance(override, dict)): + return override + out = dict(base) + for k, v in override.items(): + out[k] = _deep_merge(base[k], v) if k in base else v + return out + + +@pytest.fixture(scope="module") +def cluster_dict(pytestconfig): + cluster_file = pytestconfig.getoption("cluster_file") + if not cluster_file: + pytest.fail("--cluster_file is required") + with open(cluster_file) as fp: + d = json.load(fp) + return resolve_cluster_config_placeholders(d) + + +@pytest.fixture(scope="module") +def variant_config(pytestconfig, cluster_dict): + config_file = pytestconfig.getoption("config_file") + if not config_file: + pytest.fail("--config_file is required") + return load_variant(config_file, cluster_dict) + + +@pytest.fixture(scope="module") +def lifecycle(): + return InferenceLifecycle() + + +@pytest.fixture(scope="module") +def orch(cluster_dict, variant_config, lifecycle): + container_block = _deep_merge( + cluster_dict.get("container", {}), + # also injects roles.server.env — do not replace with .container.model_dump() + orchestrator_container_from_variant(variant_config), + ) + testsuite_config = { + "orchestrator": "container", + "container": container_block, + } + cfg = OrchestratorConfig.from_configs(cluster_dict, testsuite_config) + o = OrchestratorFactory.create_orchestrator(log, cfg) + yield o + if not lifecycle.torn_down: + log.info("orch fixture leak-guard: tearing down ATOM containers") + o.teardown_containers() + + +@pytest.fixture(scope="module") +def hf_token(variant_config): + path = variant_config.paths.hf_token_file + if not os.path.isfile(path): + pytest.skip(f"hf_token file missing: {path}") + with open(path) as fp: + return fp.read().strip() + + +@pytest.fixture(scope="module") +def server_session(): + """Tracks the active server session key to allow reuse across sweep cells when reuse_server_across_sweep=true.""" + return {"key": None} + + +@pytest.fixture(scope="module") +def inf_res_dict(): + return {} + + +def pytest_collection_modifyitems(items): + sort_lifecycle_items(items, LIFECYCLE_RANK) + + +# def pytest_html_results_table_header(cells): +# html_metric_table_header(cells) +# +# +# def pytest_html_results_table_row(report, cells): +# html_metric_table_row(report, cells) diff --git a/cvs/tests/inference/inferencemax/inferencemax_gpt_oss_120b_single.py b/cvs/tests/inference/inferencemax/inferencemax_gpt_oss_120b_single.py deleted file mode 100644 index a922b1a48..000000000 --- a/cvs/tests/inference/inferencemax/inferencemax_gpt_oss_120b_single.py +++ /dev/null @@ -1,294 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import pytest - -import re -import time -import json - -from cvs.lib.parallel_ssh_lib import * -from cvs.lib.utils_lib import * -from cvs.lib import docker_lib -from cvs.lib.inference.inference_max import InferenceMaxJob -from cvs.lib import globals - -log = globals.log - - -# Importing additional cmd line args to script .. -@pytest.fixture(scope="module") -def cluster_file(pytestconfig): - """ - Retrieve the --cluster_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the cluster JSON file specified via --cluster_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --cluster_file=/path/to/cluster.json - - Use module scope so the value is resolved once per test module. - """ - return pytestconfig.getoption("cluster_file") - - -@pytest.fixture(scope="module") -def training_config_file(pytestconfig): - """ - Retrieve the --config_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the training config JSON file specified via --config_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --config_file=/path/to/training_config.json - - Module scope avoids re-fetching the option across tests in this module. - """ - return pytestconfig.getoption("config_file") - - -# Importing the cluster and cofig files to script to access node, switch, test config params -@pytest.fixture(scope="module") -def cluster_dict(cluster_file): - """ - Load the entire cluster configuration from the provided JSON file. - - Args: - cluster_file (str): Path to the cluster JSON file. - - Returns: - dict: Parsed JSON representing the cluster (nodes, credentials, etc.). - - Notes: - - Logs the loaded structure for visibility; consider using log.debug if verbose. - """ - with open(cluster_file) as json_file: - cluster_dict = json.load(json_file) - - # Resolve path placeholders like {user-id} in cluster config - cluster_dict = resolve_cluster_config_placeholders(cluster_dict) - log.info("%s", cluster_dict) - return cluster_dict - - -@pytest.fixture(scope="module") -def inference_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - inference_dict = inference_dict_t['config'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - inference_dict = resolve_test_config_placeholders(inference_dict, cluster_dict) - return inference_dict - - -@pytest.fixture(scope="module") -def benchmark_params_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - benchmark_params_dict = inference_dict_t['benchmark_params'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - benchmark_params_dict = resolve_test_config_placeholders(benchmark_params_dict, cluster_dict) - - log.info("%s", benchmark_params_dict) - return benchmark_params_dict - - -@pytest.fixture(scope="module") -def hf_token(inference_dict): - """ - Load the Hugging Face access token from the file path specified in the training config. - - Args: - inference_dict (dict): Training configuration dict that includes: - - 'hf_token_file': Path to the file containing the HF token. - - Returns: - str: The HF token string read from the file. - - Behavior: - - Reads the token from inference_dict['hf_token_file'] (already resolved for placeholders). - - Strips the trailing newline from the token. - """ - hf_token_file = inference_dict['hf_token_file'] - try: - with open(hf_token_file, 'r') as fp: - hf_token = fp.read().rstrip("\n") - except FileNotFoundError: - log.error(f"Error: The file '{hf_token_file}' was not found.") - except Exception as e: - log.error(f"An error occurred: {e}") - return hf_token - - -@pytest.fixture(scope="module") -def s_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes. - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - s_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return s_phdl - - -@pytest.fixture(scope="module") -def c_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes. - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - c_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return c_phdl - - -@pytest.fixture(scope="module") -def gpu_type(s_phdl, cluster_dict): - """ - Provide the GPU type string for the test module. - - Args: - cluster_dict (dict): Cluster configuration that includes the GPU type. - - Returns: - str: The GPU type (e.g., 'mi300', 'mi300x') used to select model parameters and logic. - - Notes: - - Module scope ensures this is evaluated once per test module. - - Consider validating this value against an expected set of GPU types to catch typos early. - """ - - log.info("%s", s_phdl) - log.info("%s", list(dir(s_phdl))) - head_node = s_phdl.host_list[0] - smi_out_dict = s_phdl.exec('rocm-smi -a | head -30') - smi_out = smi_out_dict[head_node] - gpu_type = get_model_from_rocm_smi_output(smi_out) - return gpu_type - - -def test_cleanup_stale_containers(s_phdl, inference_dict): - """ - Pytest: Clean up potentially stale Docker containers and volumes before tests. - - Args: - s_phdl: Parallel SSH/process handle used by docker_lib to run commands on nodes. - inference_dict (dict): Training configuration dict that includes: - - 'container_name': Name of the container to be killed if running. - - Behavior: - - Kills the specific container identified by inference_dict['container_name']. - - Deletes all containers and volumes on the target nodes (broad cleanup). - - Notes: - - This performs a broad cleanup via delete_all_containers_and_volumes; ensure the - test environment is isolated so this doesn?t remove unrelated containers/volumes. - - Consider narrowing cleanup scope if other workloads may be present on the hosts. - """ - - container_name = inference_dict['container_name'] - docker_lib.kill_docker_container(s_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(s_phdl) - - -def test_launch_inference_containers(s_phdl, inference_dict, benchmark_params_dict): - """ - Launch InferenceMAX inference containers on all nodes. - - Note: Container image can be model-specific or use global default. - """ - - log.info('Testcase launch InferenceMax containers') - globals.error_list = [] - container_name = inference_dict['container_name'] - - # Get model-specific container image or use global default - container_image = benchmark_params_dict.get('gpt-oss-120b', {}).get( - 'container_image', inference_dict['container_image'] - ) - - # Launch the containers .. - docker_lib.launch_docker_container( - s_phdl, - container_name, - container_image, - inference_dict['container_config']['device_list'], - inference_dict['container_config']['volume_dict'], - inference_dict['container_config']['env_dict'], - shm_size='48G', - timeout=60 * 20, - ) - # ADD verifications .. - time.sleep(30) - log.info('Verify if the containers have been launched properly') - out_dict = s_phdl.exec('docker ps') - for node in out_dict.keys(): - if not re.search(f'{container_name}', out_dict[node], re.I): - fail_test(f'Failed to launch container on node {node}') - update_test_result() - - -def test_gpt_oss_120_single_node(c_phdl, s_phdl, gpu_type, inference_dict, benchmark_params_dict, hf_token): - globals.error_list = [] - im_obj = InferenceMaxJob( - c_phdl=c_phdl, - s_phdl=s_phdl, - model_name='gpt-oss-120b', - inference_config_dict=inference_dict, - benchmark_params_dict=benchmark_params_dict, - hf_token=hf_token, - gpu_type=gpu_type, - distributed_inference=False, - ) - im_obj.build_server_inference_job_cmd() - im_obj.start_inference_server_job() - im_obj.start_inference_client_job() - im_obj.poll_for_inference_completion() - im_obj.verify_inference_results() - update_test_result() diff --git a/cvs/tests/inference/sglang/_shared.py b/cvs/tests/inference/sglang/_shared.py new file mode 100644 index 000000000..02d0ed947 --- /dev/null +++ b/cvs/tests/inference/sglang/_shared.py @@ -0,0 +1,348 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Shared helpers and ordering for the SGLang inference suites (single-node and disaggregated). + +Used by ``sglang_single.py``, ``sglang_distributed.py``, and ``sglang_disagg_distributed.py``. +Each suite has its own stage order (single-node, unified multi-node, or PD disagg). +''' + +from __future__ import annotations + +import os +import re +from tabulate import tabulate +from typing import Any, Mapping + +from cvs.lib import globals + +log = globals.log + +__all__ = [ + "resolve_benchmark_variant_key", + "SGLANG_TEST_ORDER", + "SGLANG_SINGLE_TEST_ORDER", + "SGLANG_DISTRIBUTED_TEST_ORDER", + "test_print_results_table", +] + +_SMOKE_LINE_RE = re.compile(r"^(.+) -> (Pass|Fail) \((\d+)\)$") + + +def resolve_benchmark_variant_key(root: Mapping[str, Any], config_path: str) -> str: + """Pick which ``benchmark_params`` entry to run. + + Resolution order: + 1. Environment ``SGLANG_BENCHMARK_KEY`` (override for CI matrices). + 2. Top-level JSON ``active_benchmark`` (string key into ``benchmark_params``). + 3. If ``benchmark_params`` has exactly one key, use it. + + ``root`` is the full JSON object loaded from ``--config_file`` (not only ``config``). + """ + env_key = (os.environ.get("SGLANG_BENCHMARK_KEY") or "").strip() + bp = root.get("benchmark_params") or {} + if not isinstance(bp, dict) or not bp: + raise ValueError(f"benchmark_params missing or empty in {config_path!r}") + + if env_key: + if env_key not in bp: + raise ValueError( + f"SGLANG_BENCHMARK_KEY={env_key!r} not found in benchmark_params ({config_path}); valid: {sorted(bp)!r}" + ) + log.info("Using benchmark variant from env SGLANG_BENCHMARK_KEY=%r", env_key) + return env_key + + explicit = root.get("active_benchmark") + if explicit is not None: + if explicit not in bp: + raise ValueError( + f"active_benchmark={explicit!r} not found in benchmark_params ({config_path}); valid: {sorted(bp)!r}" + ) + log.info("Using benchmark variant from active_benchmark=%r", explicit) + return str(explicit) + + if len(bp) == 1: + only = next(iter(bp)) + log.info("Single benchmark_params entry; using %r", only) + return str(only) + + raise ValueError( + f"Multiple benchmark_params keys in {config_path!r}: {sorted(bp)!r}. " + "Set top-level \"active_benchmark\" to one of them, or export SGLANG_BENCHMARK_KEY." + ) + + +# Stable test order for sglang_disagg_distributed (PD prefill/decode/router). +SGLANG_TEST_ORDER = { + "test_launch_container": 0, + "test_rms_norm": 1, + "test_launch_prefill_servers": 2, + "test_launch_decode_servers": 3, + "test_poll_for_server_ready": 4, + "test_launch_proxy_router": 5, + "test_openai_compatible_http_endpoints": 6, + "test_run_lm_eval_hellaswag_benchmark_test": 7, + "test_run_lm_eval_gsm8k_benchmark_test": 8, + "test_run_performance_benchmark_test": 9, + "test_disagg_gpu_topology": 10, + "test_print_results_table": 11, + "test_teardown": 12, +} + +# Stable test order for sglang_single (one unified server, no PD). +SGLANG_SINGLE_TEST_ORDER = { + "test_launch_container": 0, + "test_rms_norm": 1, + "test_launch_server": 2, + "test_poll_for_server_ready": 3, + "test_openai_compatible_http_endpoints": 4, + "test_run_lm_eval_hellaswag_benchmark_test": 5, + "test_run_lm_eval_gsm8k_benchmark_test": 6, + "test_run_performance_benchmark_test": 7, + "test_print_results_table": 8, + "test_teardown": 9, +} + +# Stable test order for sglang_distributed (unified multi-node server, no PD). +SGLANG_DISTRIBUTED_TEST_ORDER = { + "test_launch_container": 0, + "test_rms_norm": 1, + "test_launch_server": 2, + "test_poll_for_server_ready": 3, + "test_openai_compatible_http_endpoints": 4, + "test_run_lm_eval_hellaswag_benchmark_test": 5, + "test_run_lm_eval_gsm8k_benchmark_test": 6, + "test_run_performance_benchmark_test": 7, + "test_distributed_gpu_topology": 8, + "test_print_results_table": 9, + "test_teardown": 10, +} + + +def _flat_threshold_specs(specs: dict) -> dict[str, float]: + """Threshold cell specs → {metric: numeric_gate}.""" + out: dict[str, float] = {} + for metric, spec in (specs or {}).items(): + if isinstance(spec, dict) and "value" in spec: + out[metric] = float(spec["value"]) + elif spec is not None: + out[metric] = float(spec) + return out + + +def _perf_result(actual, expected, metric_key: str) -> str: + if actual is None or expected is None: + return "-" + a, e = float(actual), float(expected) + if "ms" in metric_key.lower(): + return "PASS" if a <= e else "FAIL" + return "PASS" if a >= e else "FAIL" + + +def _thresholds_for_cell(variant_config, isl, osl, conc) -> dict[str, float]: + if variant_config is None: + return {} + tp = (getattr(variant_config, "benchmark_params", None) or {}).get("tensor_parallelism", "-") + pp = (getattr(variant_config, "benchmark_params", None) or {}).get("pipeline_parallelism", "-") + cell_id = f"ISL={isl},OSL={osl},TP={tp},PP={pp},CONC={conc}" + raw = (getattr(variant_config, "thresholds", None) or {}).get(cell_id) or {} + return _flat_threshold_specs(raw) + + +def test_print_results_table(inf_res_dict, lifecycle, variant_config=None): + """Log smoke, lm-eval accuracy, and perf tables (one row per metric per host per ISL/OSL cell).""" + phase_labels = getattr(lifecycle, "phase_labels", None) or {} + smoke_results = getattr(lifecycle, "smoke_results", None) + + if variant_config is None: + try: + from cvs.lib.report.registry import get_session_results + + variant_config = get_session_results().get("variant_config") + except Exception: + variant_config = None + + if smoke_results: + smoke_rows = [] + for line in smoke_results: + m = _SMOKE_LINE_RE.match(str(line).strip()) + if m: + smoke_rows.append([m.group(1), m.group(2).upper(), m.group(3)]) + else: + smoke_rows.append([str(line), "-", "-"]) + if smoke_rows: + log.info( + "\n\n\n\n======== OpenAI-compatible smoke results ========\n%s", + tabulate( + smoke_rows, + headers=["Check", "Result", "HTTP status"], + tablefmt="github", + ), + ) + + acc_rows = [] + for label, key in ( + ("HellaSwag", "accuracy_hellaswag"), + ("GSM8K", "accuracy_gsm8k"), + ): + e = phase_labels.get(key) + if isinstance(e, dict) and "task" in e: + passed = e.get("passed") + acc_rows.append( + [ + label, + e.get("task", "-"), + e.get("metric_key", "-"), + f"{float(e['actual']):.4f}" if e.get("actual") is not None else "-", + f"{float(e['expected']):.4f}", + "PASS" if passed is True else "FAIL" if passed is False else "-", + ] + ) + if acc_rows: + log.info( + "\n\n\n\n======== LM-eval accuracy results ========\n%s", + tabulate( + acc_rows, + headers=["Suite", "Task", "Metric", "Actual", "Expected", "Result"], + tablefmt="github", + ), + ) + + bp = (getattr(variant_config, "benchmark_params", None) or {}) if variant_config else {} + tp = bp.get("tensor_parallelism", "8") + pp = bp.get("pipeline_parallelism", "1") + + _ACC_CELL_RE = re.compile(r"^ACC_ISL=(?P\d+),OSL=(?P\d+)$") + + acc_by_cell = phase_labels.get("accuracy_by_cell") or {} + if acc_by_cell: + long_ctx_rows = [] + for cell_id, result in sorted(acc_by_cell.items()): + m = _ACC_CELL_RE.match(str(cell_id)) + if not m: + continue + key = f"accuracy_long_ctx_{m.group('isl')}" + e = phase_labels.get(key) or {} + long_ctx_rows.append( + [ + f"ISL={m.group('isl')}", + m.group("osl"), + f"{float(e['actual']):.4f}" if e.get("actual") is not None else "-", + f"{float(e['expected']):.4f}" if e.get("expected") is not None else "-", + result, + ] + ) + if long_ctx_rows: + log.info( + "\n\n\n\n======== Long-context accuracy (NIAH) ========\n%s", + tabulate( + long_ctx_rows, + headers=["Cell", "OSL", "Pass rate", "Expected", "Result"], + tablefmt="github", + ), + ) + + _CELL_RE = re.compile(r"^ISL=(?P\d+),OSL=(?P\d+),TP=(?P\d+),PP=(?P\d+),CONC=(?P\d+)$") + bp = (getattr(variant_config, "benchmark_params", None) or {}) if variant_config else {} + performance_by_cell = phase_labels.get("performance_by_cell") or {} + if performance_by_cell: + summary_rows = [] + for cell_id, result in sorted( + performance_by_cell.items(), + key=lambda kv: (int(m.group("isl")), int(m.group("osl")), int(m.group("conc"))) + if (m := _CELL_RE.match(str(kv[0]))) + else (0, 0, 0), + ): + m = _CELL_RE.match(str(cell_id)) + if m: + summary_rows.append( + [ + m.group("isl"), + m.group("osl"), + m.group("tp"), + m.group("pp"), + m.group("conc"), + result, + ] + ) + else: + summary_rows.append(["-", "-", tp, pp, str(cell_id), result]) + + log.info( + "\n\n\n\n======== Performance summary (by ISL/OSL cell) ========\n%s", + tabulate( + summary_rows, + headers=["ISL", "OSL", "TP", "PP", "Conc", "Result"], + tablefmt="github", + ), + ) + + PERF_METRICS = [ + ("Mean TTFT (ms)", "mean_ttft_ms"), + ("Mean TPOT (ms)", "mean_tpot_ms"), + ("P99 ITL (ms)", "p99_itl_ms"), + ("Mean E2E latency (ms)", "mean_e2e_latency_ms"), + ("Req/s", "request_throughput_per_sec"), + ("Output tok/s", "output_throughput_per_sec"), + ("Output tok/s/GPU", "output_throughput_per_gpu_per_sec"), + ("Goodput", "goodput"), + ("MFU (estimated)", "mfu"), + ] + + perf_items = [ + (k, v) for k, v in inf_res_dict.items() if isinstance(k, tuple) and len(k) == 6 and isinstance(v, dict) + ] + + perf_rows = [] + for key, host_dict in sorted( + perf_items, + key=lambda kv: (int(kv[0][2]), int(kv[0][3]), int(kv[0][5])), + ): + model, gpu, isl, osl, policy, conc = key + expected_map = _thresholds_for_cell(variant_config, isl, osl, conc) + for host, m in host_dict.items(): + for label, metric_key in PERF_METRICS: + actual = m.get(metric_key) + if actual is None: + continue + expected = expected_map.get(metric_key) + perf_rows.append( + [ + model, + gpu, + isl, + osl, + policy, + conc, + host, + label, + f"{float(actual):.4f}", + f"{float(expected):.4f}" if expected is not None else "-", + _perf_result(actual, expected, metric_key), + ] + ) + + if perf_rows: + log.info( + "\n\n\n\n======== Performance results ========\n%s", + tabulate( + perf_rows, + headers=[ + "Model", + "GPU", + "ISL", + "OSL", + "Policy", + "Conc", + "Host", + "Metric", + "Actual", + "Expected", + "Result", + ], + tablefmt="github", + ), + ) + elif not smoke_results and not acc_rows and not performance_by_cell: + log.info("inf_res_dict empty, nothing to print") diff --git a/cvs/tests/inference/sglang/conftest.py b/cvs/tests/inference/sglang/conftest.py new file mode 100644 index 000000000..7238a957b --- /dev/null +++ b/cvs/tests/inference/sglang/conftest.py @@ -0,0 +1,526 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Fixtures and hooks for SGLang inference suites (``sglang_single``, ``sglang_distributed``, +and ``sglang_disagg_distributed``). + +Both suites use ``ContainerOrchestrator`` (vLLM-style) via ``cluster_container.json`` and +``load_variant()`` from ``sglang_config_loader``. + +``sglang_single.py`` — ``SglangSingle`` (unified server on ``benchmark_serv_node`` only). +``sglang_distributed.py`` — ``SglangDistributed`` (unified multi-node TP/PP; all server +nodes from ``server_node_list`` or prefill+decode union). +``sglang_disagg_distributed.py`` — ``SglangDisaggPD`` (PD roles from inference config; +containers only on the union of prefill/decode/router/bench hosts). + +Each ``benchmark_params`` variant may set ``threshold_file`` to a JSON file beside the config; +that file supplies pass/fail thresholds for performance and lm-eval benchmarks. +''' + +from __future__ import annotations + +import json +import os +import re +import time +from typing import Any, Mapping + +import pytest + +from cvs.core.orchestrators.factory import OrchestratorConfig, OrchestratorFactory +from cvs.lib import globals +from cvs.lib.inference.sglang.sglang_common import as_node_list, cleanup_sglang_log_dir, resolve_server_node_list + +from cvs.lib.inference.sglang.sglang_config_loader import ( + SglangSingleVariantConfig, + flat_expected_from_specs, + load_variant, + orchestrator_container_from_variant, + perf_cells_from_thresholds, +) +from cvs.lib.inference.sglang.sglang_disagg_lib import SglangDisaggPD +from cvs.lib.inference.sglang.sglang_distributed_lib import SglangDistributed +from cvs.lib.inference.sglang.sglang_single_lib import SglangSingle +from cvs.lib.utils_lib import ( + get_model_from_rocm_smi_output, + resolve_cluster_config_placeholders, + resolve_test_config_placeholders, + update_test_result, +) +from cvs.tests.inference.sglang._shared import ( + SGLANG_DISTRIBUTED_TEST_ORDER, + SGLANG_SINGLE_TEST_ORDER, + SGLANG_TEST_ORDER, +) + +log = globals.log + +# Re-exported for sglang_single.py / sglang_disagg_distributed.py imports. +__all__ = ["flat_expected_from_specs"] + + +def _use_sglang_single(request) -> bool: + """``sglang_single.py`` uses unified single-node ``SglangSingle``.""" + return getattr(request.module, "__name__", "").endswith("sglang_single") + + +def _use_sglang_distributed(request) -> bool: + """``sglang_distributed.py`` uses unified multi-node ``SglangDistributed``.""" + return getattr(request.module, "__name__", "").endswith("sglang_distributed") + + +def _deep_merge(base, override): + """Recursively merge ``override`` onto ``base`` (dicts merged key-wise; scalars/lists replaced).""" + if not (isinstance(base, dict) and isinstance(override, dict)): + return override + out = dict(base) + for k, v in override.items(): + out[k] = _deep_merge(base[k], v) if k in base else v + return out + + +def _benchmark_serv_host(inference: Mapping[str, Any]) -> str: + """Single-node suite target host from inference ``benchmark_serv_node``.""" + raw = inference.get("benchmark_serv_node") + if not raw: + raise ValueError( + "sglang_single requires 'benchmark_serv_node' in the inference config " + "(config.json / variant inference dict)" + ) + hosts = as_node_list(raw) + if len(hosts) != 1: + raise ValueError(f"sglang_single requires exactly one benchmark_serv_node, got {hosts!r}") + return hosts[0] + + +def _cluster_dict_for_single_benchmark( + cluster_dict: Mapping[str, Any], + bench_host: str, +) -> dict[str, Any]: + """Restrict orchestrator SSH/container scope to ``benchmark_serv_node`` only.""" + node_dict = cluster_dict.get("node_dict") or {} + if bench_host not in node_dict: + raise ValueError( + f"benchmark_serv_node {bench_host!r} is not listed in cluster node_dict (keys: {sorted(node_dict)!r})" + ) + scoped = dict(cluster_dict) + scoped["node_dict"] = {bench_host: node_dict[bench_host]} + scoped["head_node_dict"] = {"mgmt_ip": bench_host} + return scoped + + +def _disagg_role_hosts(inference: Mapping[str, Any]) -> list[str]: + """Unique hosts referenced by PD role fields in the inference config.""" + seen: list[str] = [] + for key in ( + "prefill_node_list", + "decode_node_list", + "proxy_router_node", + "benchmark_serv_node", + ): + raw = inference.get(key) + if raw is None: + continue + for host in as_node_list(raw): + if host not in seen: + seen.append(host) + if not seen: + raise ValueError( + "sglang_disagg requires at least one of prefill_node_list, decode_node_list, " + "proxy_router_node, or benchmark_serv_node in the inference config" + ) + return seen + + +def _disagg_head_host(inference: Mapping[str, Any], role_hosts: list[str]) -> str: + """Orchestrator head: proxy, then benchmark, then first prefill node.""" + for key in ("proxy_router_node", "benchmark_serv_node", "prefill_node_list"): + raw = inference.get(key) + if raw is None: + continue + hosts = as_node_list(raw) + if hosts: + return hosts[0] + return role_hosts[0] + + +def _cluster_dict_for_disagg_roles( + cluster_dict: Mapping[str, Any], + role_hosts: list[str], + head_host: str, +) -> dict[str, Any]: + """Restrict orchestrator scope to role hosts (not every cluster.json node).""" + node_dict = cluster_dict.get("node_dict") or {} + missing = [h for h in role_hosts if h not in node_dict] + if missing: + raise ValueError(f"role hosts not in cluster node_dict: {missing!r} (cluster keys: {sorted(node_dict)!r})") + scoped = dict(cluster_dict) + scoped["node_dict"] = {h: node_dict[h] for h in role_hosts} + scoped["head_node_dict"] = {"mgmt_ip": head_host} + return scoped + + +def _distributed_orch_hosts(inference: Mapping[str, Any]) -> list[str]: + """Server ranks plus benchmark node (when bench is not already a server rank).""" + hosts = list(resolve_server_node_list(inference)) + bench_raw = inference.get("benchmark_serv_node") + if bench_raw: + bench = as_node_list(bench_raw)[0] + if bench not in hosts: + hosts.append(bench) + return hosts + + +def _distributed_head_host(inference: Mapping[str, Any], role_hosts: list[str]) -> str: + """Orchestrator head: benchmark node when set, else rank-0 server node.""" + bench_raw = inference.get("benchmark_serv_node") + if bench_raw: + return as_node_list(bench_raw)[0] + return role_hosts[0] + + +def _create_container_orchestrator(cluster_dict: Mapping[str, Any], variant_config: SglangSingleVariantConfig): + """Build a ``ContainerOrchestrator`` for single-node or disagg SGLang suites.""" + container_block = _deep_merge( + cluster_dict.get("container", {}), + orchestrator_container_from_variant(variant_config), + ) + testsuite_config = { + "orchestrator": "container", + "container": container_block, + } + cfg = OrchestratorConfig.from_configs(cluster_dict, testsuite_config) + return OrchestratorFactory.create_orchestrator(log, cfg) + + +# ---------- accuracy-cell helpers (disagg long-context parametrization) ---------- + +_ACC_CELL_RE = re.compile(r"^ACC_ISL=(?P\d+),OSL=(?P\d+)$") + + +def acc_cells_from_thresholds(thresholds: Mapping[str, Any]) -> list[dict[str, Any]]: + cells = [] + for cell_key, specs in thresholds.items(): + if str(cell_key).startswith("_"): + continue + m = _ACC_CELL_RE.match(str(cell_key)) + if not m: + continue + cells.append( + { + "cell_key": cell_key, + "isl": m.group("isl"), + "osl": m.group("osl"), + "specs": specs, + } + ) + cells.sort(key=lambda c: int(c["isl"])) + return cells + + +def _cluster_dict_for_collection(metafunc) -> dict[str, Any]: + cluster_file = metafunc.config.getoption("cluster_file") + if not cluster_file or not os.path.isfile(cluster_file): + return {} + with open(cluster_file, encoding="utf-8") as fp: + return resolve_cluster_config_placeholders(json.load(fp)) + + +def load_acc_cells_for_collection( + config_file: str, + cluster_dict: Mapping[str, Any] | None = None, +) -> list[dict[str, Any]]: + variant = load_variant(config_file, cluster_dict or {}) + cells = acc_cells_from_thresholds(variant.thresholds) + if not cells: + pytest.fail(f"No ACC_ISL=... accuracy cells in thresholds for {config_file!r}") + return cells + + +def pytest_generate_tests(metafunc): + config_file = metafunc.config.getoption("config_file") + if not config_file or not os.path.isfile(config_file): + return + + cluster_dict = _cluster_dict_for_collection(metafunc) + + if "perf_cell" in metafunc.fixturenames: + variant = load_variant(config_file, cluster_dict) + cells = perf_cells_from_thresholds(variant.thresholds) + if not cells: + pytest.fail(f"No ISL=... performance cells in thresholds for {config_file!r}") + ids = [f"isl{c['isl']}-osl{c['osl']}-c{c['conc']}" for c in cells] + metafunc.parametrize("perf_cell", cells, ids=ids) + + if "acc_cell" in metafunc.fixturenames: + cells = load_acc_cells_for_collection(config_file, cluster_dict) + ids = [f"acc-isl{c['isl']}-osl{c['osl']}" for c in cells] + metafunc.parametrize("acc_cell", cells, ids=ids) + + +# ---------- lifecycle (same model as vLLM conftest) ---------- + + +class _Lifecycle: + """Cross-test state for the lifecycle-as-tests model.""" + + def __init__(self): + self.failed = False + self.torn_down = False + self.report: dict[str, list[tuple[str, float, str]]] = {} + self.phase_labels: dict[str, Any] = {} + self.smoke_results: list | None = None + + def record(self, nodeid: str, label: str, value: float, unit: str = "s") -> None: + self.report.setdefault(nodeid, []).append((label, value, unit)) + + def skip_if_prior_failure(self) -> None: + if self.failed: + pytest.skip("a prior lifecycle stage failed") + + def complete_stage(self, request, label: str, t0: float) -> None: + self.record(request.node.nodeid, label, time.monotonic() - t0) + if globals.error_list: + self.failed = True + update_test_result() + + +# ---------- fixtures ---------- + + +@pytest.fixture(scope="module") +def cluster_dict(pytestconfig): + cluster_file = pytestconfig.getoption("cluster_file") + if not cluster_file: + pytest.fail("--cluster_file is required") + with open(cluster_file, encoding="utf-8") as fp: + d = json.load(fp) + return resolve_cluster_config_placeholders(d) + + +@pytest.fixture(scope="module") +def variant_config(pytestconfig, cluster_dict) -> SglangSingleVariantConfig: + config_file = pytestconfig.getoption("config_file") + if not config_file: + pytest.fail("--config_file is required") + return load_variant(config_file, cluster_dict) + + +@pytest.fixture(scope="module") +def lifecycle(): + return _Lifecycle() + + +@pytest.fixture(scope="module") +def inference_config_file(variant_config): + return variant_config.config_path + + +@pytest.fixture(scope="module") +def cluster_file(pytestconfig): + path = pytestconfig.getoption("cluster_file") + if not path: + pytest.fail("--cluster_file is required") + return path + + +@pytest.fixture(scope="module") +def inference_config_root(inference_config_file): + with open(inference_config_file, encoding="utf-8") as fp: + return json.load(fp) + + +@pytest.fixture(scope="module") +def inference_dict(variant_config): + return variant_config.inference + + +@pytest.fixture(scope="module") +def benchmark_params_dict(inference_config_root, cluster_dict): + bp = inference_config_root["benchmark_params"] + return resolve_test_config_placeholders(bp, cluster_dict) + + +@pytest.fixture(scope="module") +def benchmark_variant(variant_config): + return variant_config.variant_key + + +@pytest.fixture(scope="module") +def benchmark_params(variant_config): + return variant_config.benchmark_params + + +@pytest.fixture(scope="module") +def thresholds_dict(variant_config): + return variant_config.thresholds + + +@pytest.fixture(scope="module") +def hf_token(variant_config): + path = variant_config.paths.hf_token_file + if not os.path.isfile(path): + if variant_config.model.remote == 0: + return "" + pytest.skip(f"hf_token file missing: {path}") + with open(path, encoding="utf-8") as fp: + return fp.read().strip() + + +@pytest.fixture(scope="module") +def orch(request, cluster_dict, variant_config, lifecycle): + """``ContainerOrchestrator`` for single-node, distributed, and disagg SGLang suites.""" + cluster = cluster_dict + if _use_sglang_single(request): + bench_host = _benchmark_serv_host(variant_config.inference) + cluster = _cluster_dict_for_single_benchmark(cluster_dict, bench_host) + log.info("sglang_single: orchestrator scoped to benchmark_serv_node=%s", bench_host) + elif _use_sglang_distributed(request): + role_hosts = _distributed_orch_hosts(variant_config.inference) + head_host = _distributed_head_host(variant_config.inference, role_hosts) + cluster = _cluster_dict_for_disagg_roles(cluster_dict, role_hosts, head_host) + log.info( + "sglang_distributed: orchestrator scoped to server+bench hosts=%s head=%s", + role_hosts, + head_host, + ) + else: + role_hosts = _disagg_role_hosts(variant_config.inference) + head_host = _disagg_head_host(variant_config.inference, role_hosts) + cluster = _cluster_dict_for_disagg_roles(cluster_dict, role_hosts, head_host) + log.info( + "sglang_disagg: orchestrator scoped to role hosts=%s head=%s", + role_hosts, + head_host, + ) + o = _create_container_orchestrator(cluster, variant_config) + + yield o + + if not lifecycle.torn_down: + log.info("orch leak-guard: tearing down containers") + o.teardown_containers() + cleanup_sglang_log_dir(o, variant_config.paths.log_dir) + + +@pytest.fixture(scope="module") +def gpu_type(request, orch, variant_config): + if _use_sglang_single(request): + bench_host = _benchmark_serv_host(variant_config.inference) + smi_out_dict = orch.all.exec("rocm-smi -a | head -30") + smi_out = smi_out_dict.get(bench_host) or next(iter(smi_out_dict.values())) + elif _use_sglang_distributed(request): + probe_node = resolve_server_node_list(variant_config.inference)[0] + smi_out_dict = orch.all.exec("rocm-smi -a | head -30") + smi_out = smi_out_dict.get(probe_node) or next(iter(smi_out_dict.values())) + else: + # Disagg: probe first prefill node (may differ from cluster head). + prefill_nodes = variant_config.inference["prefill_node_list"] + probe_node = prefill_nodes[0] if isinstance(prefill_nodes, list) else prefill_nodes + smi_out_dict = orch.all.exec("rocm-smi -a | head -30") + smi_out = smi_out_dict.get(probe_node) or next(iter(smi_out_dict.values())) + return get_model_from_rocm_smi_output(smi_out) + + +@pytest.fixture(scope="module") +def inf_res_dict(): + return {} + + +@pytest.fixture(scope="module") +def im_obj( + request, + orch, + gpu_type, + variant_config, + hf_token, +): + model_name = variant_config.benchmark_params["model"] + + if _use_sglang_single(request): + return SglangSingle( + model_name, + variant_config.inference, + variant_config.benchmark_params, + hf_token, + orch=orch, + gpu_type=gpu_type, + ) + if _use_sglang_distributed(request): + return SglangDistributed( + model_name, + variant_config.inference, + variant_config.benchmark_params, + hf_token, + orch=orch, + gpu_type=gpu_type, + ) + return SglangDisaggPD( + model_name, + variant_config.inference, + variant_config.benchmark_params, + hf_token, + orch=orch, + gpu_type=gpu_type, + ) + + +# ---------- pytest hooks ---------- + + +def pytest_collection_modifyitems(items): + def rank_for(item): + mod = getattr(item.module, "__name__", "") + if mod.endswith("sglang_single"): + order = SGLANG_SINGLE_TEST_ORDER + elif mod.endswith("sglang_distributed"): + order = SGLANG_DISTRIBUTED_TEST_ORDER + else: + order = SGLANG_TEST_ORDER + return order.get(item.originalname or item.name.split("[")[0], 99) + + items.sort(key=rank_for) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + if report.when != "call": + return + lc = item.funcargs.get("lifecycle") + rows = getattr(lc, "report", {}).get(item.nodeid) if lc else None + if not rows: + return + try: + import pytest_html + except ImportError: + return + body = "".join(f"{label}{value:.1f}{unit}" for label, value, unit in rows) + html = f"{body}
stagevalueunit
" + extras = getattr(report, "extras", []) + extras.append(pytest_html.extras.html(html)) + report.extras = extras + + +# def pytest_html_results_table_header(cells): +# cells.insert(-1, "Value") +# cells.insert(-1, "Unit") +# +# +# def pytest_html_results_table_row(report, cells): +# props = dict(report.user_properties) +# has = "metric_value" in props +# val = props.get("metric_value") +# unit = props.get("metric_unit", "") if has else "" +# if not has: +# shown = "" +# elif val is None: +# shown = "-" +# elif isinstance(val, float): +# shown = f"{val:.3f}" +# else: +# shown = str(val) +# cells.insert(-1, f"{shown}") +# cells.insert(-1, f"{unit}") diff --git a/cvs/tests/inference/sglang/sglang_deepseek_r1_671b_distributed.py b/cvs/tests/inference/sglang/sglang_deepseek_r1_671b_distributed.py deleted file mode 100644 index e8692c441..000000000 --- a/cvs/tests/inference/sglang/sglang_deepseek_r1_671b_distributed.py +++ /dev/null @@ -1,439 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import pytest - -import re -import time -import json - -from cvs.lib.parallel_ssh_lib import * -from cvs.lib.utils_lib import * -from cvs.lib import docker_lib -from cvs.lib import sglang_disagg_lib -from cvs.lib import globals - -log = globals.log - - -# Importing additional cmd line args to script .. -@pytest.fixture(scope="module") -def cluster_file(pytestconfig): - """ - Retrieve the --cluster_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the cluster JSON file specified via --cluster_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --cluster_file=/path/to/cluster.json - - Use module scope so the value is resolved once per test module. - """ - return pytestconfig.getoption("cluster_file") - - -@pytest.fixture(scope="module") -def inference_config_file(pytestconfig): - """ - Retrieve the --config_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the training config JSON file specified via --config_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --config_file=/path/to/training_config.json - - Module scope avoids re-fetching the option across tests in this module. - """ - return pytestconfig.getoption("config_file") - - -# Importing the cluster and cofig files to script to access node, switch, test config params -@pytest.fixture(scope="module") -def cluster_dict(cluster_file): - """ - Load the entire cluster configuration from the provided JSON file. - - Args: - cluster_file (str): Path to the cluster JSON file. - - Returns: - dict: Parsed JSON representing the cluster (nodes, credentials, etc.). - - Notes: - - Logs the loaded structure for visibility; consider using log.debug if verbose. - """ - with open(cluster_file) as json_file: - cluster_dict = json.load(json_file) - - # Resolve path placeholders like {user-id} in cluster config - cluster_dict = resolve_cluster_config_placeholders(cluster_dict) - log.info("%s", cluster_dict) - return cluster_dict - - -@pytest.fixture(scope="module") -def inference_dict(inference_config_file, cluster_dict): - with open(inference_config_file) as json_file: - inference_dict_t = json.load(json_file) - inference_dict = inference_dict_t['config'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - inference_dict = resolve_test_config_placeholders(inference_dict, cluster_dict) - return inference_dict - - -@pytest.fixture(scope="module") -def benchmark_params_dict(inference_config_file, cluster_dict): - with open(inference_config_file) as json_file: - inference_dict_t = json.load(json_file) - benchmark_params_dict = inference_dict_t['benchmark_params'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - benchmark_params_dict = resolve_test_config_placeholders(benchmark_params_dict, cluster_dict) - - log.info("%s", benchmark_params_dict) - return benchmark_params_dict - - -@pytest.fixture(scope="module") -def hf_token(inference_dict): - """ - Load the Hugging Face access token from the file path specified in the training config. - - Args: - inference_dict (dict): Training configuration dict that includes: - - 'hf_token_file': Path to the file containing the HF token. - - Returns: - str: The HF token string read from the file. - - Behavior: - - Reads the token from inference_dict['hf_token_file'] (already resolved for placeholders). - - Strips the trailing newline from the token. - """ - hf_token_file = inference_dict['hf_token_file'] - try: - with open(hf_token_file, 'r') as fp: - hf_token = fp.read().rstrip("\n") - except FileNotFoundError: - log.error(f"Error: The file '{hf_token_file}' was not found.") - except Exception as e: - log.error(f"An error occurred: {e}") - return hf_token - - -@pytest.fixture(scope="module") -def p_phdl(cluster_dict, inference_dict): - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - p_phdl = Pssh( - log, - inference_dict['prefill_node_list'], - user=cluster_dict['username'], - pkey=cluster_dict['priv_key_file'], - env_vars=env_vars, - ) - return p_phdl - - -@pytest.fixture(scope="module") -def d_phdl(cluster_dict, inference_dict): - env_vars = cluster_dict.get("env_vars") - d_phdl = Pssh( - log, - inference_dict['decode_node_list'], - user=cluster_dict['username'], - pkey=cluster_dict['priv_key_file'], - env_vars=env_vars, - ) - return d_phdl - - -@pytest.fixture(scope="module") -def r_phdl(cluster_dict, inference_dict): - node_list = [] - env_vars = cluster_dict.get("env_vars") - node_list.append(inference_dict['proxy_router_node']) - r_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return r_phdl - - -@pytest.fixture(scope="module") -def b_phdl(cluster_dict, inference_dict): - node_list = [] - env_vars = cluster_dict.get("env_vars") - node_list.append(inference_dict['benchmark_serv_node']) - b_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return b_phdl - - -@pytest.fixture(scope="module") -def gpu_type(p_phdl, cluster_dict): - """ - Provide the GPU type string for the test module. - - Args: - cluster_dict (dict): Cluster configuration that includes the GPU type. - - Returns: - str: The GPU type (e.g., 'mi300', 'mi300x') used to select model parameters and logic. - - Notes: - - Module scope ensures this is evaluated once per test module. - - Consider validating this value against an expected set of GPU types to catch typos early. - """ - - log.info("%s", p_phdl) - head_node = p_phdl.host_list[0] - smi_out_dict = p_phdl.exec('rocm-smi -a | head -30') - smi_out = smi_out_dict[head_node] - gpu_type = get_model_from_rocm_smi_output(smi_out) - return gpu_type - - -def test_cleanup_stale_containers(p_phdl, d_phdl, r_phdl, b_phdl, inference_dict): - """ - Pytest: Clean up potentially stale Docker containers and volumes before tests. - - Notes: - - This performs a broad cleanup via delete_all_containers_and_volumes; ensure the - test environment is isolated so this doesn?t remove unrelated containers/volumes. - - Consider narrowing cleanup scope if other workloads may be present on the hosts. - """ - - container_name = inference_dict['container_name'] - for a_phdl in [p_phdl, d_phdl, r_phdl, b_phdl]: - docker_lib.kill_docker_container(a_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(a_phdl) - - # Cleanup log directory from one of the nodes - log.info('Cleaning up log directory') - r_phdl.exec(f"sudo rm -rf {inference_dict['log_dir']}") - time.sleep(5) - - -def test_launch_inference_containers(p_phdl, d_phdl, r_phdl, b_phdl, inference_dict): - log.info('Testcase launch SGLang containers') - globals.error_list = [] - container_name = inference_dict['container_name'] - # Launch the containers .. - hdl_list = [p_phdl, d_phdl] - # Users can use the one of the prefill, decode nodes as proxy, benchmark, so - # check before scheduling - if inference_dict['proxy_router_node'] == inference_dict['benchmark_serv_node']: - if (inference_dict['proxy_router_node'] in inference_dict['prefill_node_list']) or ( - inference_dict['proxy_router_node'] in inference_dict['decode_node_list'] - ): - log.info('Already part of the handle list, no need to add') - else: - hdl_list.extend(r_phdl) - else: - if (inference_dict['proxy_router_node'] in inference_dict['prefill_node_list']) or ( - inference_dict['proxy_router_node'] in inference_dict['decode_node_list'] - ): - log.info('Already part of the handle list, no need to add') - else: - hdl_list.extend(r_phdl) - if (inference_dict['benchmark_serv_node'] in inference_dict['prefill_node_list']) or ( - inference_dict['benchmark_serv_node'] in inference_dict['decode_node_list'] - ): - log.info('Already part of the handle list, no need to add') - else: - hdl_list.extend(b_phdl) - - for a_phdl in hdl_list: - docker_lib.launch_docker_container( - a_phdl, - container_name, - inference_dict['container_image'], - inference_dict['container_config']['device_list'], - inference_dict['container_config']['volume_dict'], - inference_dict['container_config']['env_dict'], - shm_size='48G', - timeout=60 * 20, - ) - # ADD verifications .. - time.sleep(30) - log.info('Verify if the containers have been launched properly') - for a_phdl in [p_phdl, d_phdl, r_phdl, b_phdl]: - out_dict = a_phdl.exec('docker ps') - for node in out_dict.keys(): - if not re.search(f'{container_name}', out_dict[node], re.I): - fail_test(f'Failed to launch container on node {node}') - update_test_result() - - -# Setup the ib devices and ensure they show up in the container -def test_setup_ibv_devices(im_obj): - """ - Validate that InfiniBand / RDMA devices are: - - Properly configured on the host - - Visible inside the inference container - - Ready for high-performance communication (Prefill/Decode) - - This test is foundational and must pass before any inference tests - that rely on RDMA-based KV cache transfer. - """ - globals.error_list = [] - im_obj.check_ibv_devices() - im_obj.exec_nic_setup_scripts() - update_test_result() - - -# Create the SGlang Inference Object Fixture -@pytest.fixture(scope="module") -def im_obj(p_phdl, d_phdl, r_phdl, b_phdl, gpu_type, inference_dict, benchmark_params_dict, hf_token): - globals.error_list = [] - bp_dict = benchmark_params_dict['deepseek-r1'] - im_obj = sglang_disagg_lib.SglangDisaggPD( - bp_dict['model'], inference_dict, bp_dict, hf_token, p_phdl, d_phdl, r_phdl, b_phdl, gpu_type - ) - return im_obj - - -def test_rms_norm(im_obj): - """ - Run RMSNorm operator tests to validate: - - GPU kernel correctness - - AITer backend functionality - - Basic compute stability before inference - - This serves as a low-level sanity check before launching servers. - """ - globals.error_list = [] - im_obj.run_test_rmsnorm() - update_test_result() - - -# Test to start the prefill servers using sglang.launch_server -def test_launch_prefill_servers(im_obj): - """ - Start SGLang Prefill servers in disaggregated PD mode. - - Prefill servers: - - Handle prompt processing - - Generate KV cache - - Serve as upstream for Decode servers - - This test prepares the Prefill side of the inference pipeline. - """ - globals.error_list = [] - im_obj.setup_prefill_container_env() - im_obj.launch_prefill_servers() - update_test_result() - - -# Test to start the decode servers using sglang.launch_server -def test_launch_decode_servers(im_obj): - """ - Start SGLang Decode servers in disaggregated PD mode. - - Decode servers: - - Consume KV cache from Prefill servers - - Generate output tokens - - Drive decode throughput and latency - - This test completes the inference data plane. - """ - globals.error_list = [] - im_obj.setup_decode_container_env() - im_obj.launch_decode_servers() - update_test_result() - - -# Test to validate the Prefill and Decode servers are ready to serve -# Inference traffic -def test_poll_for_server_ready(im_obj): - """ - Poll Prefill and Decode server logs to ensure: - - Servers have fully started - - Models are loaded - - HTTP endpoints are responding (200 OK) - - Systems are stable before inference begins - - This test prevents inference traffic from being sent too early. - """ - globals.error_list = [] - im_obj.poll_and_check_server_ready() - update_test_result() - - -# Start the proxy router serving using sglang_router.launch_router -def test_launch_proxy_router(im_obj): - """ - Start the SGLang Proxy Router. - - The Proxy Router: - - Accepts inference requests - - Routes Prefill and Decode traffic - - Coordinates disaggregated PD execution - - This test completes the control plane setup for inference serving. - """ - globals.error_list = [] - im_obj.setup_proxy_router_container_env() - im_obj.launch_proxy_router() - update_test_result() - - -# Test to run the canned gsm8k benchmark packaged with the container -def test_run_gsm8k_benchmark_test(im_obj): - """ - Execute the GSM8K benchmark using the SGLang inference serving stack. - - Purpose: - -------- - This test validates: - - End-to-end inference correctness using a real-world dataset - - Sustained decode throughput under realistic query patterns - - Proper interaction between Proxy Router, Prefill, and Decode servers - - GSM8K is a commonly used reasoning benchmark, making it a strong - signal for both correctness and performance regression detection. - """ - globals.error_list = [] - im_obj.setup_benchmark_serv_container_env() - im_obj.run_gsm8k_benchmark_test() - update_test_result() - - -# Test to run the sglang Benchmarking Testing using bench_serv -def test_run_benchmark_test(im_obj): - """ - Execute a synthetic serving benchmark using sglang.bench_serving - with a random dataset. - - Purpose: - -------- - This test focuses on: - - Stress-testing the serving infrastructure - - Evaluating scheduling, batching, and throughput under load - - Isolating serving performance independent of dataset semantics - - Randomized workloads are useful for detecting performance regressions - and scaling bottlenecks. - """ - globals.error_list = [] - im_obj.setup_benchmark_serv_container_env() - im_obj.benchserv_test_random(d_type='auto') - update_test_result() - - -# Test to validate the prefill/decode GPU layout -def test_disagg_gpu_topology(im_obj): - """ - Report occupied GPUs on prefill/decode nodes after model load. - """ - globals.error_list = [] - im_obj.sglang_disagg_gpu_counts() - update_test_result() diff --git a/cvs/tests/inference/sglang/sglang_disagg_distributed.py b/cvs/tests/inference/sglang/sglang_disagg_distributed.py new file mode 100644 index 000000000..dc3077b23 --- /dev/null +++ b/cvs/tests/inference/sglang/sglang_disagg_distributed.py @@ -0,0 +1,200 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Disaggregated (PD) SGLang benchmark: prefill, decode, proxy router, and benchmark +client roles from the inference config. Containers are launched only on the union +of role hosts (not every host in cluster.json unless all are assigned roles). + +Run: + pytest cvs/tests/inference/sglang/sglang_disagg_distributed.py \\ + --cluster_file cvs/input/cluster_file/cluster_container.json \\ + --config_file cvs/input/config_file/inference/sglang/mi30x_sglang_distributed.json \\ + --html=~/cvs_results/sglang_disagg.html + +``cluster_container.json`` ``node_dict`` must include all prefill/decode/router/bench hosts. +Model variant is selected from ``benchmark_params`` via ``active_benchmark`` / env / single-key auto. + +With ``--html``, session end also writes ``sglang_disagg_run_deck.html`` (plus JSON +and interactive viewer) via ``cvs.lib.report.presets.sglang_disagg_distributed``. +''' + +import pytest +import time +from cvs.lib.inference.sglang.sglang_common import cleanup_sglang_log_dir +from cvs.lib import globals +# from cvs.tests.inference.sglang.conftest import flat_expected_from_specs + +log = globals.log + + +def test_launch_container(orch, variant_config, lifecycle, request): + """Stage 1: launch SGLang containers and reset log directory.""" + log.info("Testcase launch SGLang containers (disagg PD)") + globals.error_list = [] + t0 = time.monotonic() + + if not orch.setup_containers(): + lifecycle.failed = True + lifecycle.complete_stage(request, "container_launch", t0) + pytest.fail("setup_containers() returned False") + + cleanup_sglang_log_dir(orch, variant_config.paths.log_dir) + + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + if not orch.verify_containers_running(name): + lifecycle.failed = True + lifecycle.complete_stage(request, "container_launch", t0) + pytest.fail(f"container {name} not running after setup_containers()") + + lifecycle.complete_stage(request, "container_launch", t0) + + +# def test_setup_ibv_devices(im_obj, lifecycle, request): +# globals.error_list = [] +# t0 = time.monotonic() +# im_obj.exec_nic_setup_scripts() +# im_obj.check_ibv_devices() +# lifecycle.complete_stage(request, "ibv_setup", t0) + + +def test_rms_norm(im_obj, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.run_test_rmsnorm() + lifecycle.complete_stage(request, "rms_norm", t0) + + +def test_launch_prefill_servers(im_obj, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.setup_prefill_container_env() + im_obj.launch_prefill_servers() + lifecycle.complete_stage(request, "prefill_launch", t0) + + +def test_launch_decode_servers(im_obj, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.setup_decode_container_env() + im_obj.launch_decode_servers() + lifecycle.complete_stage(request, "decode_launch", t0) + + +def test_poll_for_server_ready(im_obj, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.poll_and_check_server_ready() + lifecycle.complete_stage(request, "server_ready", t0) + + +def test_launch_proxy_router(im_obj, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.setup_proxy_router_container_env() + im_obj.launch_proxy_router() + lifecycle.complete_stage(request, "proxy_router_launch", t0) + + +def test_openai_compatible_http_endpoints(im_obj, inf_res_dict, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + results = im_obj.verify_openai_compatible_endpoints() + lifecycle.smoke_results = results + lifecycle.complete_stage(request, "smoke_endpoints", t0) + + +# def test_run_long_context_accuracy(im_obj, lifecycle, request, acc_cell): +# globals.error_list = [] +# t0 = time.monotonic() +# bench = im_obj.bp_dict["inference_tests"]["long_ctx_niah"] +# bench["input_length"] = acc_cell["isl"] +# bench["output_length"] = acc_cell["osl"] +# bench.setdefault("expected_results", {})["auto"] = flat_expected_from_specs(acc_cell["specs"]) +# im_obj.bp_dict["max_concurrency"] = "1" +# im_obj.setup_benchmark_serv_container_env() +# summary = im_obj.run_long_context_niah_accuracy( +# isl=int(acc_cell["isl"]), +# osl=int(acc_cell["osl"]), +# d_type="auto", +# ) +# lifecycle.phase_labels[f"accuracy_long_ctx_{acc_cell['isl']}"] = summary +# lifecycle.phase_labels.setdefault("accuracy_by_cell", {})[acc_cell["cell_key"]] = ( +# "PASS" if summary.get("passed") else "FAIL" +# ) +# lifecycle.complete_stage( +# request, +# f"long_ctx_niah[{acc_cell['isl']}/{acc_cell['osl']}]", +# t0, +# ) + + +def test_run_lm_eval_hellaswag_benchmark_test(im_obj, inf_res_dict, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.setup_benchmark_serv_container_env() + h = im_obj.run_lm_eval_hellaswag_benchmark_test() + lifecycle.phase_labels["accuracy_hellaswag"] = h + lifecycle.complete_stage(request, "lm_eval_hellaswag", t0) + + +def test_run_lm_eval_gsm8k_benchmark_test(im_obj, inf_res_dict, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.setup_benchmark_serv_container_env() + g = im_obj.run_lm_eval_gsm8k_benchmark_test() + lifecycle.phase_labels["accuracy_gsm8k"] = g + lifecycle.complete_stage(request, "lm_eval_gsm8k", t0) + + +def test_run_performance_benchmark_test(im_obj, inf_res_dict, lifecycle, request, perf_cell): + globals.error_list = [] + t0 = time.monotonic() + bench = im_obj.bp_dict["inference_tests"]["bench_serv_random"] + bench["input_length"] = perf_cell["isl"] + bench["output_length"] = perf_cell["osl"] + bench.setdefault("expected_results", {})["auto"] = dict(perf_cell["specs"]) + im_obj.bp_dict["max_concurrency"] = perf_cell["conc"] + im_obj.setup_benchmark_serv_container_env() + im_obj.benchserv_test_random(d_type="auto") + key = ( + im_obj.model_name, + im_obj.gpu_type, + perf_cell["isl"], + perf_cell["osl"], + "bench_serv_random", + str(perf_cell["conc"]), + ) + lifecycle.phase_labels.setdefault("performance_by_cell", {})[perf_cell["cell_key"]] = ( + "PASS" if not globals.error_list else "FAIL" + ) + inf_res_dict[key] = dict(im_obj.inference_results_dict or {}) + lifecycle.complete_stage(request, f"bench_serv_random[{perf_cell['isl']}/{perf_cell['osl']}]", t0) + + +def test_disagg_gpu_topology(im_obj, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.sglang_disagg_gpu_counts() + lifecycle.complete_stage(request, "gpu_topology", t0) + + +def test_print_results_table(inf_res_dict, lifecycle, variant_config): + from cvs.lib.report.registry import bind_session_results + from cvs.tests.inference.sglang._shared import test_print_results_table as _print + + bind_session_results( + inf_res_dict=inf_res_dict, + variant_config=variant_config, + lifecycle=lifecycle, + ) + _print(inf_res_dict, lifecycle, variant_config) + + +def test_teardown(orch, variant_config, lifecycle, request): + """Final stage: tear down containers and logs. Runs even if a prior stage failed.""" + t0 = time.monotonic() + orch.teardown_containers() + cleanup_sglang_log_dir(orch, variant_config.paths.log_dir) + lifecycle.record(request.node.nodeid, "teardown", time.monotonic() - t0) + lifecycle.torn_down = True diff --git a/cvs/tests/inference/sglang/sglang_distributed.py b/cvs/tests/inference/sglang/sglang_distributed.py new file mode 100644 index 000000000..34109bc59 --- /dev/null +++ b/cvs/tests/inference/sglang/sglang_distributed.py @@ -0,0 +1,160 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Multi-node unified SGLang benchmark: one sharded ``sglang.launch_server`` across all +server nodes (TP/PP + ``nnodes``). No PD disaggregation, no proxy router. + +Run: + pytest cvs/tests/inference/sglang/sglang_distributed.py \\ + --cluster_file \\ + --config_file \\ + --html=~/cvs_results/sglang_distributed.html + +Set ``server_node_list`` (or ``prefill_node_list`` + ``decode_node_list`` whose union +is every server rank) and matching ``nnodes`` in the inference config. All listed +nodes get a container and participate in the unified server. ``benchmark_serv_node`` +runs smoke/bench/lm-eval (defaults to rank-0 when omitted). + +With ``--html``, session end also writes ``sglang_distributed_run_deck.html`` (plus JSON +and interactive viewer) via ``cvs.lib.report.presets.sglang_distributed``. +''' + +import pytest +import time +from cvs.lib.inference.sglang.sglang_common import cleanup_sglang_log_dir +from cvs.lib import globals + +log = globals.log + + +def test_launch_container(orch, variant_config, lifecycle, request): + """Stage 1: launch containers and reset log directory on all server nodes.""" + log.info("Testcase launch SGLang container (distributed unified server)") + globals.error_list = [] + t0 = time.monotonic() + + if not orch.setup_containers(): + lifecycle.failed = True + lifecycle.complete_stage(request, "container_launch", t0) + pytest.fail("setup_containers() returned False") + + cleanup_sglang_log_dir(orch, variant_config.paths.log_dir) + + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + if not orch.verify_containers_running(name): + lifecycle.failed = True + lifecycle.complete_stage(request, "container_launch", t0) + pytest.fail(f"container {name} not running after setup_containers()") + + lifecycle.complete_stage(request, "container_launch", t0) + + +# def test_setup_ibv_devices(im_obj, lifecycle, request): +# globals.error_list = [] +# t0 = time.monotonic() +# im_obj.exec_nic_setup_scripts() +# im_obj.check_ibv_devices() +# lifecycle.complete_stage(request, "ibv_setup", t0) + + +def test_rms_norm(im_obj, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.run_test_rmsnorm() + lifecycle.complete_stage(request, "rms_norm", t0) + + +def test_launch_server(im_obj, lifecycle, request): + """Stage: setup env and launch unified multi-node ``sglang.launch_server``.""" + globals.error_list = [] + t0 = time.monotonic() + im_obj.setup_server_container_env() + im_obj.launch_server() + lifecycle.complete_stage(request, "server_launch", t0) + + +def test_poll_for_server_ready(im_obj, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.poll_and_check_server_ready() + lifecycle.complete_stage(request, "server_ready", t0) + + +def test_openai_compatible_http_endpoints(im_obj, inf_res_dict, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + results = im_obj.verify_openai_compatible_endpoints() + lifecycle.smoke_results = results + lifecycle.complete_stage(request, "smoke_endpoints", t0) + + +def test_run_lm_eval_hellaswag_benchmark_test(im_obj, inf_res_dict, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.setup_benchmark_serv_container_env() + h = im_obj.run_lm_eval_hellaswag_benchmark_test() + lifecycle.phase_labels["accuracy_hellaswag"] = h + lifecycle.complete_stage(request, "lm_eval_hellaswag", t0) + + +def test_run_lm_eval_gsm8k_benchmark_test(im_obj, inf_res_dict, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.setup_benchmark_serv_container_env() + g = im_obj.run_lm_eval_gsm8k_benchmark_test() + lifecycle.phase_labels["accuracy_gsm8k"] = g + lifecycle.complete_stage(request, "lm_eval_gsm8k", t0) + + +def test_run_performance_benchmark_test(im_obj, inf_res_dict, lifecycle, request, perf_cell): + globals.error_list = [] + t0 = time.monotonic() + bench = im_obj.bp_dict["inference_tests"]["bench_serv_random"] + bench["input_length"] = perf_cell["isl"] + bench["output_length"] = perf_cell["osl"] + bench.setdefault("expected_results", {})["auto"] = dict(perf_cell["specs"]) + im_obj.bp_dict["max_concurrency"] = perf_cell["conc"] + im_obj.setup_benchmark_serv_container_env() + im_obj.benchserv_test_random(d_type="auto") + key = ( + im_obj.model_name, + im_obj.gpu_type, + perf_cell["isl"], + perf_cell["osl"], + "bench_serv_random", + str(perf_cell["conc"]), + ) + lifecycle.phase_labels.setdefault("performance_by_cell", {})[perf_cell["cell_key"]] = ( + "PASS" if not globals.error_list else "FAIL" + ) + inf_res_dict[key] = dict(im_obj.inference_results_dict or {}) + lifecycle.complete_stage(request, f"bench_serv_random[{perf_cell['isl']}/{perf_cell['osl']}]", t0) + + +def test_distributed_gpu_topology(im_obj, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.sglang_distributed_gpu_counts() + lifecycle.complete_stage(request, "gpu_topology", t0) + + +def test_print_results_table(inf_res_dict, lifecycle, variant_config): + from cvs.lib.report.registry import bind_session_results + from cvs.tests.inference.sglang._shared import test_print_results_table as _print + + bind_session_results( + inf_res_dict=inf_res_dict, + variant_config=variant_config, + lifecycle=lifecycle, + ) + _print(inf_res_dict, lifecycle, variant_config) + + +def test_teardown(orch, variant_config, lifecycle, request): + """Final stage: tear down containers and logs. Runs even if a prior stage failed.""" + t0 = time.monotonic() + orch.teardown_containers() + cleanup_sglang_log_dir(orch, variant_config.paths.log_dir) + lifecycle.record(request.node.nodeid, "teardown", time.monotonic() - t0) + lifecycle.torn_down = True diff --git a/cvs/tests/inference/sglang/sglang_llama_70b_distributed.py b/cvs/tests/inference/sglang/sglang_llama_70b_distributed.py deleted file mode 100644 index 5ed25ce11..000000000 --- a/cvs/tests/inference/sglang/sglang_llama_70b_distributed.py +++ /dev/null @@ -1,439 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import pytest - -import re -import time -import json - -from cvs.lib.parallel_ssh_lib import * -from cvs.lib.utils_lib import * -from cvs.lib import docker_lib -from cvs.lib import sglang_disagg_lib -from cvs.lib import globals - -log = globals.log - - -# Importing additional cmd line args to script .. -@pytest.fixture(scope="module") -def cluster_file(pytestconfig): - """ - Retrieve the --cluster_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the cluster JSON file specified via --cluster_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --cluster_file=/path/to/cluster.json - - Use module scope so the value is resolved once per test module. - """ - return pytestconfig.getoption("cluster_file") - - -@pytest.fixture(scope="module") -def inference_config_file(pytestconfig): - """ - Retrieve the --config_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the training config JSON file specified via --config_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --config_file=/path/to/training_config.json - - Module scope avoids re-fetching the option across tests in this module. - """ - return pytestconfig.getoption("config_file") - - -# Importing the cluster and cofig files to script to access node, switch, test config params -@pytest.fixture(scope="module") -def cluster_dict(cluster_file): - """ - Load the entire cluster configuration from the provided JSON file. - - Args: - cluster_file (str): Path to the cluster JSON file. - - Returns: - dict: Parsed JSON representing the cluster (nodes, credentials, etc.). - - Notes: - - Logs the loaded structure for visibility; consider using log.debug if verbose. - """ - with open(cluster_file) as json_file: - cluster_dict = json.load(json_file) - - # Resolve path placeholders like {user-id} in cluster config - cluster_dict = resolve_cluster_config_placeholders(cluster_dict) - log.info("%s", cluster_dict) - return cluster_dict - - -@pytest.fixture(scope="module") -def inference_dict(inference_config_file, cluster_dict): - with open(inference_config_file) as json_file: - inference_dict_t = json.load(json_file) - inference_dict = inference_dict_t['config'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - inference_dict = resolve_test_config_placeholders(inference_dict, cluster_dict) - return inference_dict - - -@pytest.fixture(scope="module") -def benchmark_params_dict(inference_config_file, cluster_dict): - with open(inference_config_file) as json_file: - inference_dict_t = json.load(json_file) - benchmark_params_dict = inference_dict_t['benchmark_params'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - benchmark_params_dict = resolve_test_config_placeholders(benchmark_params_dict, cluster_dict) - - log.info("%s", benchmark_params_dict) - return benchmark_params_dict - - -@pytest.fixture(scope="module") -def hf_token(inference_dict): - """ - Load the Hugging Face access token from the file path specified in the training config. - - Args: - inference_dict (dict): Training configuration dict that includes: - - 'hf_token_file': Path to the file containing the HF token. - - Returns: - str: The HF token string read from the file. - - Behavior: - - Reads the token from inference_dict['hf_token_file'] (already resolved for placeholders). - - Strips the trailing newline from the token. - """ - hf_token_file = inference_dict['hf_token_file'] - try: - with open(hf_token_file, 'r') as fp: - hf_token = fp.read().rstrip("\n") - except FileNotFoundError: - log.error(f"Error: The file '{hf_token_file}' was not found.") - except Exception as e: - log.error(f"An error occurred: {e}") - return hf_token - - -@pytest.fixture(scope="module") -def p_phdl(cluster_dict, inference_dict): - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - p_phdl = Pssh( - log, - inference_dict['prefill_node_list'], - user=cluster_dict['username'], - pkey=cluster_dict['priv_key_file'], - env_vars=env_vars, - ) - return p_phdl - - -@pytest.fixture(scope="module") -def d_phdl(cluster_dict, inference_dict): - env_vars = cluster_dict.get("env_vars") - d_phdl = Pssh( - log, - inference_dict['decode_node_list'], - user=cluster_dict['username'], - pkey=cluster_dict['priv_key_file'], - env_vars=env_vars, - ) - return d_phdl - - -@pytest.fixture(scope="module") -def r_phdl(cluster_dict, inference_dict): - node_list = [] - env_vars = cluster_dict.get("env_vars") - node_list.append(inference_dict['proxy_router_node']) - r_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return r_phdl - - -@pytest.fixture(scope="module") -def b_phdl(cluster_dict, inference_dict): - node_list = [] - env_vars = cluster_dict.get("env_vars") - node_list.append(inference_dict['benchmark_serv_node']) - b_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return b_phdl - - -@pytest.fixture(scope="module") -def gpu_type(p_phdl, cluster_dict): - """ - Provide the GPU type string for the test module. - - Args: - cluster_dict (dict): Cluster configuration that includes the GPU type. - - Returns: - str: The GPU type (e.g., 'mi300', 'mi300x') used to select model parameters and logic. - - Notes: - - Module scope ensures this is evaluated once per test module. - - Consider validating this value against an expected set of GPU types to catch typos early. - """ - - log.info("%s", p_phdl) - head_node = p_phdl.host_list[0] - smi_out_dict = p_phdl.exec('rocm-smi -a | head -30') - smi_out = smi_out_dict[head_node] - gpu_type = get_model_from_rocm_smi_output(smi_out) - return gpu_type - - -def test_cleanup_stale_containers(p_phdl, d_phdl, r_phdl, b_phdl, inference_dict): - """ - Pytest: Clean up potentially stale Docker containers and volumes before tests. - - Notes: - - This performs a broad cleanup via delete_all_containers_and_volumes; ensure the - test environment is isolated so this doesn?t remove unrelated containers/volumes. - - Consider narrowing cleanup scope if other workloads may be present on the hosts. - """ - - container_name = inference_dict['container_name'] - for a_phdl in [p_phdl, d_phdl, r_phdl, b_phdl]: - docker_lib.kill_docker_container(a_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(a_phdl) - - # Cleanup log directory from one of the nodes - log.info('Cleaning up log directory') - r_phdl.exec(f"sudo rm -rf {inference_dict['log_dir']}") - time.sleep(5) - - -def test_launch_inference_containers(p_phdl, d_phdl, r_phdl, b_phdl, inference_dict): - log.info('Testcase launch SGLang containers') - globals.error_list = [] - container_name = inference_dict['container_name'] - # Launch the containers .. - hdl_list = [p_phdl, d_phdl] - # Users can use the one of the prefill, decode nodes as proxy, benchmark, so - # check before scheduling - if inference_dict['proxy_router_node'] == inference_dict['benchmark_serv_node']: - if (inference_dict['proxy_router_node'] in inference_dict['prefill_node_list']) or ( - inference_dict['proxy_router_node'] in inference_dict['decode_node_list'] - ): - log.info('Already part of the handle list, no need to add') - else: - hdl_list.extend(r_phdl) - else: - if (inference_dict['proxy_router_node'] in inference_dict['prefill_node_list']) or ( - inference_dict['proxy_router_node'] in inference_dict['decode_node_list'] - ): - log.info('Already part of the handle list, no need to add') - else: - hdl_list.extend(r_phdl) - if (inference_dict['benchmark_serv_node'] in inference_dict['prefill_node_list']) or ( - inference_dict['benchmark_serv_node'] in inference_dict['decode_node_list'] - ): - log.info('Already part of the handle list, no need to add') - else: - hdl_list.extend(b_phdl) - - for a_phdl in hdl_list: - docker_lib.launch_docker_container( - a_phdl, - container_name, - inference_dict['container_image'], - inference_dict['container_config']['device_list'], - inference_dict['container_config']['volume_dict'], - inference_dict['container_config']['env_dict'], - shm_size='48G', - timeout=60 * 20, - ) - # ADD verifications .. - time.sleep(30) - log.info('Verify if the containers have been launched properly') - for a_phdl in [p_phdl, d_phdl, r_phdl, b_phdl]: - out_dict = a_phdl.exec('docker ps') - for node in out_dict.keys(): - if not re.search(f'{container_name}', out_dict[node], re.I): - fail_test(f'Failed to launch container on node {node}') - update_test_result() - - -# Setup the ib devices and ensure they show up in the container -def test_setup_ibv_devices(im_obj): - """ - Validate that InfiniBand / RDMA devices are: - - Properly configured on the host - - Visible inside the inference container - - Ready for high-performance communication (Prefill/Decode) - - This test is foundational and must pass before any inference tests - that rely on RDMA-based KV cache transfer. - """ - globals.error_list = [] - im_obj.check_ibv_devices() - im_obj.exec_nic_setup_scripts() - update_test_result() - - -# Create the SGlang Inference Object Fixture -@pytest.fixture(scope="module") -def im_obj(p_phdl, d_phdl, r_phdl, b_phdl, gpu_type, inference_dict, benchmark_params_dict, hf_token): - globals.error_list = [] - bp_dict = benchmark_params_dict['llama-70b'] - im_obj = sglang_disagg_lib.SglangDisaggPD( - bp_dict['model'], inference_dict, bp_dict, hf_token, p_phdl, d_phdl, r_phdl, b_phdl, gpu_type - ) - return im_obj - - -def test_rms_norm(im_obj): - """ - Run RMSNorm operator tests to validate: - - GPU kernel correctness - - AITer backend functionality - - Basic compute stability before inference - - This serves as a low-level sanity check before launching servers. - """ - globals.error_list = [] - im_obj.run_test_rmsnorm() - update_test_result() - - -# Test to start the prefill servers using sglang.launch_server -def test_launch_prefill_servers(im_obj): - """ - Start SGLang Prefill servers in disaggregated PD mode. - - Prefill servers: - - Handle prompt processing - - Generate KV cache - - Serve as upstream for Decode servers - - This test prepares the Prefill side of the inference pipeline. - """ - globals.error_list = [] - im_obj.setup_prefill_container_env() - im_obj.launch_prefill_servers() - update_test_result() - - -# Test to start the decode servers using sglang.launch_server -def test_launch_decode_servers(im_obj): - """ - Start SGLang Decode servers in disaggregated PD mode. - - Decode servers: - - Consume KV cache from Prefill servers - - Generate output tokens - - Drive decode throughput and latency - - This test completes the inference data plane. - """ - globals.error_list = [] - im_obj.setup_decode_container_env() - im_obj.launch_decode_servers() - update_test_result() - - -# Test to validate the Prefill and Decode servers are ready to serve -# Inference traffic -def test_poll_for_server_ready(im_obj): - """ - Poll Prefill and Decode server logs to ensure: - - Servers have fully started - - Models are loaded - - HTTP endpoints are responding (200 OK) - - Systems are stable before inference begins - - This test prevents inference traffic from being sent too early. - """ - globals.error_list = [] - im_obj.poll_and_check_server_ready() - update_test_result() - - -# Start the proxy router serving using sglang_router.launch_router -def test_launch_proxy_router(im_obj): - """ - Start the SGLang Proxy Router. - - The Proxy Router: - - Accepts inference requests - - Routes Prefill and Decode traffic - - Coordinates disaggregated PD execution - - This test completes the control plane setup for inference serving. - """ - globals.error_list = [] - im_obj.setup_proxy_router_container_env() - im_obj.launch_proxy_router() - update_test_result() - - -# Test to run the canned gsm8k benchmark packaged with the container -def test_run_gsm8k_benchmark_test(im_obj): - """ - Execute the GSM8K benchmark using the SGLang inference serving stack. - - Purpose: - -------- - This test validates: - - End-to-end inference correctness using a real-world dataset - - Sustained decode throughput under realistic query patterns - - Proper interaction between Proxy Router, Prefill, and Decode servers - - GSM8K is a commonly used reasoning benchmark, making it a strong - signal for both correctness and performance regression detection. - """ - globals.error_list = [] - im_obj.setup_benchmark_serv_container_env() - im_obj.run_gsm8k_benchmark_test() - update_test_result() - - -# Test to run the sglang Benchmarking Testing using bench_serv -def test_run_benchmark_test(im_obj): - """ - Execute a synthetic serving benchmark using sglang.bench_serving - with a random dataset. - - Purpose: - -------- - This test focuses on: - - Stress-testing the serving infrastructure - - Evaluating scheduling, batching, and throughput under load - - Isolating serving performance independent of dataset semantics - - Randomized workloads are useful for detecting performance regressions - and scaling bottlenecks. - """ - globals.error_list = [] - im_obj.setup_benchmark_serv_container_env() - im_obj.benchserv_test_random(d_type='auto') - update_test_result() - - -# Test to validate the prefill/decode GPU layout -def test_disagg_gpu_topology(im_obj): - """ - Report occupied GPUs on prefill/decode nodes after model load. - """ - globals.error_list = [] - im_obj.sglang_disagg_gpu_counts() - update_test_result() diff --git a/cvs/tests/inference/sglang/sglang_single.py b/cvs/tests/inference/sglang/sglang_single.py new file mode 100644 index 000000000..6f9b3de60 --- /dev/null +++ b/cvs/tests/inference/sglang/sglang_single.py @@ -0,0 +1,171 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Single-node SGLang benchmark: one unified server on ``benchmark_serv_node`` +(``proxy_router_serv_port``). No PD disaggregation, no router. + +Run: + pytest cvs/tests/inference/sglang/sglang_single.py \\ + --cluster_file \\ + --config_file \\ + --html=~/cvs_results/sglang_single.html + +Set ``benchmark_serv_node`` in the inference config to the target host (must also +appear in the cluster file ``node_dict``). Only that node gets a container and +loads the model; other cluster nodes are ignored for this suite. + +With ``--html``, session end also writes ``sglang_single_run_deck.html`` (plus JSON +and interactive viewer) via ``cvs.lib.report.presets.sglang_single``. +''' + +import pytest +import time +from cvs.lib.inference.sglang.sglang_common import cleanup_sglang_log_dir +from cvs.lib import globals +# from cvs.tests.inference.sglang.conftest import flat_expected_from_specs + +log = globals.log + + +def test_launch_container(orch, variant_config, lifecycle, request): + """Stage 1: launch container and reset log directory.""" + log.info("Testcase launch SGLang container (single-node)") + globals.error_list = [] + t0 = time.monotonic() + + if not orch.setup_containers(): + lifecycle.failed = True + lifecycle.complete_stage(request, "container_launch", t0) + pytest.fail("setup_containers() returned False") + + cleanup_sglang_log_dir(orch, variant_config.paths.log_dir) + + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + if not orch.verify_containers_running(name): + lifecycle.failed = True + lifecycle.complete_stage(request, "container_launch", t0) + pytest.fail(f"container {name} not running after setup_containers()") + + lifecycle.complete_stage(request, "container_launch", t0) + + +def test_rms_norm(im_obj, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.run_test_rmsnorm() + lifecycle.complete_stage(request, "rms_norm", t0) + + +def test_launch_server(im_obj, lifecycle, request): + """Stage: setup env and launch one unified ``sglang.launch_server``.""" + globals.error_list = [] + t0 = time.monotonic() + im_obj.setup_server_container_env() + im_obj.launch_server() + lifecycle.complete_stage(request, "server_launch", t0) + + +def test_poll_for_server_ready(im_obj, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.poll_and_check_server_ready() + lifecycle.complete_stage(request, "server_ready", t0) + + +def test_openai_compatible_http_endpoints(im_obj, inf_res_dict, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + results = im_obj.verify_openai_compatible_endpoints() + lifecycle.smoke_results = results + lifecycle.complete_stage(request, "smoke_endpoints", t0) + + +# TODO: not implemented for single-node +# def test_run_long_context_accuracy(im_obj, lifecycle, request, acc_cell): +# globals.error_list = [] +# t0 = time.monotonic() +# bench = im_obj.bp_dict["inference_tests"]["long_ctx_niah"] +# bench["input_length"] = acc_cell["isl"] +# bench["output_length"] = acc_cell["osl"] +# bench.setdefault("expected_results", {})["auto"] = flat_expected_from_specs(acc_cell["specs"]) +# im_obj.bp_dict["max_concurrency"] = "1" +# im_obj.setup_server_container_env() +# summary = im_obj.run_long_context_niah_accuracy( +# isl=int(acc_cell["isl"]), +# osl=int(acc_cell["osl"]), +# d_type="auto", +# ) +# lifecycle.phase_labels[f"accuracy_long_ctx_{acc_cell['isl']}"] = summary +# lifecycle.phase_labels.setdefault("accuracy_by_cell", {})[acc_cell["cell_key"]] = ( +# "PASS" if summary.get("passed") else "FAIL" +# ) +# lifecycle.complete_stage( +# request, +# f"long_ctx_niah[{acc_cell['isl']}/{acc_cell['osl']}]", +# t0, +# ) + + +def test_run_lm_eval_hellaswag_benchmark_test(im_obj, inf_res_dict, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.setup_server_container_env() + h = im_obj.run_lm_eval_hellaswag_benchmark_test() + lifecycle.phase_labels["accuracy_hellaswag"] = h + lifecycle.complete_stage(request, "lm_eval_hellaswag", t0) + + +def test_run_lm_eval_gsm8k_benchmark_test(im_obj, inf_res_dict, lifecycle, request): + globals.error_list = [] + t0 = time.monotonic() + im_obj.setup_server_container_env() + g = im_obj.run_lm_eval_gsm8k_benchmark_test() + lifecycle.phase_labels["accuracy_gsm8k"] = g + lifecycle.complete_stage(request, "lm_eval_gsm8k", t0) + + +def test_run_performance_benchmark_test(im_obj, inf_res_dict, lifecycle, request, perf_cell): + globals.error_list = [] + t0 = time.monotonic() + bench = im_obj.bp_dict["inference_tests"]["bench_serv_random"] + bench["input_length"] = perf_cell["isl"] + bench["output_length"] = perf_cell["osl"] + bench.setdefault("expected_results", {})["auto"] = dict(perf_cell["specs"]) + im_obj.bp_dict["max_concurrency"] = perf_cell["conc"] + im_obj.setup_server_container_env() + im_obj.benchserv_test_random(d_type="auto") + key = ( + im_obj.model_name, + im_obj.gpu_type, + perf_cell["isl"], + perf_cell["osl"], + "bench_serv_random", + str(perf_cell["conc"]), + ) + lifecycle.phase_labels.setdefault("performance_by_cell", {})[perf_cell["cell_key"]] = ( + "PASS" if not globals.error_list else "FAIL" + ) + inf_res_dict[key] = dict(im_obj.inference_results_dict or {}) + lifecycle.complete_stage(request, f"bench_serv_random[{perf_cell['isl']}/{perf_cell['osl']}]", t0) + + +def test_print_results_table(inf_res_dict, lifecycle, variant_config): + from cvs.lib.report.registry import bind_session_results + from cvs.tests.inference.sglang._shared import test_print_results_table as _print + + bind_session_results( + inf_res_dict=inf_res_dict, + variant_config=variant_config, + lifecycle=lifecycle, + ) + _print(inf_res_dict, lifecycle, variant_config) + + +def test_teardown(orch, variant_config, lifecycle, request): + """Final stage: tear down container and logs. Runs even if a prior stage failed.""" + t0 = time.monotonic() + orch.teardown_containers() + cleanup_sglang_log_dir(orch, variant_config.paths.log_dir) + lifecycle.record(request.node.nodeid, "teardown", time.monotonic() - t0) + lifecycle.torn_down = True diff --git a/cvs/tests/inference/vllm/_shared.py b/cvs/tests/inference/vllm/_shared.py new file mode 100644 index 000000000..daedddde3 --- /dev/null +++ b/cvs/tests/inference/vllm/_shared.py @@ -0,0 +1,70 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Shared test helpers for the unified vllm suite. + +`test_print_results_table` is exported and imported by `vllm.py` as a sibling +test that pytest runs LAST (lexically after `test_vllm_inference`). +''' + +from tabulate import tabulate + +from cvs.lib import globals + +log = globals.log + +__all__ = ["test_print_results_table"] + + +def _cell(m, key): + """Table cell: a missing OR present-but-None metric renders as '-'.""" + v = m.get(key) + return "-" if v is None else v + + +def test_print_results_table(inf_res_dict): + if not inf_res_dict: + log.info("inf_res_dict empty, nothing to print") + return + headers = [ + "Model", + "GPU", + "ISL", + "OSL", + "Policy", + "Conc", + "Host", + "Req/s", + "Total tok/s", + "Mean TTFT (ms)", + "P95 TTFT (ms)", + "Mean TPOT (ms)", + "P95 TPOT (ms)", + "P99 ITL (ms)", + "Goodput (req/s)", + ] + rows = [] + for key, host_dict in inf_res_dict.items(): + model, gpu, isl, osl, policy, conc = key + for host, m in host_dict.items(): + rows.append( + [ + model, + gpu, + isl, + osl, + policy, + conc, + host, + _cell(m, "client.request_throughput"), + _cell(m, "client.total_token_throughput"), + _cell(m, "client.mean_ttft_ms"), + _cell(m, "client.p95_ttft_ms"), + _cell(m, "client.mean_tpot_ms"), + _cell(m, "client.p95_tpot_ms"), + _cell(m, "client.p99_itl_ms"), + _cell(m, "client.goodput"), + ] + ) + log.info("\n" + tabulate(rows, headers=headers, tablefmt="github")) diff --git a/cvs/tests/inference/vllm/conftest.py b/cvs/tests/inference/vllm/conftest.py new file mode 100644 index 000000000..3ccbf9b4b --- /dev/null +++ b/cvs/tests/inference/vllm/conftest.py @@ -0,0 +1,180 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +import json +import os + +import pytest + +from cvs.core.orchestrators.factory import OrchestratorConfig, OrchestratorFactory +from cvs.lib import globals +from cvs.lib.inference.utils.vllm_config_loader import load_variant +from cvs.lib.utils_lib import resolve_cluster_config_placeholders + +log = globals.log + + +def _deep_merge(base, override): + """Recursively merge `override` onto `base` (dicts merged key-wise, scalars/lists replaced). + + Protects cluster-set SCALAR and DICT container keys (e.g. shm_size, an env + map) from being wiped by a top-level replace: they survive unless the variant + overrides that same key. List keys (e.g. runtime.args, volume mounts) are + REPLACED here, not unioned -- the cluster's list values are recombined with + the variant's additively further downstream, in container.py's getters. + """ + if not (isinstance(base, dict) and isinstance(override, dict)): + return override + out = dict(base) + for k, v in override.items(): + out[k] = _deep_merge(base[k], v) if k in base else v + return out + + +@pytest.fixture(scope="module") +def cluster_dict(pytestconfig): + cluster_file = pytestconfig.getoption("cluster_file") + if not cluster_file: + pytest.fail("--cluster_file is required") + with open(cluster_file) as fp: + d = json.load(fp) + return resolve_cluster_config_placeholders(d) + + +@pytest.fixture(scope="module") +def variant_config(pytestconfig, cluster_dict): + config_file = pytestconfig.getoption("config_file") + if not config_file: + pytest.fail("--config_file is required") + return load_variant(config_file, cluster_dict) + + +class _Lifecycle: + """Cross-test state for the lifecycle-as-tests model. + + The container launch / sshd / fetch / teardown stages are individual tests + (so each is a timed, pass/fail row in the HTML) rather than fixture body + code. They share this object: `failed` lets a broken stage skip the rest + instead of cascading; `torn_down` lets the explicit teardown test suppress + the fixture's leak-guard finalizer; `report` maps a test's nodeid to the + rows it recorded, each carrying its own unit, so attach_inference_suite_lifecycle_table + renders only that test's stages -- not every stage on every row. + """ + + def __init__(self): + self.failed = False + self.torn_down = False + self.report = {} # nodeid -> list[(label, value, unit)] + + def record(self, nodeid, label, value, unit="s"): + self.report.setdefault(nodeid, []).append((label, value, unit)) + + +@pytest.fixture(scope="module") +def lifecycle(): + return _Lifecycle() + + +@pytest.fixture(scope="module") +def orch(cluster_dict, variant_config, lifecycle): + """Construct a ContainerOrchestrator and own ONLY its teardown safety net. + + The actual launch/sshd happen in test_launch_container / test_setup_sshd + so they appear as timed rows. This fixture builds the object and registers a + leak-guard finalizer: if a mid-sweep test fails before test_teardown runs, + the container is still torn down here. When test_teardown ran successfully + it sets lifecycle.torn_down, so the finalizer no-ops (no double teardown). + """ + # OrchestratorConfig.from_configs does a top-level dict.update, so a bare variant + # container block would wipe the cluster file's container settings. Deep-merge the + # variant ONTO the cluster block so cluster-set scalar/dict keys survive, with the + # variant winning on conflicting keys. (List keys like runtime.args are replaced + # here but recombined additively downstream in container.py's getters.) + container_block = _deep_merge( + cluster_dict.get("container", {}), + variant_config.container.model_dump(), + ) + testsuite_config = { + "orchestrator": "container", + "container": container_block, + } + cfg = OrchestratorConfig.from_configs(cluster_dict, testsuite_config) + o = OrchestratorFactory.create_orchestrator(log, cfg) + yield o + if not lifecycle.torn_down: + log.info("orch fixture leak-guard: tearing down container (explicit teardown did not run)") + o.teardown_containers() + + +@pytest.fixture(scope="module") +def hf_token(variant_config): + path = variant_config.paths.hf_token_file + if not os.path.isfile(path): + if variant_config.model.remote == 0: + # Pre-staged model: token not needed for download; server env sets + # HF_HUB_OFFLINE=1 to skip Hub auth checks entirely. + return "" + pytest.skip(f"hf_token file missing: {path}") + with open(path) as fp: + return fp.read().strip() + + +@pytest.fixture(scope="module") +def inf_res_dict(): + return {} + + +def pytest_collection_modifyitems(items): + """Pin the lifecycle order explicitly instead of relying on definition order. + + `test_print_results_table` is an imported function (its source line points + into _shared.py), so default ordering collects it FIRST -- which would log an + empty table before any cell ran. Sort deterministically: launch, sshd, fetch, + the benchmark cells, the results table, then teardown last. Items from other + modules keep their relative order. + """ + rank = { + "test_launch_container": 0, + "test_setup_sshd": 1, + "test_discover_topology": 2, + "test_model_fetch": 3, + "test_openai_compatible_smoke": 4, + "test_vllm_inference": 5, + "test_metric": 6, + "test_gpu_metric": 6, + "test_prom_metric": 6, + "test_accuracy_eval": 7, + "test_print_results_table": 8, + "test_teardown": 9, + } + items.sort(key=lambda it: rank.get(it.originalname or it.name.split("[")[0], 99)) + + +# def pytest_html_results_table_header(cells): +# """Add Value + Unit columns just before the trailing Links column. +# +# Populated for test_metric rows; blank for lifecycle/inference rows (they +# record no metric_value user-property). Scoped to this suite's conftest, so +# other suites' result tables are unaffected. +# """ +# cells.insert(-1, "Value") +# cells.insert(-1, "Unit") +# +# +# def pytest_html_results_table_row(report, cells): +# props = dict(report.user_properties) +# has = "metric_value" in props +# val = props.get("metric_value") +# unit = props.get("metric_unit", "") if has else "" +# if not has: +# shown = "" +# elif val is None: +# shown = "-" +# elif isinstance(val, float): +# shown = f"{val:.3f}" +# else: +# shown = str(val) +# cells.insert(-1, f"{shown}") +# cells.insert(-1, f"{unit}") diff --git a/cvs/tests/inference/vllm/vllm.py b/cvs/tests/inference/vllm/vllm.py new file mode 100644 index 000000000..c8bba6c18 --- /dev/null +++ b/cvs/tests/inference/vllm/vllm.py @@ -0,0 +1,566 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Unified vLLM benchmark suite for single-node and multinode distributed runs. + +Replaces vllm_single.py (single-node) and tests/inference/vllm_distributed/ +vllm_distributed.py (distributed) with one parametrized suite. + +The topology is determined entirely by the config file: + nnodes=1 (default) -> single-node, no distributed flags + nnodes=2 + pipeline_parallel_size=2 -> 2-node PP distributed + +IB device discovery (test_discover_topology) runs once per lifecycle for +distributed runs, before the benchmark sweep. Results are stored in the +lifecycle object and passed into VllmJob per cell. +''' + +import json +import os +import pathlib +import shlex +import time + +import pytest + +from cvs.lib import globals +from cvs.lib.inference.utils.vllm_config_loader import GoodputSlo, validate_sweep_selector +from cvs.lib.utils.gpu import ( + GPU_METRICS, + GPU_METRIC_UNITS, + agg_readings, + capture_gpu_metrics, + start_gpu_poller, + stop_and_collect_gpu_poller, +) +from cvs.lib.utils.verdict import evaluate_all +from cvs.lib.inference.utils.vllm_parsing import CLIENT_METRICS as _METRICS, CLIENT_METRIC_UNITS as _METRIC_UNITS +from cvs.lib.inference.utils.inference_suite_lifecycle import test_accuracy_eval # noqa: F401 +from cvs.lib.inference.utils.vllm_server_metrics import ( + PROM_METRICS, + PROM_METRIC_UNITS, + to_prom_metrics, +) +from cvs.lib.inference.vllm_job import VllmJob, scrape_vllm_metrics + +import importlib.util as _ilu +import pathlib as _pl + +_spec = _ilu.spec_from_file_location("_vllm_shared", _pl.Path(__file__).with_name("_shared.py")) +_mod = _ilu.module_from_spec(_spec) +_spec.loader.exec_module(_mod) +test_print_results_table = _mod.test_print_results_table # exported as a sibling test # noqa: F841 + +log = globals.log + +_FETCH_POLL_COUNT = 80 +_FETCH_POLL_WAIT_S = 30 +_FETCH_PRESENCE_RETRIES = 5 + +# Smoke-probe cell: smallest isl/osl that gets a server up to answer +# GET/POST /v1/* without spending sweep-scale time/GPU-minutes. concurrency/ +# num_prompts are irrelevant here -- the probe never calls run_client. +_SMOKE_ISL = 128 +_SMOKE_OSL = 32 + +# max-model-len for the smoke server, used only when the config doesn't set +# one explicitly. Sized for OpenAIProbe's actual requests (the structured- +# output-book probe alone needs ~50 prompt + 256 response tokens), not +# derived from _SMOKE_ISL/_SMOKE_OSL -- those describe an unrelated +# benchmark-sweep cell size and have no connection to the probe's fixed +# message content. +_SMOKE_MAX_MODEL_LEN = 512 + + +def pytest_generate_tests(metafunc): + """Parametrize test_vllm_inference from the sweep's named-combo + runs selector. + + Runs at collection time (before fixtures exist), so it reads the raw + config_file JSON directly. Validates GoodputSlo and sweep selector against + the same rules the typed loader uses so collection-time and load-time checks + cannot drift. + """ + config_file = metafunc.config.getoption("config_file") + if not config_file or not os.path.isfile(config_file): + return + with open(config_file) as fp: + raw = json.load(fp) + sweep = raw.get("sweep", {}) + combos = sweep.get("sequence_combinations", []) + runs = sweep.get("runs", []) + for combo in combos: + if combo.get("goodput_slo") is not None: + GoodputSlo(**combo["goodput_slo"]) + validate_sweep_selector([c["name"] for c in combos], [r["combo"] for r in runs]) + by_name = {c["name"]: c for c in combos} + cases = [] + ids = [] + for run in runs: + combo = by_name[run["combo"]] + conc = run["concurrency"] + cases.append((combo, conc)) + ids.append(run["combo"] + "-conc" + str(conc)) + if "metric" in metafunc.fixturenames: + if cases: + metric_cases = [] + metric_ids = [] + for (combo, c), cid in zip(cases, ids): + for short, _unit in _METRICS: + metric_cases.append((combo, c, short)) + metric_ids.append(cid + "-" + short) + metafunc.parametrize("seq_combo,concurrency,metric", metric_cases, ids=metric_ids) + elif "gpu_metric" in metafunc.fixturenames: + if cases: + gpu_metric_cases = [] + gpu_metric_ids = [] + for (combo, c), cid in zip(cases, ids): + for short, _unit in GPU_METRICS: + gpu_metric_cases.append((combo, c, short)) + gpu_metric_ids.append(cid + "-" + short) + metafunc.parametrize("seq_combo,concurrency,gpu_metric", gpu_metric_cases, ids=gpu_metric_ids) + elif "accuracy_task" in metafunc.fixturenames: + task_ids = [t["id"] for t in raw.get("accuracy", {}).get("tasks", [])] + # Parametrize even when empty: pytest auto-skips a test whose + # parametrize call got an empty list, with the same one-row-skipped + # UX as every other opt-in metric branch above -- no manual + # pytest.skip needed in the test body for the "no tasks" case. + metafunc.parametrize("accuracy_task", task_ids, ids=task_ids) + elif "prom_metric" in metafunc.fixturenames: + if cases: + prom_metric_cases = [] + prom_metric_ids = [] + for (combo, c), cid in zip(cases, ids): + for short, _unit in PROM_METRICS: + prom_metric_cases.append((combo, c, short)) + prom_metric_ids.append(cid + "-" + short) + metafunc.parametrize("seq_combo,concurrency,prom_metric", prom_metric_cases, ids=prom_metric_ids) + elif "seq_combo" in metafunc.fixturenames and "concurrency" in metafunc.fixturenames and cases: + metafunc.parametrize("seq_combo,concurrency", cases, ids=ids) + + +def _du_bytes(orch, path): + """Total bytes under `path` inside the container, or 0 if it doesn't exist yet.""" + out = orch.exec(f"bash -c {shlex.quote(f'du -sb {shlex.quote(path)} 2>/dev/null | cut -f1')}") + total = 0 + for text in (out or {}).values(): + for tok in (text or "").split(): + if tok.isdigit(): + total = max(total, int(tok)) + return total + + +def test_launch_container(orch, variant_config, lifecycle, request): + """Stage 1: launch the container. Asserts it is independently observed running.""" + t = time.monotonic() + ok = orch.setup_containers() + lifecycle.record(request.node.nodeid, "container_launch", time.monotonic() - t) + if not ok: + lifecycle.failed = True + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + pytest.fail(f"setup_containers() returned False for {name}") + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + if not orch.verify_containers_running(name): + lifecycle.failed = True + pytest.fail(f"container {name} not running after setup_containers()") + + +def test_setup_sshd(orch, lifecycle, request): + """Stage 2: no-op for vllm — distributed execution uses NCCL/gloo over host network, not MPI/sshd.""" + pytest.skip("vllm uses --distributed-executor-backend mp + NCCL; no inter-container sshd needed") + + +def test_discover_topology(orch, variant_config, lifecycle, request): + """Stage 3: discover IB HCA devices on all nodes. + + Skipped for single-node runs (nnodes=1) since IB HCA selection is not + needed for NCCL_IB_HCA on single-node. + + For distributed runs: + - Runs ibv_devinfo -l on all nodes + - If ib_hca_devices in config is an explicit list, validates it against + the discovered devices (fails loudly if a named device is absent) + - If ib_hca_devices is absent or "auto", uses the full discovered list + - Stores the resolved HCA list in lifecycle.ib_hcas for use per cell + """ + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + nn = int(variant_config.params.nnodes) + if nn == 1: + lifecycle.ib_hcas = [] + return + + from cvs.lib.utils.ib_discovery import discover_ib_hca_names, validate_ib_hca_preflight + + t = time.monotonic() + try: + discovered = discover_ib_hca_names(orch) + except RuntimeError as e: + lifecycle.failed = True + lifecycle.record(request.node.nodeid, "topology_discovery", time.monotonic() - t) + pytest.fail(str(e)) + + requested = variant_config.roles.server.ib_hca_devices + if requested and requested != "auto": + try: + validate_ib_hca_preflight(discovered, requested) + except RuntimeError as e: + lifecycle.failed = True + lifecycle.record(request.node.nodeid, "topology_discovery", time.monotonic() - t) + pytest.fail(str(e)) + resolved = requested + else: + # "auto" or absent: use whatever the first host reported (symmetry verified above). + resolved = next(iter(discovered.values())) + + lifecycle.ib_hcas = resolved + lifecycle.record(request.node.nodeid, "topology_discovery", time.monotonic() - t) + log.info("test_discover_topology: resolved HCAs=%s", resolved) + + +def test_model_fetch(orch, variant_config, lifecycle, request): + """Stage 4: ensure the model is present in the HF cache (mounted models dir).""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + models_dir = variant_config.paths.models_dir + if not models_dir: + pytest.skip("paths.models_dir unset; cannot locate/verify the HF cache") + + remote = getattr(variant_config.model, "remote", 0) + t = time.monotonic() + orch.exec(f"mkdir -p {shlex.quote(models_dir)}") + + if not remote: + final = 0 + for it in range(_FETCH_PRESENCE_RETRIES): + final = _du_bytes(orch, models_dir) + log.info("[fetch presence %d] size=%.1fGB", it, final / 1e9) + if final > 0: + break + time.sleep(_FETCH_POLL_WAIT_S) + else: + fetch = ( + f"HF_HUB_CACHE={shlex.quote(models_dir)} " + f"nohup hf download {shlex.quote(variant_config.model.id)} " + f"> /tmp/hf_fetch.log 2>&1 &" + ) + orch.exec("bash -c " + shlex.quote(fetch)) + + prev = -1 + stable = 0 + final = _du_bytes(orch, models_dir) + for it in range(_FETCH_POLL_COUNT): + cur = _du_bytes(orch, models_dir) + final = cur + log.info("[fetch poll %d] size=%.1fGB", it, cur / 1e9) + if cur > 0 and cur == prev: + stable += 1 + if stable >= 2: + break + else: + stable = 0 + prev = cur + time.sleep(_FETCH_POLL_WAIT_S) + + lifecycle.record(request.node.nodeid, "model_fetch", time.monotonic() - t) + lifecycle.record(request.node.nodeid, "model_size", final / 1e9, "GB") + if final <= 0: + lifecycle.failed = True + pytest.fail(f"no model bytes under {models_dir} after fetch") + + +def _gpu_snap(orch): + """One-shot GPU snapshot that degrades to {} on any amd-smi/parsing failure.""" + try: + return capture_gpu_metrics(orch) + except Exception: + return {} + + +def test_openai_compatible_smoke(orch, variant_config, hf_token, lifecycle, request): + """Stage: smoke-test the OpenAI-compatible HTTP API, once per module. + + Independent of the sweep -- brings up its own short-lived server at a + small fixed cell (isl/osl above; concurrency/num_prompts are irrelevant, + the probe never runs the benchmark client) purely to answer GET/POST + /v1/models, /v1/chat/completions, /v1/completions, and a structured-JSON + chat completion. Runs before the sweep so a broken server/endpoint fails + fast instead of burning a full benchmark cell first. Always stops its + server afterward (success or failure) so test_vllm_inference's first + `job.stop_server()` isn't papering over a smoke server left running. + """ + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + job = VllmJob( + orch=orch, + variant=variant_config, + hf_token=hf_token, + isl=_SMOKE_ISL, + osl=_SMOKE_OSL, + concurrency=1, + num_prompts=1, + ib_hcas=getattr(lifecycle, "ib_hcas", []), + client_poll_count=int(variant_config.params.client_poll_count), + ) + job.serve_args.setdefault("max-model-len", str(_SMOKE_MAX_MODEL_LEN)) + t = time.monotonic() + try: + job.stop_server() + job.build_server_cmd() + job.start_server() + job.wait_ready() + summary = job.probe_openai_endpoints() + except Exception: + lifecycle.failed = True + # This job owns its own short-lived server, so unlike the sweep there + # is no reuse path to consult -- dump ours. Needed because the poll + # loop's tails are no longer echoed to the console, so without this a + # bringup timeout leaves nothing from the failing server in the log. + job.dump_server_log() + raise + finally: + job.stop_server() + lifecycle.record(request.node.nodeid, "openai_smoke", time.monotonic() - t) + log.info("OpenAI-compatible smoke results:\n%s", "\n".join(summary)) + + +def test_vllm_inference(orch, variant_config, hf_token, seq_combo, concurrency, inf_res_dict, lifecycle, request): + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + isl = seq_combo["isl"] + osl = seq_combo["osl"] + job = VllmJob( + orch=orch, + variant=variant_config, + hf_token=hf_token, + isl=isl, + osl=osl, + concurrency=concurrency, + num_prompts=variant_config.params.num_prompts, + ib_hcas=getattr(lifecycle, "ib_hcas", []), + goodput_slo=seq_combo.get("goodput_slo"), + client_poll_count=int(variant_config.params.client_poll_count), + ) + + load_s = None + load_mb = None + poll_readings = [] + try: + # Reuse the already-running server when this cell needs an identical one + # (cells that differ only in concurrency share a server signature, since + # concurrency is a client-only knob). This skips a full stop + weight + # reload + warmup between such cells. The server keeps serving on the + # same port; only the client args change. + sig = job.server_signature() + if getattr(lifecycle, "live_server_sig", None) == sig: + log.info("reusing running vllm server (same server args); skipping restart") + lifecycle.record(request.node.nodeid, "server_ready", 0.0) + else: + job.stop_server() + job.build_server_cmd() + # Attribute the server-log dump target to this job as soon as it + # owns the (about-to-be-)running server, so a start_server/wait_ready + # failure below still dumps the right log via the except handler. + lifecycle.live_server_job = job + pre_snap = _gpu_snap(orch) + t = time.monotonic() + job.start_server() + job.wait_ready() + load_s = time.monotonic() - t + lifecycle.record(request.node.nodeid, "server_ready", load_s) + lifecycle.live_server_sig = sig + post_snap = _gpu_snap(orch) + load_mb = ((post_snap.get("gpu.used_vram") or 0) - (pre_snap.get("gpu.used_vram") or 0)) or None + + _htmlpath = getattr(request.config.option, "htmlpath", None) + _html_dir = getattr(request.config, "_test_html_dir", "test_html") + _gpu_log = ( + pathlib.Path(_htmlpath).parent / _html_dir / f"gpu_poll_isl{isl}_osl{osl}_conc{concurrency}.log" + if _htmlpath + else None + ) + + # Client is launched backgrounded (run_client returns immediately); a + # detached remote script snapshots amd-smi to a file on each node + # while wait_client_complete blocks the main thread on the client + # log -- no second OS thread sharing the orchestrator's SSH transport. + handle = start_gpu_poller( + orch, + run_id=f"{request.node.nodeid}_{isl}_{osl}_{concurrency}", + nodes=None if int(variant_config.params.nnodes) == 1 else list(job.orch.hosts), + ) + # One-shot scrape of vLLM's own /metrics endpoint, immediately before + # the client run -- not before the server-reuse branch above, since a + # reused server has no guaranteed-zero baseline (it may have already + # served the smoke test and/or prior cells). Two single, sequential, + # main-thread exec calls -- safe from the SSH-session race the GPU + # poller hit. + prom_before = scrape_vllm_metrics(orch, job.base_url, job.port_no) + try: + job.run_client() + job.wait_client_complete() + finally: + poll_readings = stop_and_collect_gpu_poller( + orch, + handle, + log_path=str(_gpu_log) if _gpu_log else None, + model_load_s=load_s, + model_load_memory_mb=load_mb, + ) + prom_after = scrape_vllm_metrics(orch, job.base_url, job.port_no) + results = job.parse_results() + except Exception: + lifecycle.failed = True + # A failed cell may have left the server in a bad state; force the next + # cell to do a clean bringup rather than reuse a possibly-dead server. + lifecycle.live_server_sig = None + # Dump from whichever job actually owns the running server -- on the + # reuse path (this cell only differs by concurrency) that's an earlier + # cell's job, not this one, since only the job that called + # build_server_cmd()/start_server() has a server_log path the server + # process is actually writing to. Only dumped on failure: the server + # log is per-server (not per-cell), so a success-path dump would + # re-emit the same growing log after every cell sharing a reused + # server. + getattr(lifecycle, "live_server_job", job).dump_server_log() + raise + + agg = agg_readings(poll_readings) + gpu_results = { + "gpu.peak_gpu_memory_mb": agg.get("peak_gpu_memory_mb"), + "gpu.model_load_memory_mb": load_mb, + "gpu.model_load_s": load_s, + "gpu.gpu_bandwidth_util_pct": agg.get("gpu_bandwidth_util_pct"), + "gpu.gpu_compute_util_pct": agg.get("gpu_compute_util_pct"), + } + prom_results = to_prom_metrics(prom_before, prom_after) + for host_actuals in results.values(): + host_actuals.update(gpu_results) + host_actuals.update(prom_results) + + key = ( + variant_config.model.id, + variant_config.gpu_arch, + isl, + osl, + seq_combo.get("name", "default"), + concurrency, + ) + inf_res_dict[key] = results + + +def test_metric(seq_combo, concurrency, metric, inf_res_dict, variant_config, lifecycle, request): + """One pytest test (= one HTML row) per perf metric per cell.""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + isl = seq_combo["isl"] + osl = seq_combo["osl"] + key = ( + variant_config.model.id, + variant_config.gpu_arch, + isl, + osl, + seq_combo.get("name", "default"), + concurrency, + ) + if key not in inf_res_dict: + pytest.skip(f"no recorded results for cell {key!r} (inference did not run)") + host_dict = inf_res_dict[key] + _host, actuals = next(iter(host_dict.items())) + full = "client." + metric + value = actuals.get(full) + unit = _METRIC_UNITS.get(metric, "-") + request.node.user_properties.append(("metric_value", value)) + request.node.user_properties.append(("metric_unit", unit)) + + if not variant_config.enforce_thresholds: + return + cell = variant_config.cell_key(isl, osl, concurrency) + spec = (variant_config.thresholds.get(cell) or {}).get(full) + if spec is None: + return + evaluate_all(actuals, {full: spec}) + + +def test_gpu_metric(seq_combo, concurrency, gpu_metric, inf_res_dict, variant_config, lifecycle, request): + """One pytest test (= one HTML row) per GPU metric per cell.""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + isl = seq_combo["isl"] + osl = seq_combo["osl"] + key = ( + variant_config.model.id, + variant_config.gpu_arch, + isl, + osl, + seq_combo.get("name", "default"), + concurrency, + ) + if key not in inf_res_dict: + pytest.skip(f"no recorded results for cell {key!r} (inference did not run)") + host_dict = inf_res_dict[key] + _host, actuals = next(iter(host_dict.items())) + full = "gpu." + gpu_metric + value = actuals.get(full) + unit = GPU_METRIC_UNITS.get(gpu_metric, "-") + request.node.user_properties.append(("metric_value", value)) + request.node.user_properties.append(("metric_unit", unit)) + + if value is None: + pytest.skip(f"{full}: no value recorded (amd-smi unavailable or polling failed)") + + if not variant_config.enforce_thresholds: + return + cell = variant_config.cell_key(isl, osl, concurrency) + spec = (variant_config.thresholds.get(cell) or {}).get(full) + if spec is None: + return + evaluate_all(actuals, {full: spec}) + + +def test_prom_metric(seq_combo, concurrency, prom_metric, inf_res_dict, variant_config, lifecycle, request): + """One pytest test (= one HTML row) per vLLM /metrics-derived metric per cell.""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + isl = seq_combo["isl"] + osl = seq_combo["osl"] + key = ( + variant_config.model.id, + variant_config.gpu_arch, + isl, + osl, + seq_combo.get("name", "default"), + concurrency, + ) + if key not in inf_res_dict: + pytest.skip(f"no recorded results for cell {key!r} (inference did not run)") + host_dict = inf_res_dict[key] + _host, actuals = next(iter(host_dict.items())) + full = "prom." + prom_metric + value = actuals.get(full) + unit = PROM_METRIC_UNITS.get(prom_metric, "-") + request.node.user_properties.append(("metric_value", value)) + request.node.user_properties.append(("metric_unit", unit)) + + if value is None: + pytest.skip(f"{full}: no value recorded (/metrics scrape unavailable or unparseable)") + + if not variant_config.enforce_thresholds: + return + cell = variant_config.cell_key(isl, osl, concurrency) + spec = (variant_config.thresholds.get(cell) or {}).get(full) + if spec is None: + return + evaluate_all(actuals, {full: spec}) + + +def test_teardown(orch, lifecycle, request): + """Final stage: explicit container teardown, timed, asserting it is gone.""" + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + t = time.monotonic() + orch.teardown_containers() + lifecycle.record(request.node.nodeid, "teardown", time.monotonic() - t) + if orch.verify_containers_running(name): + pytest.fail(f"container {name} still running after teardown_containers()") + lifecycle.torn_down = True diff --git a/cvs/tests/inference/vllm/vllm_deepseek31_685b_single.py b/cvs/tests/inference/vllm/vllm_deepseek31_685b_single.py deleted file mode 100644 index 429f5e889..000000000 --- a/cvs/tests/inference/vllm/vllm_deepseek31_685b_single.py +++ /dev/null @@ -1,453 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import pytest - -import re -import os -import time -import json -from pprint import pprint -from tabulate import tabulate - - -from cvs.lib.parallel_ssh_lib import * -from cvs.lib.utils_lib import * -from cvs.lib import docker_lib -from cvs.lib.inference.vllm import VllmJob -from cvs.lib import globals - -log = globals.log - -# Model name for this test suite -MODEL_NAME = "deepseek-v31" - -inf_res_dict = {} - - -# Importing additional cmd line args to script .. -@pytest.fixture(scope="module") -def cluster_file(pytestconfig): - """ - Retrieve the --cluster_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the cluster JSON file specified via --cluster_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --cluster_file=/path/to/cluster.json - - Use module scope so the value is resolved once per test module. - """ - return pytestconfig.getoption("cluster_file") - - -@pytest.fixture(scope="module") -def training_config_file(pytestconfig): - """ - Retrieve the --config_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the training config JSON file specified via --config_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --config_file=/path/to/training_config.json - - Module scope avoids re-fetching the option across tests in this module. - """ - return pytestconfig.getoption("config_file") - - -# Importing the cluster and cofig files to script to access node, switch, test config params -@pytest.fixture(scope="module") -def cluster_dict(cluster_file): - """ - Load the entire cluster configuration from the provided JSON file. - - Args: - cluster_file (str): Path to the cluster JSON file. - - Returns: - dict: Parsed JSON representing the cluster (nodes, credentials, etc.). - - Notes: - - Logs the loaded structure for visibility; consider using log.debug if verbose. - """ - with open(cluster_file) as json_file: - cluster_dict = json.load(json_file) - - # Resolve path placeholders like {user-id} in cluster config - cluster_dict = resolve_cluster_config_placeholders(cluster_dict) - log.info("%s", cluster_dict) - return cluster_dict - - -@pytest.fixture(scope="module") -def inference_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - inference_dict = inference_dict_t['config'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - inference_dict = resolve_test_config_placeholders(inference_dict, cluster_dict) - return inference_dict - - -@pytest.fixture(scope="module") -def benchmark_params_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - benchmark_params_dict = inference_dict_t['benchmark_params'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - benchmark_params_dict = resolve_test_config_placeholders(benchmark_params_dict, cluster_dict) - - log.info("%s", benchmark_params_dict) - return benchmark_params_dict - - -def pytest_generate_tests(metafunc): - """ - Dynamically parametrize inference tests based on sequence combinations and concurrency levels - for the DeepSeek-V3.1 model. - - Behavior: - - Reads the config file path from pytest's --config_file option. - - Loads the JSON and extracts deepseek-v31 model configuration. - - Extracts sequence_combinations (ISL/OSL pairs) and concurrency_levels. - - Creates test cases for each sequence combination × concurrency level. - - Test Matrix: - - Test IDs: "combination_name-concX" (e.g., "balanced-conc16") - - Notes: - - If no config_file is provided, the hook returns without parametrizing. - - Each combination gets a separate test case with clear ID. - """ - config_file = metafunc.config.getoption("config_file") - if not config_file or not os.path.exists(config_file): - log.warning(f'Warning: Missing or invalid config file {config_file}') - return - - with open(config_file) as fp: - cfg = json.load(fp) - - # Extract deepseek-v31 model config (now directly under benchmark_params, no single_node nesting) - benchmark_params = cfg.get("benchmark_params", {}) - model_config = benchmark_params.get(MODEL_NAME, {}) - - if not model_config: - log.warning(f'Warning: Model {MODEL_NAME} not found in config') - return - - # Build test parameters: list of (seq_combo_dict, concurrency, test_id) - test_params = [] - - # Check if model uses sequence_combinations or legacy ISL/OSL - seq_combos = model_config.get("sequence_combinations", []) - - if seq_combos: - # New format: multiple combinations per model - for combo in seq_combos: - combo_name = combo.get("name", f"isl{combo['isl']}_osl{combo['osl']}") - - # Check if model has concurrency_levels array or single max_concurrency - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - # Parametrize across concurrency levels - for conc in conc_levels: - test_id = f"{combo_name}-conc{conc}" - test_params.append((combo, conc, test_id)) - else: - # Backward compatibility: use max_concurrency as single value - max_conc = int(model_config.get("max_concurrency", "64")) - test_id = f"{combo_name}" - test_params.append((combo, max_conc, test_id)) - else: - # Legacy format: single ISL/OSL values - isl = model_config.get("input_sequence_length", "1024") - osl = model_config.get("output_sequence_length", "1024") - combo = {"isl": isl, "osl": osl, "name": "default"} - - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - for conc in conc_levels: - test_id = f"conc{conc}" - test_params.append((combo, conc, test_id)) - else: - max_conc = int(model_config.get("max_concurrency", "64")) - test_params.append((combo, max_conc, "default")) - - # Parametrize if test uses these fixtures - if "seq_combo" in metafunc.fixturenames and "concurrency" in metafunc.fixturenames: - if test_params: - combos, concs, ids = zip(*test_params) - metafunc.parametrize("seq_combo,concurrency", list(zip(combos, concs)), ids=ids) - - -@pytest.fixture(scope="module") -def hf_token(inference_dict): - """ - Load the Hugging Face access token from the file path specified in the training config. - - Args: - inference_dict (dict): Training configuration dict that includes: - - 'hf_token_file': Path to the file containing the HF token. - - Returns: - str: The HF token string read from the file. - - Behavior: - - Reads the token from inference_dict['hf_token_file'] (already resolved for placeholders). - - Strips the trailing newline from the token. - """ - hf_token_file = inference_dict['hf_token_file'] - try: - with open(hf_token_file, 'r') as fp: - hf_token = fp.read().rstrip("\n") - except FileNotFoundError: - log.error(f"Error: The file '{hf_token_file}' was not found.") - raise - except Exception as e: - log.error(f"An error occurred: {e}") - raise - return hf_token - - -@pytest.fixture(scope="module") -def s_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (server). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - s_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return s_phdl - - -@pytest.fixture(scope="module") -def c_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (client). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - c_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return c_phdl - - -def test_cleanup_stale_containers(s_phdl, inference_dict): - """ - Pytest: Clean up potentially stale Docker containers and volumes before tests. - - Args: - s_phdl: Parallel SSH/process handle used by docker_lib to run commands on nodes. - inference_dict (dict): Training configuration dict that includes: - - 'container_name': Name of the container to be killed if running. - - Behavior: - - Kills the specific container identified by inference_dict['container_name']. - - Deletes all containers and volumes on the target nodes (broad cleanup). - - Notes: - - This performs a broad cleanup via delete_all_containers_and_volumes; ensure the - test environment is isolated so this doesn't remove unrelated containers/volumes. - - Consider narrowing cleanup scope if other workloads may be present on the hosts. - """ - - container_name = inference_dict['container_name'] - docker_lib.kill_docker_container(s_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(s_phdl) - - -def test_launch_inference_containers(s_phdl, inference_dict, benchmark_params_dict): - """ - Launch vLLM inference containers on all nodes. - - Note: Container image can be model-specific or use global default. - """ - - log.info(f'Testcase launch vLLM containers for {MODEL_NAME}') - globals.error_list = [] - container_name = inference_dict['container_name'] - - # Get model-specific container image or use global default - container_image = benchmark_params_dict.get(MODEL_NAME, {}).get( - 'container_image', inference_dict['container_image'] - ) - - # Launch the containers .. - docker_lib.launch_docker_container( - s_phdl, - container_name, - container_image, - inference_dict['container_config']['device_list'], - inference_dict['container_config']['volume_dict'], - inference_dict['container_config']['env_dict'], - shm_size='16G', - timeout=60 * 20, - ) - # ADD verifications .. - time.sleep(30) - log.info('Verify if the containers have been launched properly') - out_dict = s_phdl.exec('docker ps') - for node in out_dict.keys(): - if not re.search(f'{container_name}', out_dict[node], re.I): - fail_test(f'Failed to launch container on node {node}') - update_test_result() - - -def test_vllm_inference(c_phdl, s_phdl, inference_dict, benchmark_params_dict, hf_token, seq_combo, concurrency): - """ - Test vLLM inference for DeepSeek-V3.1 with specific sequence combination and concurrency level. - - This test is parametrized via pytest_generate_tests to run once per: - - Sequence combination (ISL/OSL pair) defined in model's sequence_combinations - - Concurrency level defined in model's concurrency_levels - - The factory will automatically create the correct VllmJob instance with - the model-specific container image and parameters. - - Args: - seq_combo: Dict with 'isl', 'osl', 'name' keys for this test iteration - concurrency: Integer concurrency level for this test iteration - """ - # Since this is a per-GPU-type config file (mi355x), gpu_type is implicit - gpu_type = "mi355x" - - log.info( - f"Starting inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']} (ISL={seq_combo['isl']}, OSL={seq_combo['osl']}), concurrency: {concurrency}" - ) - globals.error_list = [] - - # Override ISL/OSL and concurrency in benchmark_params for this specific test iteration - # Config is now fully flattened, so access directly under benchmark_params - model_params = benchmark_params_dict[MODEL_NAME] - model_params['input_sequence_length'] = seq_combo['isl'] - model_params['output_sequence_length'] = seq_combo['osl'] - model_params['max_concurrency'] = str(concurrency) - - # Calculate num_prompts based on OSL (matching recipe logic) - osl = int(seq_combo['osl']) - if osl == 8192: - model_params['num_prompts'] = str(concurrency * 20) - else: - model_params['num_prompts'] = str(concurrency * 50) - - # Create VllmJob instance - vllm_job = VllmJob( - c_phdl=c_phdl, - s_phdl=s_phdl, - model_name=MODEL_NAME, - inference_config_dict=inference_dict, - benchmark_params_dict=benchmark_params_dict, - hf_token=hf_token, - gpu_type=gpu_type, - distributed_inference=False, - ) - - # Stop any existing server process for clean state - vllm_job.stop_server() - - # Build and start server with current test parameters - vllm_job.build_server_inference_job_cmd() - vllm_job.start_inference_server_job() - - # Run benchmark client - vllm_job.start_inference_client_job() - # res_dict will have status, results from base inference class - # - {"status": "success", "results": self.inference_result_dict} - res_dict = vllm_job.poll_for_inference_completion() - res_index = (MODEL_NAME, gpu_type, seq_combo['isl'], seq_combo['osl'], seq_combo['name'], concurrency) - inf_res_dict[res_index] = res_dict - vllm_job.verify_inference_results() - update_test_result() - - log.info( - f"Completed inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']}, concurrency: {concurrency}" - ) - - -def test_print_results_table(): - globals.error_list = [] - log.info("%s", inf_res_dict) - pprint(inf_res_dict, depth=3) - rows = [] - headers = [ - "Model", - "GPU", - "ISL", - "OSL", - "Policy", - "Concurrency", - "Host", - "Req/s", - "Total tok/s", - "Mean TTFT (ms)", - "Mean TPOT (ms)", - "P99 ITL (ms)", - ] - - for (model, gpu, isl, osl, policy, concurrency), entry in inf_res_dict.items(): - for host, m in entry["results"].items(): - rows.append( - [ - model, - gpu, - isl, - osl, - policy, - concurrency, - host, - m["successful_requests"], - m["total_throughput_per_sec"], - m["mean_ttft_ms"], - m["mean_tpot_ms"], - m["p99_itl_ms"], - ] - ) - - log.info(tabulate(rows, headers=headers, tablefmt="github")) - update_test_result() diff --git a/cvs/tests/inference/vllm/vllm_gpt_oss_120b_single.py b/cvs/tests/inference/vllm/vllm_gpt_oss_120b_single.py deleted file mode 100644 index 5c6c8e5f6..000000000 --- a/cvs/tests/inference/vllm/vllm_gpt_oss_120b_single.py +++ /dev/null @@ -1,451 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import pytest - -import re -import os -import time -import json -from pprint import pprint -from tabulate import tabulate - -from cvs.lib.parallel_ssh_lib import * -from cvs.lib.utils_lib import * -from cvs.lib import docker_lib -from cvs.lib.inference.vllm import VllmJob -from cvs.lib import globals - -log = globals.log - -# Model name for this test suite -MODEL_NAME = "gpt-oss-120b" - -inf_res_dict = {} - - -# Importing additional cmd line args to script .. -@pytest.fixture(scope="module") -def cluster_file(pytestconfig): - """ - Retrieve the --cluster_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the cluster JSON file specified via --cluster_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --cluster_file=/path/to/cluster.json - - Use module scope so the value is resolved once per test module. - """ - return pytestconfig.getoption("cluster_file") - - -@pytest.fixture(scope="module") -def training_config_file(pytestconfig): - """ - Retrieve the --config_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the training config JSON file specified via --config_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --config_file=/path/to/training_config.json - - Module scope avoids re-fetching the option across tests in this module. - """ - return pytestconfig.getoption("config_file") - - -# Importing the cluster and cofig files to script to access node, switch, test config params -@pytest.fixture(scope="module") -def cluster_dict(cluster_file): - """ - Load the entire cluster configuration from the provided JSON file. - - Args: - cluster_file (str): Path to the cluster JSON file. - - Returns: - dict: Parsed JSON representing the cluster (nodes, credentials, etc.). - - Notes: - - Logs the loaded structure for visibility; consider using log.debug if verbose. - """ - with open(cluster_file) as json_file: - cluster_dict = json.load(json_file) - - # Resolve path placeholders like {user-id} in cluster config - cluster_dict = resolve_cluster_config_placeholders(cluster_dict) - log.info("%s", cluster_dict) - return cluster_dict - - -@pytest.fixture(scope="module") -def inference_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - inference_dict = inference_dict_t['config'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - inference_dict = resolve_test_config_placeholders(inference_dict, cluster_dict) - return inference_dict - - -@pytest.fixture(scope="module") -def benchmark_params_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - benchmark_params_dict = inference_dict_t['benchmark_params'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - benchmark_params_dict = resolve_test_config_placeholders(benchmark_params_dict, cluster_dict) - - log.info("%s", benchmark_params_dict) - return benchmark_params_dict - - -def pytest_generate_tests(metafunc): - """ - Dynamically parametrize inference tests based on sequence combinations and concurrency levels - for the GPT-OSS-120B model. - - Behavior: - - Reads the config file path from pytest's --config_file option. - - Loads the JSON and extracts gpt-oss-120b model configuration. - - Extracts sequence_combinations (ISL/OSL pairs) and concurrency_levels. - - Creates test cases for each sequence combination × concurrency level. - - Test Matrix: - - Test IDs: "combination_name-concX" (e.g., "balanced-conc16") - - Notes: - - If no config_file is provided, the hook returns without parametrizing. - - Each combination gets a separate test case with clear ID. - """ - config_file = metafunc.config.getoption("config_file") - if not config_file or not os.path.exists(config_file): - log.warning(f'Warning: Missing or invalid config file {config_file}') - return - - with open(config_file) as fp: - cfg = json.load(fp) - - # Extract gpt-oss-120b model config (now directly under benchmark_params, no single_node nesting) - benchmark_params = cfg.get("benchmark_params", {}) - model_config = benchmark_params.get(MODEL_NAME, {}) - - if not model_config: - log.warning(f'Warning: Model {MODEL_NAME} not found in config') - return - - # Build test parameters: list of (seq_combo_dict, concurrency, test_id) - test_params = [] - - # Check if model uses sequence_combinations or legacy ISL/OSL - seq_combos = model_config.get("sequence_combinations", []) - - if seq_combos: - # New format: multiple combinations per model - for combo in seq_combos: - combo_name = combo.get("name", f"isl{combo['isl']}_osl{combo['osl']}") - - # Check if model has concurrency_levels array or single max_concurrency - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - # Parametrize across concurrency levels - for conc in conc_levels: - test_id = f"{combo_name}-conc{conc}" - test_params.append((combo, conc, test_id)) - else: - # Backward compatibility: use max_concurrency as single value - max_conc = int(model_config.get("max_concurrency", "64")) - test_id = f"{combo_name}" - test_params.append((combo, max_conc, test_id)) - else: - # Legacy format: single ISL/OSL values - isl = model_config.get("input_sequence_length", "1024") - osl = model_config.get("output_sequence_length", "1024") - combo = {"isl": isl, "osl": osl, "name": "default"} - - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - for conc in conc_levels: - test_id = f"conc{conc}" - test_params.append((combo, conc, test_id)) - else: - max_conc = int(model_config.get("max_concurrency", "64")) - test_params.append((combo, max_conc, "default")) - - # Parametrize if test uses these fixtures - if "seq_combo" in metafunc.fixturenames and "concurrency" in metafunc.fixturenames: - if test_params: - combos, concs, ids = zip(*test_params) - metafunc.parametrize("seq_combo,concurrency", list(zip(combos, concs)), ids=ids) - - -@pytest.fixture(scope="module") -def hf_token(inference_dict): - """ - Load the Hugging Face access token from the file path specified in the training config. - - Args: - inference_dict (dict): Training configuration dict that includes: - - 'hf_token_file': Path to the file containing the HF token. - - Returns: - str: The HF token string read from the file. - - Behavior: - - Reads the token from inference_dict['hf_token_file'] (already resolved for placeholders). - - Strips the trailing newline from the token. - """ - hf_token_file = inference_dict['hf_token_file'] - try: - with open(hf_token_file, 'r') as fp: - hf_token = fp.read().rstrip("\n") - except FileNotFoundError: - log.error(f"Error: The file '{hf_token_file}' was not found.") - raise - except Exception as e: - log.error(f"An error occurred: {e}") - raise - return hf_token - - -@pytest.fixture(scope="module") -def s_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (server). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - s_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return s_phdl - - -@pytest.fixture(scope="module") -def c_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (client). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - c_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return c_phdl - - -def test_cleanup_stale_containers(s_phdl, inference_dict): - """ - Pytest: Clean up potentially stale Docker containers and volumes before tests. - - Args: - s_phdl: Parallel SSH/process handle used by docker_lib to run commands on nodes. - inference_dict (dict): Training configuration dict that includes: - - 'container_name': Name of the container to be killed if running. - - Behavior: - - Kills the specific container identified by inference_dict['container_name']. - - Deletes all containers and volumes on the target nodes (broad cleanup). - - Notes: - - This performs a broad cleanup via delete_all_containers_and_volumes; ensure the - test environment is isolated so this doesn't remove unrelated containers/volumes. - - Consider narrowing cleanup scope if other workloads may be present on the hosts. - """ - - container_name = inference_dict['container_name'] - docker_lib.kill_docker_container(s_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(s_phdl) - - -def test_launch_inference_containers(s_phdl, inference_dict, benchmark_params_dict): - """ - Launch vLLM inference containers on all nodes. - - Note: Container image can be model-specific or use global default. - """ - - log.info(f'Testcase launch vLLM containers for {MODEL_NAME}') - globals.error_list = [] - container_name = inference_dict['container_name'] - - # Get model-specific container image or use global default - container_image = benchmark_params_dict.get(MODEL_NAME, {}).get( - 'container_image', inference_dict['container_image'] - ) - - # Launch the containers .. - docker_lib.launch_docker_container( - s_phdl, - container_name, - container_image, - inference_dict['container_config']['device_list'], - inference_dict['container_config']['volume_dict'], - inference_dict['container_config']['env_dict'], - shm_size='16G', - timeout=60 * 20, - ) - # ADD verifications .. - time.sleep(30) - log.info('Verify if the containers have been launched properly') - out_dict = s_phdl.exec('docker ps') - for node in out_dict.keys(): - if not re.search(f'{container_name}', out_dict[node], re.I): - fail_test(f'Failed to launch container on node {node}') - update_test_result() - - -def test_vllm_inference(c_phdl, s_phdl, inference_dict, benchmark_params_dict, hf_token, seq_combo, concurrency): - """ - Test vLLM inference for GPT-OSS-120B with specific sequence combination and concurrency level. - - This test is parametrized via pytest_generate_tests to run once per: - - Sequence combination (ISL/OSL pair) defined in model's sequence_combinations - - Concurrency level defined in model's concurrency_levels - - The factory will automatically create the correct VllmJob instance with - the model-specific container image and parameters. - - Args: - seq_combo: Dict with 'isl', 'osl', 'name' keys for this test iteration - concurrency: Integer concurrency level for this test iteration - """ - # Since this is a per-GPU-type config file (mi355x), gpu_type is implicit - gpu_type = "mi355x" - - log.info( - f"Starting inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']} (ISL={seq_combo['isl']}, OSL={seq_combo['osl']}), concurrency: {concurrency}" - ) - globals.error_list = [] - - # Override ISL/OSL and concurrency in benchmark_params for this specific test iteration - # Config is now fully flattened, so access directly under benchmark_params - model_params = benchmark_params_dict[MODEL_NAME] - model_params['input_sequence_length'] = seq_combo['isl'] - model_params['output_sequence_length'] = seq_combo['osl'] - model_params['max_concurrency'] = str(concurrency) - - # Calculate num_prompts based on OSL (matching recipe logic) - osl = int(seq_combo['osl']) - if osl == 8192: - model_params['num_prompts'] = str(concurrency * 20) - else: - model_params['num_prompts'] = str(concurrency * 50) - - # Create VllmJob instance - vllm_job = VllmJob( - c_phdl=c_phdl, - s_phdl=s_phdl, - model_name=MODEL_NAME, - inference_config_dict=inference_dict, - benchmark_params_dict=benchmark_params_dict, - hf_token=hf_token, - gpu_type=gpu_type, - distributed_inference=False, - ) - - # Stop any existing server process for clean state - vllm_job.stop_server() - - # Build and start server with current test parameters - vllm_job.build_server_inference_job_cmd() - vllm_job.start_inference_server_job() - - # Run benchmark client - vllm_job.start_inference_client_job() - # res_dict will have status, results from base inference class - # - {"status": "success", "results": self.inference_result_dict} - res_dict = vllm_job.poll_for_inference_completion() - res_index = (MODEL_NAME, gpu_type, seq_combo['isl'], seq_combo['osl'], seq_combo['name'], concurrency) - inf_res_dict[res_index] = res_dict - vllm_job.verify_inference_results() - update_test_result() - - log.info( - f"Completed inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']}, concurrency: {concurrency}" - ) - - -def test_print_results_table(): - globals.error_list = [] - pprint(inf_res_dict, depth=3) - rows = [] - headers = [ - "Model", - "GPU", - "ISL", - "OSL", - "Policy", - "Concurrency", - "Host", - "Req/s", - "Total tok/s", - "Mean TTFT (ms)", - "Mean TPOT (ms)", - "P99 ITL (ms)", - ] - - for (model, gpu, isl, osl, policy, concurrency), entry in inf_res_dict.items(): - for host, m in entry["results"].items(): - rows.append( - [ - model, - gpu, - isl, - osl, - policy, - concurrency, - host, - m["successful_requests"], - m["total_throughput_per_sec"], - m["mean_ttft_ms"], - m["mean_tpot_ms"], - m["p99_itl_ms"], - ] - ) - - log.info(tabulate(rows, headers=headers, tablefmt="github")) - update_test_result() diff --git a/cvs/tests/inference/vllm/vllm_qwen3_235b_single.py b/cvs/tests/inference/vllm/vllm_qwen3_235b_single.py deleted file mode 100644 index f46c44c10..000000000 --- a/cvs/tests/inference/vllm/vllm_qwen3_235b_single.py +++ /dev/null @@ -1,449 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import pytest - -import re -import os -import time -import json -from pprint import pprint -from tabulate import tabulate - -from cvs.lib.parallel_ssh_lib import * -from cvs.lib.utils_lib import * -from cvs.lib import docker_lib -from cvs.lib.inference.vllm import VllmJob -from cvs.lib import globals - -log = globals.log - -# Model name for this test suite -MODEL_NAME = "qwen3-235b" - -inf_res_dict = {} - - -# Importing additional cmd line args to script .. -@pytest.fixture(scope="module") -def cluster_file(pytestconfig): - """ - Retrieve the --cluster_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the cluster JSON file specified via --cluster_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --cluster_file=/path/to/cluster.json - - Use module scope so the value is resolved once per test module. - """ - return pytestconfig.getoption("cluster_file") - - -@pytest.fixture(scope="module") -def training_config_file(pytestconfig): - """ - Retrieve the --config_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the training config JSON file specified via --config_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --config_file=/path/to/training_config.json - - Module scope avoids re-fetching the option across tests in this module. - """ - return pytestconfig.getoption("config_file") - - -# Importing the cluster and cofig files to script to access node, switch, test config params -@pytest.fixture(scope="module") -def cluster_dict(cluster_file): - """ - Load the entire cluster configuration from the provided JSON file. - - Args: - cluster_file (str): Path to the cluster JSON file. - - Returns: - dict: Parsed JSON representing the cluster (nodes, credentials, etc.). - - Notes: - - Logs the loaded structure for visibility; consider using log.debug if verbose. - """ - with open(cluster_file) as json_file: - cluster_dict = json.load(json_file) - - # Resolve path placeholders like {user-id} in cluster config - cluster_dict = resolve_cluster_config_placeholders(cluster_dict) - log.info("%s", cluster_dict) - return cluster_dict - - -@pytest.fixture(scope="module") -def inference_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - inference_dict = inference_dict_t['config'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - inference_dict = resolve_test_config_placeholders(inference_dict, cluster_dict) - return inference_dict - - -@pytest.fixture(scope="module") -def benchmark_params_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - benchmark_params_dict = inference_dict_t['benchmark_params'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - benchmark_params_dict = resolve_test_config_placeholders(benchmark_params_dict, cluster_dict) - - log.info("%s", benchmark_params_dict) - return benchmark_params_dict - - -def pytest_generate_tests(metafunc): - """ - Dynamically parametrize inference tests based on sequence combinations and concurrency levels - for the Qwen3-235B model. - - Behavior: - - Reads the config file path from pytest's --config_file option. - - Loads the JSON and extracts qwen3-235b model configuration. - - Extracts sequence_combinations (ISL/OSL pairs) and concurrency_levels. - - Creates test cases for each sequence combination × concurrency level. - - Test Matrix: - - Test IDs: "combination_name-concX" (e.g., "balanced-conc16") - - Notes: - - If no config_file is provided, the hook returns without parametrizing. - - Each combination gets a separate test case with clear ID. - """ - config_file = metafunc.config.getoption("config_file") - if not config_file or not os.path.exists(config_file): - log.warning(f'Warning: Missing or invalid config file {config_file}') - return - - with open(config_file) as fp: - cfg = json.load(fp) - - # Extract qwen3-235b model config (now directly under benchmark_params, no single_node nesting) - benchmark_params = cfg.get("benchmark_params", {}) - model_config = benchmark_params.get(MODEL_NAME, {}) - - if not model_config: - log.warning(f'Warning: Model {MODEL_NAME} not found in config') - return - - # Build test parameters: list of (seq_combo_dict, concurrency, test_id) - test_params = [] - - # Check if model uses sequence_combinations or legacy ISL/OSL - seq_combos = model_config.get("sequence_combinations", []) - - if seq_combos: - # New format: multiple combinations per model - for combo in seq_combos: - combo_name = combo.get("name", f"isl{combo['isl']}_osl{combo['osl']}") - - # Check if model has concurrency_levels array or single max_concurrency - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - # Parametrize across concurrency levels - for conc in conc_levels: - test_id = f"{combo_name}-conc{conc}" - test_params.append((combo, conc, test_id)) - else: - # Backward compatibility: use max_concurrency as single value - max_conc = int(model_config.get("max_concurrency", "64")) - test_id = f"{combo_name}" - test_params.append((combo, max_conc, test_id)) - else: - # Legacy format: single ISL/OSL values - isl = model_config.get("input_sequence_length", "1024") - osl = model_config.get("output_sequence_length", "1024") - combo = {"isl": isl, "osl": osl, "name": "default"} - - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - for conc in conc_levels: - test_id = f"conc{conc}" - test_params.append((combo, conc, test_id)) - else: - max_conc = int(model_config.get("max_concurrency", "64")) - test_params.append((combo, max_conc, "default")) - - # Parametrize if test uses these fixtures - if "seq_combo" in metafunc.fixturenames and "concurrency" in metafunc.fixturenames: - if test_params: - combos, concs, ids = zip(*test_params) - metafunc.parametrize("seq_combo,concurrency", list(zip(combos, concs)), ids=ids) - - -@pytest.fixture(scope="module") -def hf_token(inference_dict): - """ - Load the Hugging Face access token from the file path specified in the training config. - - Args: - inference_dict (dict): Training configuration dict that includes: - - 'hf_token_file': Path to the file containing the HF token. - - Returns: - str: The HF token string read from the file. - - Behavior: - - Reads the token from inference_dict['hf_token_file'] (already resolved for placeholders). - - Strips the trailing newline from the token. - """ - hf_token_file = inference_dict['hf_token_file'] - try: - with open(hf_token_file, 'r') as fp: - hf_token = fp.read().rstrip("\n") - except FileNotFoundError: - log.error(f"Error: The file '{hf_token_file}' was not found.") - raise - except Exception as e: - log.error(f"An error occurred: {e}") - raise - return hf_token - - -@pytest.fixture(scope="module") -def s_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (server). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - s_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return s_phdl - - -@pytest.fixture(scope="module") -def c_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (client). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - c_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return c_phdl - - -def test_cleanup_stale_containers(s_phdl, inference_dict): - """ - Pytest: Clean up potentially stale Docker containers and volumes before tests. - - Args: - s_phdl: Parallel SSH/process handle used by docker_lib to run commands on nodes. - inference_dict (dict): Training configuration dict that includes: - - 'container_name': Name of the container to be killed if running. - - Behavior: - - Kills the specific container identified by inference_dict['container_name']. - - Deletes all containers and volumes on the target nodes (broad cleanup). - - Notes: - - This performs a broad cleanup via delete_all_containers_and_volumes; ensure the - test environment is isolated so this doesn't remove unrelated containers/volumes. - - Consider narrowing cleanup scope if other workloads may be present on the hosts. - """ - - container_name = inference_dict['container_name'] - docker_lib.kill_docker_container(s_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(s_phdl) - - -def test_launch_inference_containers(s_phdl, inference_dict, benchmark_params_dict): - """ - Launch vLLM inference containers on all nodes. - - Note: Container image can be model-specific or use global default. - """ - - log.info(f'Testcase launch vLLM containers for {MODEL_NAME}') - globals.error_list = [] - container_name = inference_dict['container_name'] - - # Get model-specific container image or use global default - container_image = benchmark_params_dict.get(MODEL_NAME, {}).get( - 'container_image', inference_dict['container_image'] - ) - - # Launch the containers .. - docker_lib.launch_docker_container( - s_phdl, - container_name, - container_image, - inference_dict['container_config']['device_list'], - inference_dict['container_config']['volume_dict'], - inference_dict['container_config']['env_dict'], - shm_size='16G', - timeout=60 * 20, - ) - # ADD verifications .. - time.sleep(30) - log.info('Verify if the containers have been launched properly') - out_dict = s_phdl.exec('docker ps') - for node in out_dict.keys(): - if not re.search(f'{container_name}', out_dict[node], re.I): - fail_test(f'Failed to launch container on node {node}') - update_test_result() - - -def test_vllm_inference(c_phdl, s_phdl, inference_dict, benchmark_params_dict, hf_token, seq_combo, concurrency): - """ - Test vLLM inference for Qwen3-235B with specific sequence combination and concurrency level. - - This test is parametrized via pytest_generate_tests to run once per: - - Sequence combination (ISL/OSL pair) defined in model's sequence_combinations - - Concurrency level defined in model's concurrency_levels - - The factory will automatically create the correct VllmJob instance with - the model-specific container image and parameters. - - Args: - seq_combo: Dict with 'isl', 'osl', 'name' keys for this test iteration - concurrency: Integer concurrency level for this test iteration - """ - # Since this is a per-GPU-type config file (mi355x), gpu_type is implicit - gpu_type = "mi355x" - - log.info( - f"Starting inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']} (ISL={seq_combo['isl']}, OSL={seq_combo['osl']}), concurrency: {concurrency}" - ) - globals.error_list = [] - - # Override ISL/OSL and concurrency in benchmark_params for this specific test iteration - # Config is now fully flattened, so access directly under benchmark_params - model_params = benchmark_params_dict[MODEL_NAME] - model_params['input_sequence_length'] = seq_combo['isl'] - model_params['output_sequence_length'] = seq_combo['osl'] - model_params['max_concurrency'] = str(concurrency) - - # Calculate num_prompts based on OSL (matching recipe logic) - osl = int(seq_combo['osl']) - if osl == 8192: - model_params['num_prompts'] = str(concurrency * 20) - else: - model_params['num_prompts'] = str(concurrency * 50) - - # Create VllmJob instance - vllm_job = VllmJob( - c_phdl=c_phdl, - s_phdl=s_phdl, - model_name=MODEL_NAME, - inference_config_dict=inference_dict, - benchmark_params_dict=benchmark_params_dict, - hf_token=hf_token, - gpu_type=gpu_type, - distributed_inference=False, - ) - - # Stop any existing server process for clean state - vllm_job.stop_server() - - # Build and start server with current test parameters - vllm_job.build_server_inference_job_cmd() - vllm_job.start_inference_server_job() - - # Run benchmark client - vllm_job.start_inference_client_job() - res_dict = vllm_job.poll_for_inference_completion() - res_index = (MODEL_NAME, gpu_type, seq_combo['isl'], seq_combo['osl'], seq_combo['name'], concurrency) - inf_res_dict[res_index] = res_dict - vllm_job.verify_inference_results() - update_test_result() - - log.info( - f"Completed inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']}, concurrency: {concurrency}" - ) - - -def test_print_results_table(): - globals.error_list = [] - pprint(inf_res_dict, depth=3) - rows = [] - headers = [ - "Model", - "GPU", - "ISL", - "OSL", - "Policy", - "Concurrency", - "Host", - "Req/s", - "Total tok/s", - "Mean TTFT (ms)", - "Mean TPOT (ms)", - "P99 ITL (ms)", - ] - - for (model, gpu, isl, osl, policy, concurrency), entry in inf_res_dict.items(): - for host, m in entry["results"].items(): - rows.append( - [ - model, - gpu, - isl, - osl, - policy, - concurrency, - host, - m["successful_requests"], - m["total_throughput_per_sec"], - m["mean_ttft_ms"], - m["mean_tpot_ms"], - m["p99_itl_ms"], - ] - ) - - log.info(tabulate(rows, headers=headers, tablefmt="github")) - update_test_result() diff --git a/cvs/tests/inference/vllm/vllm_qwen3_80b_single.py b/cvs/tests/inference/vllm/vllm_qwen3_80b_single.py deleted file mode 100644 index 0c9e68170..000000000 --- a/cvs/tests/inference/vllm/vllm_qwen3_80b_single.py +++ /dev/null @@ -1,480 +0,0 @@ -''' -Copyright 2025 Advanced Micro Devices, Inc. -All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. -The year included in the foregoing notice is the year of creation of the work. -All code contained here is Property of Advanced Micro Devices, Inc. -''' - -import pytest - -import re -import os -import time -import json -from pprint import pprint -from tabulate import tabulate - -from cvs.lib.parallel_ssh_lib import * -from cvs.lib.utils_lib import * -from cvs.lib import docker_lib -from cvs.lib.inference.vllm import VllmJob -from cvs.lib import globals - -log = globals.log - -# Model name for this test suite -MODEL_NAME = "qwen3-80b" - -inf_res_dict = {} - - -# Importing additional cmd line args to script .. -@pytest.fixture(scope="module") -def cluster_file(pytestconfig): - """ - Retrieve the --cluster_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the cluster JSON file specified via --cluster_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --cluster_file=/path/to/cluster.json - - Use module scope so the value is resolved once per test module. - """ - return pytestconfig.getoption("cluster_file") - - -@pytest.fixture(scope="module") -def training_config_file(pytestconfig): - """ - Retrieve the --config_file CLI option provided to pytest. - - Args: - pytestconfig: Built-in pytest fixture exposing command-line options. - - Returns: - str: Path to the training config JSON file specified via --config_file. - - Notes: - - Ensure your pytest.ini or CLI includes: --config_file=/path/to/training_config.json - - Module scope avoids re-fetching the option across tests in this module. - """ - return pytestconfig.getoption("config_file") - - -# Importing the cluster and cofig files to script to access node, switch, test config params -@pytest.fixture(scope="module") -def cluster_dict(cluster_file): - """ - Load the entire cluster configuration from the provided JSON file. - - Args: - cluster_file (str): Path to the cluster JSON file. - - Returns: - dict: Parsed JSON representing the cluster (nodes, credentials, etc.). - - Notes: - - Logs the loaded structure for visibility; consider using log.debug if verbose. - """ - with open(cluster_file) as json_file: - cluster_dict = json.load(json_file) - - # Resolve path placeholders like {user-id} in cluster config - cluster_dict = resolve_cluster_config_placeholders(cluster_dict) - log.info("%s", cluster_dict) - return cluster_dict - - -@pytest.fixture(scope="module") -def inference_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - inference_dict = inference_dict_t['config'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - inference_dict = resolve_test_config_placeholders(inference_dict, cluster_dict) - return inference_dict - - -@pytest.fixture(scope="module") -def benchmark_params_dict(training_config_file, cluster_dict): - with open(training_config_file) as json_file: - inference_dict_t = json.load(json_file) - benchmark_params_dict = inference_dict_t['benchmark_params'] - - # Resolve path placeholders like {user-id}, {home-mount-dir}, etc. - benchmark_params_dict = resolve_test_config_placeholders(benchmark_params_dict, cluster_dict) - - log.info("%s", benchmark_params_dict) - return benchmark_params_dict - - -def pytest_generate_tests(metafunc): - """ - Dynamically parametrize inference tests based on sequence combinations and concurrency levels - for the Qwen3-80B model. - - Behavior: - - Reads the config file path from pytest's --config_file option. - - Loads the JSON and extracts qwen3-80b model configuration. - - Extracts sequence_combinations (ISL/OSL pairs) and concurrency_levels. - - Creates test cases for each sequence combination × concurrency level. - - Test Matrix: - - Test IDs: "combination_name-concX" (e.g., "balanced-conc16") - - Notes: - - If no config_file is provided, the hook returns without parametrizing. - - Each combination gets a separate test case with clear ID. - """ - config_file = metafunc.config.getoption("config_file") - if not config_file or not os.path.exists(config_file): - log.warning(f'Warning: Missing or invalid config file {config_file}') - return - - with open(config_file) as fp: - cfg = json.load(fp) - - # Extract qwen3-80b model config (now directly under benchmark_params, no single_node nesting) - benchmark_params = cfg.get("benchmark_params", {}) - model_config = benchmark_params.get(MODEL_NAME, {}) - - if not model_config: - log.warning(f'Warning: Model {MODEL_NAME} not found in config') - return - - # Build test parameters: list of (seq_combo_dict, concurrency, test_id) - test_params = [] - - # Check if model uses sequence_combinations or legacy ISL/OSL - seq_combos = model_config.get("sequence_combinations", []) - - if seq_combos: - # New format: multiple combinations per model - for combo in seq_combos: - combo_name = combo.get("name", f"isl{combo['isl']}_osl{combo['osl']}") - - # Check if model has concurrency_levels array or single max_concurrency - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - # Parametrize across concurrency levels - for conc in conc_levels: - test_id = f"{combo_name}-conc{conc}" - test_params.append((combo, conc, test_id)) - else: - # Backward compatibility: use max_concurrency as single value - max_conc = int(model_config.get("max_concurrency", "64")) - test_id = f"{combo_name}" - test_params.append((combo, max_conc, test_id)) - else: - # Legacy format: single ISL/OSL values - isl = model_config.get("input_sequence_length", "1024") - osl = model_config.get("output_sequence_length", "1024") - combo = {"isl": isl, "osl": osl, "name": "default"} - - conc_levels = model_config.get("concurrency_levels", []) - if conc_levels: - for conc in conc_levels: - test_id = f"conc{conc}" - test_params.append((combo, conc, test_id)) - else: - max_conc = int(model_config.get("max_concurrency", "64")) - test_params.append((combo, max_conc, "default")) - - # Parametrize if test uses these fixtures - if "seq_combo" in metafunc.fixturenames and "concurrency" in metafunc.fixturenames: - if test_params: - combos, concs, ids = zip(*test_params) - metafunc.parametrize("seq_combo,concurrency", list(zip(combos, concs)), ids=ids) - - -@pytest.fixture(scope="module") -def hf_token(inference_dict): - """ - Load the Hugging Face access token from the file path specified in the training config. - - Args: - inference_dict (dict): Training configuration dict that includes: - - 'hf_token_file': Path to the file containing the HF token. - - Returns: - str: The HF token string read from the file. - - Behavior: - - Reads the token from inference_dict['hf_token_file'] (already resolved for placeholders). - - Strips the trailing newline from the token. - """ - hf_token_file = inference_dict['hf_token_file'] - try: - with open(hf_token_file, 'r') as fp: - hf_token = fp.read().rstrip("\n") - except FileNotFoundError: - log.error(f"Error: The file '{hf_token_file}' was not found.") - raise - except Exception as e: - log.error(f"An error occurred: {e}") - raise - return hf_token - - -@pytest.fixture(scope="module") -def s_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (server). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - s_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return s_phdl - - -@pytest.fixture(scope="module") -def c_phdl(cluster_dict): - """ - Create and return a parallel SSH handle for all cluster nodes (client). - - Args: - cluster_dict (dict): Cluster configuration loaded by another fixture. Expected keys: - - 'node_dict': dict of node_name -> node_details (used to derive the node list) - - 'username': SSH username for connecting to nodes - - 'priv_key_file': path to the SSH private key file - - Returns: - Pssh: An initialized Pssh handle for issuing commands across all nodes. - - Behavior: - - Prints the full cluster_dict for quick debugging (consider switching to log.debug to reduce noise). - - Collects all node names from cluster_dict['node_dict'] and constructs a Pssh handle. - - Notes: - - This fixture has module scope, so a single connection handle is reused for all tests in the module. - """ - log.info("%s", cluster_dict) - env_vars = cluster_dict.get("env_vars") - node_list = list(cluster_dict['node_dict'].keys()) - c_phdl = Pssh(log, node_list, user=cluster_dict['username'], pkey=cluster_dict['priv_key_file'], env_vars=env_vars) - return c_phdl - - -@pytest.fixture(scope="module", autouse=True) -def cleanup_on_exit(s_phdl, inference_dict): - """ - Automatically clean up containers after all tests in the module complete. - - This fixture runs automatically (autouse=True) and ensures cleanup happens - even if tests fail, providing proper test isolation. - - Args: - s_phdl: Parallel SSH handle for server nodes - inference_dict: Inference configuration containing container_name - - Yields: - None (all tests run between yield statement and cleanup) - - Behavior: - - Runs setup code before yield (currently none) - - Yields control to run all module tests - - After all tests complete (success or failure), kills container and cleans up volumes - """ - # Setup (before tests) - nothing needed currently - yield - # Teardown (after all tests, even on failure) - try: - container_name = inference_dict['container_name'] - log.info(f"Cleaning up container {container_name} after test module completion") - docker_lib.kill_docker_container(s_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(s_phdl) - except Exception as e: - log.warning(f"Cleanup failed (non-critical): {e}") - - -def test_cleanup_stale_containers(s_phdl, inference_dict): - """ - Pytest: Clean up potentially stale Docker containers and volumes before tests. - - Args: - s_phdl: Parallel SSH/process handle used by docker_lib to run commands on nodes. - inference_dict (dict): Training configuration dict that includes: - - 'container_name': Name of the container to be killed if running. - - Behavior: - - Kills the specific container identified by inference_dict['container_name']. - - Deletes all containers and volumes on the target nodes (broad cleanup). - - Notes: - - This performs a broad cleanup via delete_all_containers_and_volumes; ensure the - test environment is isolated so this doesn't remove unrelated containers/volumes. - - Consider narrowing cleanup scope if other workloads may be present on the hosts. - """ - - container_name = inference_dict['container_name'] - docker_lib.kill_docker_container(s_phdl, container_name) - docker_lib.delete_all_containers_and_volumes(s_phdl) - - -def test_launch_inference_containers(s_phdl, inference_dict, benchmark_params_dict): - """ - Launch vLLM inference containers on all nodes. - - Note: Container image can be model-specific or use global default. - """ - - log.info(f'Testcase launch vLLM containers for {MODEL_NAME}') - globals.error_list = [] - container_name = inference_dict['container_name'] - - # Get model-specific container image or use global default - container_image = benchmark_params_dict.get(MODEL_NAME, {}).get( - 'container_image', inference_dict['container_image'] - ) - - # Launch the containers .. - docker_lib.launch_docker_container( - s_phdl, - container_name, - container_image, - inference_dict['container_config']['device_list'], - inference_dict['container_config']['volume_dict'], - inference_dict['container_config']['env_dict'], - shm_size='16G', - timeout=60 * 20, - ) - # ADD verifications .. - time.sleep(30) - log.info('Verify if the containers have been launched properly') - out_dict = s_phdl.exec('docker ps') - for node in out_dict.keys(): - if not re.search(f'{container_name}', out_dict[node], re.I): - fail_test(f'Failed to launch container on node {node}') - update_test_result() - - -def test_vllm_inference(c_phdl, s_phdl, inference_dict, benchmark_params_dict, hf_token, seq_combo, concurrency): - """ - Test vLLM inference for Qwen3-80B with specific sequence combination and concurrency level. - - This test is parametrized via pytest_generate_tests to run once per: - - Sequence combination (ISL/OSL pair) defined in model's sequence_combinations - - Concurrency level defined in model's concurrency_levels - - The vllm_server fixture provides a running server that is reused across all iterations. - - Args: - vllm_server: VllmJob instance with running server (from fixture) - seq_combo: Dict with 'isl', 'osl', 'name' keys for this test iteration - concurrency: Integer concurrency level for this test iteration - """ - gpu_type = "mi355x" - - log.info( - f"Starting inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']} (ISL={seq_combo['isl']}, OSL={seq_combo['osl']}), concurrency: {concurrency}" - ) - globals.error_list = [] - - # Override ISL/OSL and concurrency in benchmark_params for this specific test iteration - model_params = benchmark_params_dict[MODEL_NAME] - model_params['input_sequence_length'] = seq_combo['isl'] - model_params['output_sequence_length'] = seq_combo['osl'] - model_params['max_concurrency'] = str(concurrency) - - # Calculate num_prompts based on OSL (matching recipe logic) - osl = int(seq_combo['osl']) - if osl == 8192: - model_params['num_prompts'] = str(concurrency * 20) - else: - model_params['num_prompts'] = str(concurrency * 50) - - # Create VllmJob instance - vllm_job = VllmJob( - c_phdl=c_phdl, - s_phdl=s_phdl, - model_name=MODEL_NAME, - inference_config_dict=inference_dict, - benchmark_params_dict=benchmark_params_dict, - hf_token=hf_token, - gpu_type=gpu_type, - distributed_inference=False, - server_launch_poll_count=30, - ) - - # Stop any existing server process for clean state - vllm_job.stop_server() - - # Build and start server with current test parameters - vllm_job.build_server_inference_job_cmd() - vllm_job.start_inference_server_job() - - # Run benchmark client - vllm_job.start_inference_client_job() - res_dict = vllm_job.poll_for_inference_completion() - res_index = (MODEL_NAME, gpu_type, seq_combo['isl'], seq_combo['osl'], seq_combo['name'], concurrency) - inf_res_dict[res_index] = res_dict - vllm_job.verify_inference_results() - update_test_result() - - log.info( - f"Completed inference test for model: {MODEL_NAME}, GPU: {gpu_type}, combination: {seq_combo['name']}, concurrency: {concurrency}" - ) - - -def test_print_results_table(): - globals.error_list = [] - pprint(inf_res_dict, depth=3) - rows = [] - headers = [ - "Model", - "GPU", - "ISL", - "OSL", - "Policy", - "Concurrency", - "Host", - "Req/s", - "Total tok/s", - "Mean TTFT (ms)", - "Mean TPOT (ms)", - "P99 ITL (ms)", - ] - - for (model, gpu, isl, osl, policy, concurrency), entry in inf_res_dict.items(): - for host, m in entry["results"].items(): - rows.append( - [ - model, - gpu, - isl, - osl, - policy, - concurrency, - host, - m["successful_requests"], - m["total_throughput_per_sec"], - m["mean_ttft_ms"], - m["mean_tpot_ms"], - m["p99_itl_ms"], - ] - ) - - log.info(tabulate(rows, headers=headers, tablefmt="github")) - update_test_result() diff --git a/cvs/tests/preflight/preflight_checks.py b/cvs/tests/preflight/preflight_checks.py index 8ae5e7fbb..72a34f234 100644 --- a/cvs/tests/preflight/preflight_checks.py +++ b/cvs/tests/preflight/preflight_checks.py @@ -15,6 +15,7 @@ from cvs.lib.preflight.ifoe_l2_connectivity import IfoeL2ConnectivityCheck from cvs.lib.preflight.scaleup_fabric import NodeHealthCheck from cvs.lib.preflight.transferbench_smoke import TransferBenchSmokeCheck +from cvs.lib.preflight.node_smoke import NodeSmokeCheck # RdmaConnectivityCheck not used - using legacy function temporarily from cvs.lib.preflight.report import PreflightReportGenerator @@ -679,6 +680,63 @@ def test_gid_consistency(phdl, config_dict): preflight_update_test_result() +def test_node_smoke(phdl, config_dict): + """ + Run Primus ``node_smoke`` checks on each reachable node via primus-cli. + + Opt-in via ``node_smoke.connectivity_mode`` in the preflight config (default + ``skip``). Uses parallel SSH — no Slurm required. + + Optional Tier 2 perf sanity (``node_smoke.tier2_perf``) enables + ``--tier2-perf``: large GEMM TFLOPS floor, HBM D2D bandwidth, and local + multi-GPU RCCL all-reduce thresholds (``gemm_tflops_min``, ``hbm_gbs_min``, + ``rccl_gbs_min``, etc.). + + Nodes that fail are reported but are **not** pruned from ``phdl``. + """ + global preflight_results + + if not phdl.reachable_hosts: + log.warning("Primus node_smoke skipped: no reachable hosts remain after earlier preflight pruning") + preflight_results['node_smoke'] = { + 'mode': 'skip', + 'skipped': True, + 'message': 'No reachable nodes available for Primus node_smoke', + 'node_results': {}, + } + preflight_update_test_result() + return + + node_list = list(phdl.reachable_hosts) + log.info("Running Primus node_smoke on %d reachable host(s)", len(node_list)) + + checker = NodeSmokeCheck(phdl, node_list, config_dict) + results = checker.run() + preflight_results['node_smoke'] = results + + if results.get('skipped'): + log.info("Primus node_smoke: %s", results.get('message', 'skipped')) + preflight_update_test_result() + return + + failed_nodes = results.get('failed_nodes') or [] + unknown_nodes = results.get('unknown_nodes') or [] + total = results.get('total_nodes', 0) + passing = len(results.get('passing_nodes') or []) + + if failed_nodes or unknown_nodes: + log.warning( + "Primus node_smoke FAIL on %d/%d node(s): %s", + len(failed_nodes) + len(unknown_nodes), + total, + ", ".join(failed_nodes + unknown_nodes), + ) + else: + log.info("Primus node_smoke PASS on %d/%d nodes", passing, total) + + preflight_update_test_result() + + def _l2ping_config(config_dict): """Return the customer-facing l2ping configuration.""" config = _ifoe_config(config_dict).get('l2ping', {}) @@ -1240,6 +1298,7 @@ def test_generate_preflight_report(phdl, config_dict, request): 'gid_consistency', 'rocm_versions', 'interface_names', + 'node_smoke', 'ifoe_l2_connectivity', 'transferbench_smoke', 'rdma_connectivity', diff --git a/cvs/tests/training/jaxmaxtext/README.md b/cvs/tests/training/jaxmaxtext/README.md new file mode 100644 index 000000000..b0f8a17b8 --- /dev/null +++ b/cvs/tests/training/jaxmaxtext/README.md @@ -0,0 +1,177 @@ +# JAX MaxText Training Suite (single-node and distributed) + +Cluster validation suite that runs **JAX MaxText** pre-training on AMD Instinct +GPUs (single-node or multi-node) and gates the run on performance and +correctness metrics with a PASS/FAIL HTML report. + +## Overview + +The suite drives a MaxText training job inside a container on one or more +cluster nodes, then parses the training log to produce metrics and verdicts. It +provides: + +1. **Two suites** - `jaxmaxtext_single` (single node) and + `jaxmaxtext_distributed` (multi-node, adds RDMA/NIC setup). +2. **Parameter sweeps** - one full training run per enabled sweep (e.g. BF16 and + FP8), each with its own result rows in the report. +3. **Metric gating** - per-sweep, per-metric PASS/FAIL against a threshold file + (throughput, TFLOP/s, step-time, loss, scaling efficiency, convergence, ...). +4. **Loss curve** - a per-sweep training-loss PNG plus a decreasing-trend check. +5. **Training-log error scanning** - configurable regex signatures (NCCL, GPU HW, + OOM, segfault, ...) fail a run early with a clear reason. +6. **HTML report + console summary** - per-test rows, a consolidated metric + results page, per-sweep loss curves, and an aggregated failure summary. + +The mode (single vs distributed) is reflected in the suite file name, the metric +results HTML title, and the loss-curve titles/artifacts. + +## Quick Start + +Single-node run: + +```bash +cvs run jaxmaxtext_single \ + --cluster_file ./p3_1n_cluster.json \ + --config_file cvs/input/config_file/training/jaxmaxtext/mi300x_jaxmaxtext_llama-3.3-70b_single.json \ + --html ./logs/jaxmaxtext_single.html --self-contained-html -vvv +``` + +Distributed (multi-node) run: + +```bash +cvs run jaxmaxtext_distributed \ + --cluster_file ./p3_2n_cluster.json \ + --config_file cvs/input/config_file/training/jaxmaxtext/mi325x_jaxmaxtext_llama-3.3-70b_distributed.json \ + --html ./logs/jaxmaxtext_distributed.html --self-contained-html -vvv +``` + +- `--cluster_file` - JSON describing the node(s); the first node in `node_dict` + is used as the JAX coordinator when `jax_distributed.coordinator_ip` is `auto`. +- `--config_file` - one of the config files in + `cvs/input/config_file/training/jaxmaxtext/` (see that folder's README for the + variable-by-variable reference and what to change for your cluster). +- `--html` / `--self-contained-html` - write the report; a `_html/` bundle + dir alongside it holds per-test logs, the metric results page, and loss-curve + PNGs. + +> Use a **single-node** config with `jaxmaxtext_single` and a **distributed** +> config with `jaxmaxtext_distributed`. The config's `training.distributed` flag +> must match the suite. + +## The two suites + +| Suite (`cvs run `) | File | Distributed stages | Use with | +|---|---|---|---| +| `jaxmaxtext_single` | `jaxmaxtext_single.py` | none | single-node config (`distributed: false`) | +| `jaxmaxtext_distributed` | `jaxmaxtext_distributed.py` | `test_setup_rdma` | multi-node config (`distributed: true`) | + +Both suites share their implementations from `_common.py`; sweep parametrization +and all fixtures/hooks live in `conftest.py`. `_common.py` and `conftest.py` are +helpers, not runnable suites. + +## Test lifecycle (report rows) + +Tests run in this pinned order. `[sweep]` = one row per enabled sweep; +`[sweep-metric]` = one row per metric per sweep. + +| Order | Test | Runs on | Purpose | +|---|---|---|---| +| 1 | `test_launch_container` | once | Launch and verify the container | +| 2 | `test_setup_rdma` | distributed only | Copy RDMA lib into container (thor2 NIC) and verify `ibv_devinfo` | +| 3 | `test_setup_tokenizer` | once | Download the HF tokenizer | +| 4 | `test_training_run[sweep]` | per sweep | Build cmd, train, poll, parse results | +| 5 | `test_metric[sweep-metric]` | per sweep x metric | Threshold PASS/FAIL per metric | +| 6 | `test_loss_curve[sweep]` | per sweep | Render loss PNG; gate on downward trend | +| 7 | `test_print_results_table` | once | Console tables + metric results HTML + failure summary | +| 8 | `test_teardown` | once | Tear the container down | + +On a training failure/timeout, lingering ranks are killed (`stop_training`) so +the next sweep does not launch on top of them. + +A training failure is isolated to that sweep's `test_training_run` row; other +sweeps still run. When a sweep's training does not complete, its downstream +`test_metric`/`test_loss_curve` rows are skipped. + +## Sweeps + +A **sweep** is one full training run with per-run MaxText overrides. Sweeps are +declared in the config under `training.sweeps` and selected with +`training.enabled_sweep_list`. The sweep `name` is also the **threshold cell +key**. For now precision is the swept dimension (BF16, FP8). + +Each sweep gets a compact, unique **label** derived from its name - +`PRECISION[-SL][-B]`, e.g. `BF16-SL8192-B3`. The label appears in +every parametrized row: `test_training_run[BF16-SL8192-B3]`, +`test_metric[BF16-SL8192-B3-tflops_per_sec_per_gpu]`, +`test_loss_curve[BF16-SL8192-B3]`, and in the metric results/loss-curve reports. + +## Metrics and PASS/FAIL + +Each `test_metric[sweep-metric]` compares the parsed metric against its threshold +spec in the sweep's cell of the threshold file and reports one of: + +| Status | Meaning | +|---|---| +| PASS | value satisfies the threshold | +| FAIL | value violates the threshold (row is red; also aggregated in the summary) | +| N/A | metric was not produced this run (feature disabled / rampup) - not a failure | +| RECORD | no threshold, or `enforce_thresholds` is false - value logged, not gated | + +Metrics surfaced (namespace `training.*`): `tflops_per_sec_per_gpu`, +`tokens_per_sec_per_gpu`, `tokens_per_sec_total`, `scaling_efficiency_pct`, +`step_time_seconds`, `step_time_mean_ms`, `step_time_p50_ms`, `step_time_p95_ms`, +`final_loss`, `loss_decreased`, `eval_loss`, `steps_to_target`, +`time_to_target_seconds`. + +Gating is threshold-driven and requires `enforce_thresholds: true` in the config. +A threshold entry with `"kind": "info"` always passes (record-only). See the +input-config README for threshold kinds and defaults. + +## Reports and logs + +- **Results table** - one row per test; metric rows show PASS/FAIL from the + threshold check. +- **Full Log** - each test row links to its own captured log. +- **Metric Results** - every `test_metric` row also links to a single shared + `metric_results.html` (Sweep | Metric | Expected | Actual | Unit | Status), + titled with the mode (single/distributed). +- **Loss Curve** - each `test_loss_curve` row links to a per-sweep PNG. +- **Console summary** - `test_print_results_table` prints per-sweep tables and, + via `globals.error_list` + `update_test_result()`, an aggregated list of all + failed `(sweep, metric)` checks in the pytest final summary. + +## Training-log error detection + +During polling, each node's `training.log` is scanned for the regexes in +`training.error_patterns` (config-driven; falls back to built-in defaults). +Defaults cover NCCL, GPU HW faults, assertion/JAX stack traces, ROCm init +errors, Python fatal errors, TF coordination errors, `RESOURCE_EXHAUSTED`/OOM, +and segfault signatures. A match fails that sweep's `test_training_run` with the +matched signature name and the last part of the log. Add/remove signatures in +the config as you encounter new ones. + +## Config and threshold files + +Located in `cvs/input/config_file/training/jaxmaxtext/` (each config has a +sibling threshold file named by its `threshold_json` field): + +| Config | Threshold | Arch / mode | +|---|---|---| +| `mi300x_jaxmaxtext_llama-3.3-70b_single.json` | `..._single_threshold.json` | MI300X, single-node | +| `mi300x_jaxmaxtext_llama-3.3-70b_distributed.json` | `..._distributed_threshold.json` | MI300X, distributed | +| `mi325x_jaxmaxtext_llama-3.3-70b_distributed.json` | `..._distributed_threshold.json` | MI325X, distributed | + +(Add analogous configs for other archs, e.g. MI355X, as needed.) + +See `cvs/input/config_file/training/jaxmaxtext/README.md` for the full variable +reference and the values you must change for your cluster and container image. + +## Prerequisites + +- Passwordless SSH from the control host to each cluster node (key in the + cluster file), and Docker available on the nodes. +- A container image bundling MaxText/JAX for ROCm (config `container.image`). +- A Hugging Face token file at `paths.hf_token_file` (used to fetch the + tokenizer). The tokenizer download requires network access on the nodes. +- A shared filesystem path (`paths.shared_fs`) reachable from all nodes for + distributed runs (models cache and logs). diff --git a/cvs/tests/training/jaxmaxtext/__init__.py b/cvs/tests/training/jaxmaxtext/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/cvs/tests/training/jaxmaxtext/_common.py b/cvs/tests/training/jaxmaxtext/_common.py new file mode 100644 index 000000000..5b621fedd --- /dev/null +++ b/cvs/tests/training/jaxmaxtext/_common.py @@ -0,0 +1,495 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +Shared implementations for the JAX MaxText training suites. This is NOT a +runnable suite (leading underscore -> excluded by `cvs list`/`cvs run`); the two +suite files import these helpers and wrap each as an explicit `test_*` method: + + - jaxmaxtext_single.py (single-node: no RDMA stage) + - jaxmaxtext_distributed.py (adds the test_setup_rdma stage) + +Kept deliberately simple: plain shared functions + a couple of small helpers, +no framework-y generalization. The single vs distributed mode is recorded in +`training_res_dict["mode"]` and reflected in the console tables, the metric +results HTML title, and the loss-curve title/artifact. +''' + +import html as _html +import json +import re +import shlex +import time +import uuid as _uuid +from pathlib import Path as _Path + +import pytest +from tabulate import tabulate + +from cvs.lib import globals +from cvs.lib.training.jaxmaxtext.jaxmaxtext_training_lib import MaxTextTrainingJob +from cvs.lib.training.jaxmaxtext.utils.maxtext_parsing import ( + TRAINING_METRICS, + TRAINING_METRIC_UNITS, + compute_scaling_efficiency, + compute_convergence, + sample_loss_curve, + evaluate_loss_decreasing, +) +from cvs.lib.training.jaxmaxtext.utils.loss_curve import render_loss_curve_png +from cvs.lib.utils.verdict import evaluate_all, ThresholdViolation +from cvs.lib.utils_lib import fail_test, update_test_result + +log = globals.log + +_STATUS_COLORS = { + "PASS": "#2e7d32", + "FAIL": "#c62828", + "N/A": "#f9a825", + "RECORD": "#555555", +} + + +# ---------- small helpers ---------- + + +def _sweep_label(name): + """Compact, unique-per-sweep id used in every parametrized test row and in + the reports: PRECISION[-SL][-B], e.g. "BF16-SL4096-B3". + + The full sweep name still drives results/threshold lookups; this is only the + display label. Falls back to a sanitized full name when PRECISION is absent. + """ + name = name or "" + + def _tok(key): + m = re.search(rf"{key}=([^,]+)", name) + return m.group(1).strip() if m else None + + precision = _tok("PRECISION") + seqlen = _tok("SEQLEN") + batch = _tok("BATCH") + + parts = [] + if precision: + parts.append(precision) + if seqlen: + parts.append(f"SL{seqlen}") + if batch: + parts.append(f"B{batch}") + if parts: + return "-".join(parts) + return (re.sub(r"[^A-Za-z0-9]+", "_", name).strip("_")) or "default" + + +def _enabled_sweep_names(config_file): + """Read sweep names to run from the raw config (collection time, no fixtures). + + Honors training.enabled_sweep_list (subset selector); falls back to every + declared sweep, or a single implicit "default" when none are declared. + """ + try: + with open(config_file) as fp: + raw = json.load(fp) + except Exception: + return ["default"] + training = raw.get("training", {}) + names = [s.get("name") for s in training.get("sweeps", []) if s.get("name")] + if not names: + return ["default"] + enabled = training.get("enabled_sweep_list") or names + return [n for n in enabled if n in names] or names + + +def _find_sweep(variant_config, sweep_name): + for s in variant_config.enabled_sweeps(): + if s.name == sweep_name: + return s + return None + + +def _mode(variant_config): + return "distributed" if variant_config.training.distributed else "single" + + +def _format_expected(spec): + """Human-readable expected-threshold string for the console log + summary file.""" + if not spec: + return "-" + kind = spec.get("kind") + value = spec.get("value") + if kind == "info": + return f"info ({value})" if value is not None else "info" + if kind in ("min", "min_tok_s"): + return f">= {value}" + if kind == "max": + return f"<= {value}" + if kind == "max_ms": + return f"<= {value} ms" + if kind == "within": + return f"{value} +/-{spec.get('tolerance_pct')}%" + if kind == "min_ratio": + return f">= {value} x {spec.get('reference')}" + return str(spec) + + +def _format_value(value): + if value is None: + return "None" + if isinstance(value, float): + return f"{value:.4f}" + return str(value) + + +# ---------- lifecycle stage implementations ---------- +# Plain helpers (no test_ prefix): the suite files wrap each as a `test_*` +# method with a docstring so the suites read as self-documenting. + + +def _precreate_tmp_bind_mounts(orch): + """Create host-side ``/tmp/...`` bind-mount source dirs before launch. + + Docker auto-creates a missing bind-mount source directory owned by root; a + leftover root-owned dir (``docker system prune`` does not remove host bind + dirs) then blocks the next user on a shared GPU node with a permission + error. Creating them here over SSH -- via ``exec_on_host``, which runs on + the cluster host OS as the invoking user -- makes them user-owned instead. + + Only ``/tmp/`` sources are touched so device/system mounts (``/dev/*``, + ``/lib/*``) are never created. + """ + exec_host = getattr(orch, "exec_on_host", None) + if not callable(exec_host): + return + try: + volumes = orch.get_volumes() + except Exception: # noqa: BLE001 - best-effort; docker still auto-creates + return + sources, seen = [], set() + for vol in volumes or []: + src = str(vol).split(":", 1)[0].strip() + if src.startswith("/tmp/") and src not in seen: + seen.add(src) + sources.append(src) + if not sources: + return + quoted = " ".join(shlex.quote(p) for p in sources) + try: + exec_host(f"mkdir -p {quoted}") + except Exception: # noqa: BLE001 - non-fatal; fall back to docker auto-create + pass + + +def launch_container(orch, variant_config, lifecycle, request): + """Stage 1: launch the container. Verify it is running.""" + t = time.monotonic() + _precreate_tmp_bind_mounts(orch) + ok = orch.setup_containers() + lifecycle.record(request.node.nodeid, "container_launch", time.monotonic() - t) + if not ok: + lifecycle.failed = True + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + pytest.fail(f"setup_containers() returned False for {name}") + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + if not orch.verify_containers_running(name): + lifecycle.failed = True + pytest.fail(f"container {name} not running after setup_containers()") + + +def setup_rdma(orch, variant_config, hf_token, lifecycle, request): + """Distributed-only: copy RDMA library into container (thor2 NIC only).""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + if not variant_config.training.distributed: + pytest.skip("single-node: RDMA not needed") + if not variant_config.training.nic_type or "thor" not in variant_config.training.nic_type.lower(): + pytest.skip(f"nic_type={variant_config.training.nic_type}: RDMA lib copy not needed") + t = time.monotonic() + job = MaxTextTrainingJob(orch, variant_config, hf_token) + job.setup_rdma_lib() + lifecycle.record(request.node.nodeid, "rdma_setup", time.monotonic() - t) + + +def setup_tokenizer(orch, variant_config, hf_token, lifecycle, request): + """Download HF tokenizer into models dir.""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + t = time.monotonic() + job = MaxTextTrainingJob(orch, variant_config, hf_token) + job.setup_tokenizer() + lifecycle.record(request.node.nodeid, "tokenizer_setup", time.monotonic() - t) + + +def training_run(orch, variant_config, hf_token, sweep_name, training_res_dict, lifecycle, request): + """Per sweep: build the command, train, poll, parse results. + + Runs once per enabled sweep with that sweep's maxtext overrides. A failure is + isolated to this sweep's row (it does NOT set lifecycle.failed) so the other + sweeps still run and report. + """ + training_res_dict.setdefault("mode", _mode(variant_config)) + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + sweep = _find_sweep(variant_config, sweep_name) + job = MaxTextTrainingJob(orch, variant_config, hf_token, sweep=sweep) + try: + job.setup_training_env() + job.build_training_cmd() + t = time.monotonic() + job.start_training() + job.poll_for_completion() + wall_time = time.monotonic() - t + results = job.parse_results() + # Scan host dmesg over this run's window for GPU/HW/kernel faults. Uses + # fail_test internally (rolled into the aggregated failure summary) and + # is best-effort, so it never raises here. + job.scan_dmesg_for_errors() + except Exception as e: # noqa: BLE001 - isolate the failure to this sweep + log.error("training run failed for sweep '%s': %s", sweep_name, e) + # Reap any lingering ranks so the next sweep does not launch on top of + # them (and so persistent containers are not left with orphan processes). + try: + job.stop_training() + except Exception: # noqa: BLE001 + pass + pytest.fail(f"training run failed for sweep '{sweep_name}': {e}") + + # wall_time is this sweep's measured wall-clock; logged for diagnostics only. + # Convergence is surfaced/asserted via the registered steps_to_target / + # time_to_target_seconds metrics below -- the old ad-hoc + # training.wall_time_seconds / convergence_* keys were never in + # TRAINING_METRICS, so nothing displayed or gated them. + log.info("[training] sweep '%s' wall-clock: %.1fs", sweep_name, wall_time) + + baseline = variant_config.training.scaling_baseline + results["training.scaling_efficiency_pct"] = compute_scaling_efficiency( + results.get("training.tokens_per_sec_total"), + job.num_nodes, + baseline.tokens_per_sec_total, + baseline.num_nodes, + ) + + conv = variant_config.training.convergence + steps_to_target, time_to_target = compute_convergence( + job.step_metrics, + job.eval_metrics, + conv.target_metric, + conv.target_value, + ) + results["training.steps_to_target"] = steps_to_target + results["training.time_to_target_seconds"] = time_to_target + + training_res_dict.setdefault("sweeps", {})[sweep_name] = { + "results": results, + "step_metrics": job.step_metrics, + "eval_metrics": job.eval_metrics, + "num_nodes": job.num_nodes, + } + + +def metric(sweep_name, metric, training_res_dict, variant_config, lifecycle, request): + """One test (row) per (sweep, metric). Threshold-driven PASS/FAIL; logs + `sweep | metric | expected | actual | status` and collects rows for the + single metric-results HTML file (linked from every metric row).""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + rec = training_res_dict.get("sweeps", {}).get(sweep_name) + results = rec.get("results") if rec else None + if not results: + pytest.skip(f"no results for sweep '{sweep_name}' (training did not complete)") + + label = _sweep_label(sweep_name) + full = "training." + metric + value = results.get(full) + unit = TRAINING_METRIC_UNITS.get(metric, "-") + # The sweep name IS the threshold cell key. + spec = (variant_config.thresholds.get(sweep_name) or {}).get(full) + expected = _format_expected(spec) + actual = _format_value(value) + + rows = training_res_dict.setdefault("metric_rows", []) + + def _record(status): + rows.append( + { + "sweep": label, + "metric": metric, + "expected": expected, + "actual": actual, + "unit": unit, + "status": status, + } + ) + + if value is None: + log.info("[metric] %-6s %-24s | expected %-14s | actual None | %s -> N/A", label, metric, expected, unit) + _record("N/A") + pytest.skip(f"{metric}: no value produced this run") + + if spec is None or not variant_config.enforce_thresholds: + log.info( + "[metric] %-6s %-24s | expected %-14s | actual %s | %s -> RECORD", label, metric, expected, actual, unit + ) + _record("RECORD") + return + + try: + evaluate_all(results, {full: spec}) + except ThresholdViolation as e: + log.error( + "[metric] %-6s %-24s | expected %-14s | actual %s | %s -> FAIL", label, metric, expected, actual, unit + ) + _record("FAIL") + training_res_dict.setdefault("metric_failures", []).append( + f"[{label}] {metric}: expected {expected}, actual {actual}" + ) + pytest.fail(str(e)) + else: + log.info("[metric] %-6s %-24s | expected %-14s | actual %s | %s -> PASS", label, metric, expected, actual, unit) + _record("PASS") + + +def loss_curve(sweep_name, training_res_dict, variant_config, lifecycle, request): + """Row 32 (per sweep): sample the training loss, render a PNG, gate on trend.""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + label = _sweep_label(sweep_name) + mode = _mode(variant_config) + rec = training_res_dict.get("sweeps", {}).get(sweep_name) + step_metrics = rec.get("step_metrics") if rec else None + if not step_metrics: + pytest.skip(f"no step metrics for sweep '{sweep_name}' (training did not complete)") + + cfg = variant_config.training.loss_curve + points = sample_loss_curve(step_metrics, cfg.sample_every, cfg.milestone_steps) + verdict = evaluate_loss_decreasing(points, cfg.max_slope) + + mgr = getattr(request.config, "_html_report_manager", None) + if mgr is not None and getattr(mgr, "is_enabled", False): + out_dir = mgr.log_dir + else: + out_dir = _Path(variant_config.paths.log_dir) + png_path = None + try: + _Path(out_dir).mkdir(parents=True, exist_ok=True) + fname = f"loss_curve_{variant_config.model.id}_{mode}_{label}_{str(_uuid.uuid4()).split('-')[-1]}.png" + abs_path = _Path(out_dir) / fname + title = f"Training Loss Curve — {variant_config.model.id} [{mode}/{label}]" + png_path = render_loss_curve_png(points, abs_path, title=title) + except Exception as e: # noqa: BLE001 - plotting must never break the verdict + log.warning("loss curve: could not prepare PNG output (%s)", e) + + if png_path and mgr is not None and getattr(mgr, "is_enabled", False): + try: + rel_path = str(_Path(png_path).relative_to(mgr.htmlpath.parent)) + lifecycle.add_artifact(request.node.nodeid, f"Loss Curve [{mode}/{label}]", rel_path, str(png_path)) + except Exception as e: # noqa: BLE001 + log.warning("loss curve: could not register report link (%s)", e) + + if verdict is not None: + _decreasing, _slope, detail = verdict + log.info("loss curve: %s", detail) + + if verdict is None: + pytest.skip(f"loss curve needs >= 2 sampled points (got {len(points)})") + decreasing, _slope, detail = verdict + if cfg.enforce and not decreasing: + pytest.fail(f"training loss is not decreasing: {detail}") + + +# ---------- reporting ---------- + + +def _write_metric_results_html(training_res_dict, request): + """Write ALL metric verdicts to ONE HTML file in the report bundle dir.""" + metric_rows = training_res_dict.get("metric_rows") or [] + mgr = getattr(request.config, "_html_report_manager", None) + if not metric_rows or mgr is None or not getattr(mgr, "is_enabled", False): + return + mode = training_res_dict.get("mode", "") + try: + out_dir = mgr.log_dir + out_dir.mkdir(parents=True, exist_ok=True) + path = out_dir / "metric_results.html" + body = "" + for r in metric_rows: + color = _STATUS_COLORS.get(r["status"], "#000000") + body += ( + "" + f"{_html.escape(str(r.get('sweep', '-')))}" + f"{_html.escape(str(r['metric']))}" + f"{_html.escape(str(r['expected']))}" + f"{_html.escape(str(r['actual']))}" + f"{_html.escape(str(r['unit']))}" + f"{_html.escape(str(r['status']))}" + "" + ) + title = f"Training Metric Results ({mode})" if mode else "Training Metric Results" + doc = ( + f"{_html.escape(title)}" + f"

{_html.escape(title)}

" + "" + "" + f"{body}
SweepMetricExpectedActualUnitStatus
" + ) + path.write_text(doc, encoding="utf-8") + log.info("wrote metric results HTML: %s", path) + except Exception as e: # noqa: BLE001 - reporting must never break the run + log.warning("could not write metric results HTML: %s", e) + + +def _print_sweep_tables(training_res_dict): + """Log a per-sweep metric table + loss curve to the console.""" + sweeps = training_res_dict.get("sweeps", {}) + mode = training_res_dict.get("mode", "") + if not sweeps: + log.info("no sweep results to print") + return + for sweep_name, rec in sweeps.items(): + results = rec.get("results", {}) + rows = [] + for short, unit in TRAINING_METRICS: + val = results.get("training." + short) + rows.append([short, f"{val:.4f}" if isinstance(val, float) else str(val), unit]) + log.info( + "\n[%s | sweep %s]\n%s", + mode, + sweep_name, + tabulate(rows, headers=["Metric", "Value", "Unit"], tablefmt="github"), + ) + loss_rows = [[s["step"], f"{s['loss']:.6f}"] for s in rec.get("step_metrics", []) if "loss" in s] + if loss_rows: + log.info( + "\nLoss Curve [%s]:\n%s", sweep_name, tabulate(loss_rows, headers=["Step", "Loss"], tablefmt="github") + ) + + +def print_results_table(training_res_dict, request): + """Summarize all sweeps: console tables, single metric-results HTML, and a + consolidated PASS/FAIL summary recorded via globals.error_list for the pytest + final summary.""" + if not training_res_dict.get("sweeps"): + log.info("training_res_dict empty, nothing to print") + return + + _print_sweep_tables(training_res_dict) + _write_metric_results_html(training_res_dict, request) + + failures = training_res_dict.get("metric_failures", []) + globals.error_list = [] + for f in failures: + fail_test(f) + update_test_result() + + +def teardown(orch, lifecycle, request): + """Final stage: explicit container teardown.""" + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + t = time.monotonic() + orch.teardown_containers() + lifecycle.record(request.node.nodeid, "teardown", time.monotonic() - t) + if orch.verify_containers_running(name): + pytest.fail(f"container {name} still running after teardown_containers()") + lifecycle.torn_down = True diff --git a/cvs/tests/training/jaxmaxtext/conftest.py b/cvs/tests/training/jaxmaxtext/conftest.py new file mode 100644 index 000000000..68a15b7d2 --- /dev/null +++ b/cvs/tests/training/jaxmaxtext/conftest.py @@ -0,0 +1,235 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. +''' + +import json +import os + +import pytest + +from cvs.core.orchestrators.factory import OrchestratorConfig, OrchestratorFactory +from cvs.lib import globals +from cvs.lib.training.jaxmaxtext.utils.maxtext_parsing import TRAINING_METRICS +from cvs.lib.training.jaxmaxtext.utils.training_config_loader import ( + load_training_variant, + validate_thresholds_cover_training, +) +from cvs.lib.utils_lib import resolve_cluster_config_placeholders +from cvs.tests.training.jaxmaxtext import _common + +log = globals.log + + +def pytest_generate_tests(metafunc): + """Parametrize per-sweep tests for BOTH suites (single + distributed): + training_run/loss_curve over sweeps, metric over (sweep x TRAINING_METRICS).""" + config_file = metafunc.config.getoption("config_file") + if config_file and os.path.isfile(config_file): + names = _common._enabled_sweep_names(config_file) + else: + names = ["default"] + labels = [_common._sweep_label(n) for n in names] + + if "metric" in metafunc.fixturenames and "sweep_name" in metafunc.fixturenames: + cases, ids = [], [] + for name, label in zip(names, labels): + for short, _unit in TRAINING_METRICS: + cases.append((name, short)) + ids.append(f"{label}-{short}") + metafunc.parametrize("sweep_name,metric", cases, ids=ids) + elif "sweep_name" in metafunc.fixturenames: + metafunc.parametrize("sweep_name", names, ids=labels) + + +def _deep_merge(base, override): + """Recursively merge `override` onto `base` (dicts merged key-wise, scalars/lists replaced).""" + if not (isinstance(base, dict) and isinstance(override, dict)): + return override + out = dict(base) + for k, v in override.items(): + out[k] = _deep_merge(base[k], v) if k in base else v + return out + + +@pytest.fixture(scope="module") +def cluster_dict(pytestconfig): + cluster_file = pytestconfig.getoption("cluster_file") + if not cluster_file: + pytest.fail("--cluster_file is required") + with open(cluster_file) as fp: + d = json.load(fp) + return resolve_cluster_config_placeholders(d) + + +@pytest.fixture(scope="module") +def variant_config(pytestconfig, cluster_dict): + config_file = pytestconfig.getoption("config_file") + if not config_file: + pytest.fail("--config_file is required") + variant = load_training_variant(config_file, cluster_dict) + # Fail fast on a sweep-name/threshold-key mismatch: otherwise metric() would + # silently take the "spec is None" path and emit non-gating RECORD rows, so + # the suite would report green while gating nothing. Raises when + # enforce_thresholds is true; warns otherwise. + validate_thresholds_cover_training( + expected_cells=variant.expected_cells(), + thresholds=variant.thresholds, + enforce_thresholds=variant.enforce_thresholds, + ) + return variant + + +@pytest.fixture(scope="module", autouse=True) +def _guard_suite_matches_config(request, variant_config): + """Fail fast when the suite and the config disagree on distributed mode. + + Both jaxmaxtext_single and jaxmaxtext_distributed share this conftest. Without + this guard a mismatched pairing -- e.g. `cvs run jaxmaxtext_single` with a + distributed config (skips RDMA, still launches multi-node JAX), or + `jaxmaxtext_distributed` with a single-node config -- would start and fail + late with confusing errors. Catch it at setup instead. + """ + mod = (request.module.__name__ or "").rsplit(".", 1)[-1] + distributed = variant_config.training.distributed + if mod.endswith("_single") and distributed: + pytest.fail( + "suite/config mismatch: jaxmaxtext_single requires a single-node config " + "(training.distributed=false), but this config has distributed=true. " + "Run jaxmaxtext_distributed or point at a single-node config." + ) + if mod.endswith("_distributed") and not distributed: + pytest.fail( + "suite/config mismatch: jaxmaxtext_distributed requires a distributed config " + "(training.distributed=true), but this config has distributed=false. " + "Run jaxmaxtext_single or point at a distributed config." + ) + + +class _Lifecycle: + """Cross-test state for the lifecycle-as-tests model. + + `failed` lets a broken stage skip the rest instead of cascading; + `torn_down` lets the explicit teardown test suppress the fixture's + leak-guard finalizer. + """ + + def __init__(self): + self.failed = False + self.torn_down = False + self.report = {} + self.artifacts = {} + + def record(self, nodeid, label, value, unit="s"): + self.report.setdefault(nodeid, []).append((label, value, unit)) + + def add_artifact(self, nodeid, name, rel_path, abs_path): + """Register a per-test report artifact (e.g. loss-curve PNG) for linking.""" + self.artifacts.setdefault(nodeid, []).append((name, rel_path, abs_path)) + + +@pytest.fixture(scope="module") +def lifecycle(): + return _Lifecycle() + + +@pytest.fixture(scope="module") +def orch(cluster_dict, variant_config, lifecycle): + """Construct a ContainerOrchestrator and own ONLY its teardown safety net.""" + container_block = _deep_merge( + cluster_dict.get("container", {}), + variant_config.container.model_dump(), + ) + testsuite_config = { + "orchestrator": "container", + "container": container_block, + } + cfg = OrchestratorConfig.from_configs(cluster_dict, testsuite_config) + o = OrchestratorFactory.create_orchestrator(log, cfg) + yield o + if not lifecycle.torn_down: + log.info("orch fixture leak-guard: tearing down container (explicit teardown did not run)") + o.teardown_containers() + + +@pytest.fixture(scope="module") +def hf_token(variant_config): + path = variant_config.paths.hf_token_file + if not os.path.isfile(path): + pytest.skip(f"hf_token file missing: {path}") + with open(path) as fp: + return fp.read().strip() + + +@pytest.fixture(scope="module") +def training_res_dict(): + return {} + + +def pytest_collection_modifyitems(items): + """Pin the lifecycle order explicitly.""" + rank = { + "test_launch_container": 0, + "test_setup_rdma": 1, + "test_setup_tokenizer": 2, + "test_training_run": 3, + "test_metric": 4, + "test_loss_curve": 5, + "test_print_results_table": 6, + "test_teardown": 7, + } + items.sort(key=lambda it: rank.get(it.originalname or it.name.split("[")[0], 99)) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """Attach THIS test's recorded rows to its HTML report detail panel.""" + outcome = yield + report = outcome.get_result() + if report.when != "call": + return + try: + import pytest_html + except ImportError: + return + + lc = item.funcargs.get("lifecycle") + rows = getattr(lc, "report", {}).get(item.nodeid) if lc else None + artifacts = getattr(lc, "artifacts", {}).get(item.nodeid) if lc else None + + # Each metric row gets an EXTRA "Metric Results" link to the single shared + # metric-results HTML file (written by test_print_results_table), in addition + # to its own per-test "Full Log" link (kept as-is). Two links per metric row. + metric_link = None + if (item.originalname or "") == "test_metric": + mgr = getattr(item.config, "_html_report_manager", None) + if mgr is not None and getattr(mgr, "is_enabled", False): + metric_link = f"{mgr._test_html_dir}/metric_results.html" + + if not rows and not artifacts and not metric_link: + return + + extras = getattr(report, "extras", []) + + if metric_link: + extras.append(pytest_html.extras.url(metric_link, name="Metric Results")) + + if rows: + body = "".join(f"{label}{value:.1f}{unit}" for label, value, unit in rows) + html = f"{body}
stagevalueunit
" + extras.append(pytest_html.extras.html(html)) + + for name, rel_path, abs_path in artifacts or []: + # Primary: a clickable link to the PNG bundled next to the report. + extras.append(pytest_html.extras.url(rel_path, name=name)) + # Best-effort inline thumbnail (base64); never break the row if it fails. + try: + import base64 + + with open(abs_path, "rb") as fp: + b64 = base64.b64encode(fp.read()).decode("ascii") + extras.append(pytest_html.extras.png(b64, name=name)) + except Exception: # noqa: BLE001 + pass + + report.extras = extras diff --git a/cvs/tests/training/jaxmaxtext/jaxmaxtext_distributed.py b/cvs/tests/training/jaxmaxtext/jaxmaxtext_distributed.py new file mode 100644 index 000000000..d18e5f717 --- /dev/null +++ b/cvs/tests/training/jaxmaxtext/jaxmaxtext_distributed.py @@ -0,0 +1,82 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +JAX MaxText Distributed (multi-node) Training Validation Suite. + +Tests performed (in order): +1. test_launch_container - Launch the training container on all nodes +2. test_setup_rdma - Copy the RDMA lib into the container (thor2 NIC) + and verify ibv_devinfo +3. test_setup_tokenizer - Download the HuggingFace tokenizer +4. test_training_run[sweep] - Run MaxText training per sweep (e.g. BF16, FP8) +5. test_metric[sweep-metric] - Validate each metric against its threshold +6. test_loss_curve[sweep] - Render the loss curve and check it decreases +7. test_print_results_table - Console tables + metric-results HTML + summary +8. test_teardown - Tear the container down + +Metrics validated per sweep (namespace training.*): tflops_per_sec_per_gpu, +tokens_per_sec_per_gpu, tokens_per_sec_total, scaling_efficiency_pct, +step_time_{seconds,mean_ms,p50_ms,p95_ms}, final_loss, loss_decreased, +eval_loss, steps_to_target, time_to_target_seconds. + +This is the DISTRIBUTED variant - it adds the RDMA setup stage. Sweeps are +parametrized in conftest.py from the config's `enabled_sweep_list`; the shared +stage logic lives in _common.py. The JAX coordinator is the first cluster node +when `jax_distributed.coordinator_ip` is `auto`. + +Example usage: + cvs run jaxmaxtext_distributed --cluster_file .json \ + --config_file --html log_dir/.html --self-contained-html \ + --log-file=log_dir/log.txt +''' + +from cvs.tests.training.jaxmaxtext import _common + + +def test_launch_container(orch, variant_config, lifecycle, request): + """Launch and verify the MaxText training container is running.""" + return _common.launch_container(orch, variant_config, lifecycle, request) + + +def test_setup_rdma(orch, variant_config, hf_token, lifecycle, request): + """Distributed-only: copy the host RDMA library into the container (thor2 + NIC workaround) and verify ibv_devinfo reports the expected HCA.""" + return _common.setup_rdma(orch, variant_config, hf_token, lifecycle, request) + + +def test_setup_tokenizer(orch, variant_config, hf_token, lifecycle, request): + """Download the HuggingFace tokenizer for the model into the models dir.""" + return _common.setup_tokenizer(orch, variant_config, hf_token, lifecycle, request) + + +def test_training_run(orch, variant_config, hf_token, sweep_name, training_res_dict, lifecycle, request): + """Run one full MaxText training for this sweep, then parse its metrics. + + Parametrized per sweep in conftest.py (e.g. BF16, FP8). A failure is isolated + to this sweep's row so other sweeps still run. + """ + return _common.training_run(orch, variant_config, hf_token, sweep_name, training_res_dict, lifecycle, request) + + +def test_metric(sweep_name, metric, training_res_dict, variant_config, lifecycle, request): + """One row per (sweep, metric): assert the parsed value against the sweep's + threshold cell and record PASS / FAIL / N/A / RECORD.""" + return _common.metric(sweep_name, metric, training_res_dict, variant_config, lifecycle, request) + + +def test_loss_curve(sweep_name, training_res_dict, variant_config, lifecycle, request): + """Sample the training loss, render a per-sweep PNG, and fail if the curve is + not decreasing (least-squares slope check).""" + return _common.loss_curve(sweep_name, training_res_dict, variant_config, lifecycle, request) + + +def test_print_results_table(training_res_dict, request): + """Log per-sweep result tables, write the consolidated metric-results HTML, + and record the aggregated failure summary for the pytest final summary.""" + return _common.print_results_table(training_res_dict, request) + + +def test_teardown(orch, lifecycle, request): + """Tear the container down and verify it is gone.""" + return _common.teardown(orch, lifecycle, request) diff --git a/cvs/tests/training/jaxmaxtext/jaxmaxtext_single.py b/cvs/tests/training/jaxmaxtext/jaxmaxtext_single.py new file mode 100644 index 000000000..6440e14a8 --- /dev/null +++ b/cvs/tests/training/jaxmaxtext/jaxmaxtext_single.py @@ -0,0 +1,73 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. + +JAX MaxText Single-Node Training Validation Suite. + +Tests performed (in order): +1. test_launch_container - Launch the training container +2. test_setup_tokenizer - Download the HuggingFace tokenizer +3. test_training_run[sweep] - Run MaxText training per sweep (e.g. BF16, FP8) +4. test_metric[sweep-metric] - Validate each metric against its threshold +5. test_loss_curve[sweep] - Render the loss curve and check it decreases +6. test_print_results_table - Console tables + metric-results HTML + summary +7. test_teardown - Tear the container down + +Metrics validated per sweep (namespace training.*): tflops_per_sec_per_gpu, +tokens_per_sec_per_gpu, tokens_per_sec_total, scaling_efficiency_pct, +step_time_{seconds,mean_ms,p50_ms,p95_ms}, final_loss, loss_decreased, +eval_loss, steps_to_target, time_to_target_seconds. + +This is the SINGLE-NODE variant - no RDMA setup stage. Sweeps are parametrized +in conftest.py from the config's `enabled_sweep_list`; the shared stage logic +lives in _common.py. + +Example usage: + cvs run jaxmaxtext_single --cluster_file .json \ + --config_file --html log_dir/.html --self-contained-html \ + --log-file=log_dir/log.txt +''' + +from cvs.tests.training.jaxmaxtext import _common + + +def test_launch_container(orch, variant_config, lifecycle, request): + """Launch and verify the MaxText training container is running.""" + return _common.launch_container(orch, variant_config, lifecycle, request) + + +def test_setup_tokenizer(orch, variant_config, hf_token, lifecycle, request): + """Download the HuggingFace tokenizer for the model into the models dir.""" + return _common.setup_tokenizer(orch, variant_config, hf_token, lifecycle, request) + + +def test_training_run(orch, variant_config, hf_token, sweep_name, training_res_dict, lifecycle, request): + """Run one full MaxText training for this sweep, then parse its metrics. + + Parametrized per sweep in conftest.py (e.g. BF16, FP8). A failure is isolated + to this sweep's row so other sweeps still run. + """ + return _common.training_run(orch, variant_config, hf_token, sweep_name, training_res_dict, lifecycle, request) + + +def test_metric(sweep_name, metric, training_res_dict, variant_config, lifecycle, request): + """One row per (sweep, metric): assert the parsed value against the sweep's + threshold cell and record PASS / FAIL / N/A / RECORD.""" + return _common.metric(sweep_name, metric, training_res_dict, variant_config, lifecycle, request) + + +def test_loss_curve(sweep_name, training_res_dict, variant_config, lifecycle, request): + """Sample the training loss, render a per-sweep PNG, and fail if the curve is + not decreasing (least-squares slope check).""" + return _common.loss_curve(sweep_name, training_res_dict, variant_config, lifecycle, request) + + +def test_print_results_table(training_res_dict, request): + """Log per-sweep result tables, write the consolidated metric-results HTML, + and record the aggregated failure summary for the pytest final summary.""" + return _common.print_results_table(training_res_dict, request) + + +def test_teardown(orch, lifecycle, request): + """Tear the container down and verify it is gone.""" + return _common.teardown(orch, lifecycle, request) diff --git a/cvs/tests/training/megatron/README.md b/cvs/tests/training/megatron/README.md new file mode 100644 index 000000000..e3cbe2b04 --- /dev/null +++ b/cvs/tests/training/megatron/README.md @@ -0,0 +1,159 @@ +# Megatron Training Suite (single-node and distributed) + +Cluster validation suite that runs Megatron-LM pre-training on AMD Instinct GPUs (single-node or multi-node) and gates the run on performance and correctness metrics with a PASS/FAIL HTML report. + +## Overview + +The suite drives a Megatron-LM training job inside a Docker container on one or more cluster nodes, then parses the training log to produce metrics and verdicts. It provides: + +- **Two suites** — `megatron_single` (single-node) and `megatron_distributed` (multi-node, adds RDMA/NIC setup). +- **Parameter sweeps** — one full training run per enabled combo (e.g. FP8 and BF16), each with its own result rows in the report. +- **Loss curve** — a per-combo decreasing-trend check on `lm_loss` at steps 100 / 500 / 1k / 5k. +- **Training-log error scanning** — NCCL, GPU HW faults, OOM, and other signatures fail a run early with a clear reason. +- **HTML report** — per-test rows with linked logs and a consolidated metric results page. + +The mode (single vs distributed) is determined by the config file's `framework` field (`megatron_single` or `megatron_distributed`). + +## Quick Start + +### Single-node + +```bash +cvs run megatron_single \ + --cluster_file input/cluster_file/cluster.json \ + --config_file input/config_file/training/megatron/mi325x_megatron_llama-3.1-8b_single.json \ + --html ./logs/megatron_single.html --self-contained-html -vvv -s +``` + +### Distributed (multi-node) + +```bash +cvs run megatron_distributed \ + --cluster_file input/cluster_file/cluster.json \ + --config_file input/config_file/training/megatron/mi325x_megatron_llama-3.3-70b_distributed.json \ + --html ./logs/megatron_distributed.html --self-contained-html -vvv -s +``` + +- `--cluster_file` — JSON describing the node(s); see `cvs/input/cluster_file/README.md`. +- `--config_file` — one of the config files in `cvs/input/config_file/training/megatron/`; see that folder's README for the full variable reference. +- `--html` / `--self-contained-html` — write the HTML report. + +Use a single-node config with `megatron_single` and a distributed config with `megatron_distributed`. The config's `framework` field must match the suite. + +### Run a specific stage + +```bash +cvs run megatron_single test_smoke \ + --cluster_file input/cluster_file/cluster.json \ + --config_file input/config_file/training/megatron/mi325x_megatron_llama-3.1-8b_single.json +``` + +## The Two Suites + +| Suite (`cvs run `) | File | Distributed stages | Use with | +|---|---|---|---| +| `megatron_single` | `megatron_single.py` | none | single-node config (`framework: megatron_single`) | +| `megatron_distributed` | `megatron_distributed.py` | `test_setup_rdma` | multi-node config (`framework: megatron_distributed`) | + +Both suites share fixtures and hooks from `conftest.py`. + +## Test Lifecycle + +Tests run in this pinned order. `[combo]` = one row per enabled sweep combo. + +| Order | Test | Runs on | Purpose | +|---|---|---|---| +| 0 | `test_launch_container` | once | Launch and verify the container | +| 1 | `test_setup_rdma` | distributed only | Copy RDMA lib into container (thor2 NIC) and verify `ibv_devinfo` | +| 2 | `test_download_tokenizer` | once | Download HF tokenizer for models that require a local file (DeepSeek, Mixtral) | +| 3 | `test_smoke` | once | Fixed small run confirming the model loads and trains without error | +| 4 | `test_training[combo]` | per combo | Build cmd, train, poll logs, parse results; GPU memory freed between combos | +| 5 | `test_metric[combo]` | per combo | Threshold PASS/FAIL per metric | +| 6 | `test_loss_curve[combo]` | per combo | Gate on downward `lm_loss` trend at steps 100 / 500 / 1k / 5k | +| 7 | `test_teardown` | once | Tear the container down | + +A training failure is isolated to that combo's `test_training` row; other combos still run. When a combo's training does not complete, its downstream `test_metric` and `test_loss_curve` rows are skipped. If an early lifecycle stage fails, all subsequent stages are skipped via `lifecycle.failed`. + +On a training failure, lingering GPU processes are killed (`stop_training_processes`) so the next combo does not launch on top of them. + +## Sweeps + +A sweep combo is one full training run declared in `sweep.combinations`. `sweep.runs` is the ordered list of combo IDs to execute; set it to a subset to run only selected combos without editing `combinations`. + +The combo ID (e.g. `llama3_1_8b-mi325-bs128-mbs4-fp8`) appears in every parametrized row: `test_training[llama3_1_8b-mi325-bs128-mbs4-fp8]`, `test_metric[llama3_1_8b-mi325-bs128-mbs4-fp8]`, and `test_loss_curve[llama3_1_8b-mi325-bs128-mbs4-fp8]`. + +## Metrics and PASS/FAIL + +Each `test_metric[combo]` compares the parsed metric against its threshold spec and reports one of: + +| Status | Meaning | +|---|---| +| PASS | value satisfies the threshold | +| FAIL | value violates the threshold (row is red; aggregated in the summary) | +| RECORD | no threshold defined, or `enforce_thresholds: false` — value logged, not gated | + +Metrics surfaced (namespace `training.*`): + +| Metric | Description | +|---|---| +| `training.throughput_per_gpu` | TFLOP/s per GPU | +| `training.tokens_per_gpu` | Tokens per GPU per second | +| `training.elapsed_time_per_iteration` | Wall time per training step (ms) | +| `training.mem_usage` | GPU memory usage | +| `training.scaling_efficiency_pct` | Multi-node scaling efficiency % vs single-node baseline (distributed only) | + +Gating requires `enforce_thresholds: true` in the config. Set to `false` for record-only runs. + +## Scaling Efficiency (distributed only) + +`test_training` computes scaling efficiency as: + +``` +efficiency % = (actual_total_tok/s / (actual_nodes / baseline_nodes)) / baseline_total_tok/s × 100 +``` + +Populate `scaling_baseline.tokens_per_sec_total` in the config from a completed single-node run (`tok/s/GPU × 8`). Set to `0.0` to disable and collect data only. + +## Training-Log Error Detection + +During polling, each node's `training.log` is scanned for known error patterns. Defaults cover: + +- NCCL errors and timeouts +- GPU hardware faults and hangs +- PyTorch distributed errors + +A match fails that combo's `test_training` with the matched pattern name and the last lines of the log. + +## Reports and Logs + +- **Results table** — one row per test; metric rows show PASS/FAIL from the threshold check. +- **Full log** — each test row links to its own captured log. +- **Training logs** — written inside the container at `/megatron-logs//out-node/training.log`. + +Log path fields: + +| Placeholder | Source | +|---|---| +| `` | `config.log_dir` in the config file | +| `` | Sweep run ID (e.g. `llama3_1_8b-mi325-bs128-mbs4-fp8`) | +| `out-node` | One directory per node; `out-node0` for single-node | + +## Config and Threshold Files + +Located in `cvs/input/config_file/training/megatron/`: + +| Config | Threshold | Arch / mode | +|---|---|---| +| `mi325x_megatron_llama-3.1-8b_single.json` | `mi325x_megatron_llama-3.1-8b_single_threshold.json` | MI325X, single-node | +| `mi325x_megatron_llama-3.3-70b_single.json` | `mi325x_megatron_llama-3.3-70b_single_threshold.json` | MI325X, single-node | +| `mi325x_megatron_llama-3.3-70b_distributed.json` | `mi325x_megatron_llama-3.3-70b_distributed_threshold.json` | MI325X, distributed | +| `mi325x_megatron_deepseek-v2-lite_single.json` | `mi325x_megatron_deepseek-v2-lite_single_threshold.json` | MI325X, single-node | + +See [`cvs/input/config_file/training/megatron/README.md`](../../input/config_file/training/megatron/README.md) for the full variable reference and the values you must change for your cluster and container image. + +## Prerequisites + +- Passwordless SSH from the control host to each cluster node (key in the cluster file) and Docker available on the nodes. +- A container image bundling Megatron-LM for ROCm (`container.image` in the config); Megatron-LM must be present at `config.megatron_root` (default `/workspace/Megatron-LM`). +- A Hugging Face token file at `config.hf_token_file` (used to fetch the tokenizer). Tokenizer download requires network access on the nodes. For gated models (LLaMA, DeepSeek), model access must be granted on huggingface.co. +- For distributed runs: RDMA interfaces configured and reachable on all nodes; a shared filesystem path reachable from all nodes for logs and scripts. diff --git a/cvs/tests/training/megatron/conftest.py b/cvs/tests/training/megatron/conftest.py new file mode 100644 index 000000000..bdcf6a784 --- /dev/null +++ b/cvs/tests/training/megatron/conftest.py @@ -0,0 +1,187 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. +The year included in the foregoing notice is the year of creation of the work. +All code contained here is Property of Advanced Micro Devices, Inc. +''' + +import json +import os + +import pytest + +from cvs.core.orchestrators.factory import OrchestratorConfig, OrchestratorFactory +from cvs.lib import globals +from cvs.lib.utils_lib import resolve_cluster_config_placeholders +from cvs.lib.training.megatron.utils.training_config_loader import load_training_variant + +log = globals.log + + +def _deep_merge(base, override): + """Recursively merge `override` onto `base` (dicts merged key-wise, scalars/lists replaced). + + Protects cluster-set scalar and dict container keys from being wiped by a + top-level replace: they survive unless the training block overrides that same + key. List keys (e.g. runtime.args, volumes) are replaced here and recombined + additively downstream in container.py's getters. + """ + if not (isinstance(base, dict) and isinstance(override, dict)): + return override + out = dict(base) + for k, v in override.items(): + out[k] = _deep_merge(base[k], v) if k in base else v + return out + + +@pytest.fixture(scope="module") +def cluster_dict(pytestconfig): + cluster_file = pytestconfig.getoption("cluster_file") + if not cluster_file: + pytest.fail("--cluster_file is required") + with open(cluster_file) as fp: + d = json.load(fp) + return resolve_cluster_config_placeholders(d) + + +@pytest.fixture(scope="module") +def variant_config(pytestconfig, cluster_dict): + config_file = pytestconfig.getoption("config_file") + if not config_file: + pytest.fail("--config_file is required") + return load_training_variant(config_file, cluster_dict) + + +@pytest.fixture(scope="module") +def hf_token(variant_config): + path = variant_config.config['hf_token_file'] + if not os.path.isfile(path): + pytest.skip(f"hf_token file missing: {path}") + with open(path) as fp: + return fp.read().strip() + + +class _Lifecycle: + """Cross-test state for the lifecycle-as-tests model. + + The container is launched once (test_launch_container), all sweep combos + run inside it (test_training), GPU memory is freed between combos via + stop_training_processes(), and the container is torn down once at the end + (test_teardown). `failed` lets a broken stage skip the rest. `torn_down` + suppresses the orch fixture leak-guard when test_teardown already ran. + `report` maps each nodeid to its recorded (label, value, unit) rows. + """ + + def __init__(self): + self.failed = False + self.torn_down = False + self.report = {} # nodeid -> list[(label, value, unit)] + self.artifacts = {} # nodeid -> list[(link_name, rel_path)] + + def record(self, nodeid, label, value, unit="s"): + self.report.setdefault(nodeid, []).append((label, value, unit)) + + def add_artifact(self, nodeid, link_name, rel_path, abs_path=None): + self.artifacts.setdefault(nodeid, []).append((link_name, rel_path)) + + +@pytest.fixture(scope="module") +def lifecycle(): + return _Lifecycle() + + +@pytest.fixture(scope="module") +def train_res_dict(): + return {} + + +@pytest.fixture(scope="module") +def orch(cluster_dict, variant_config, lifecycle): + """Construct a ContainerOrchestrator and own a final teardown safety net. + + The container is launched once in test_launch_container and torn down once + in test_teardown, which sets lifecycle.torn_down=True. This finalizer only + fires when torn_down is False -- i.e. test_teardown did not run (e.g. a + crash before teardown) -- so nothing leaks past the module without + double-tearing down in the normal case. + """ + container_block = _deep_merge(cluster_dict.get("container", {}), variant_config.container.model_dump()) + testsuite_config = {"orchestrator": "container", "container": container_block} + cfg = OrchestratorConfig.from_configs(cluster_dict, testsuite_config) + o = OrchestratorFactory.create_orchestrator(log, cfg) + yield o + if not lifecycle.torn_down: + log.info("orch fixture leak-guard: tearing down container (per-combo teardown did not run)") + o.teardown_containers() + + +def pytest_collection_modifyitems(items): + """Pin lifecycle order: launch → training combos → metric → teardown.""" + rank = { + "test_launch_container": 0, + "test_download_tokenizer": 1, + "test_smoke": 2, + "test_training": 3, + "test_metric": 4, + "test_loss_curve": 5, + "test_teardown": 6, + } + items.sort(key=lambda it: rank.get(it.originalname or it.name.split("[")[0], 99)) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """Attach this test's recorded timing rows to its HTML report detail panel.""" + outcome = yield + report = outcome.get_result() + if report.when != "call": + return + lc = item.funcargs.get("lifecycle") + if not lc: + return + rows = getattr(lc, "report", {}).get(item.nodeid) + artifacts = getattr(lc, "artifacts", {}).get(item.nodeid) + if not rows and not artifacts and not report.failed: + return + try: + import pytest_html + except ImportError: + return + extras = getattr(report, "extras", []) + if rows: + body = "".join(f"{label}{value:.1f}{unit}" for label, value, unit in rows) + html = f"{body}
stagevalueunit
" + extras.append(pytest_html.extras.html(html)) + if artifacts: + for link_name, rel_path in artifacts: + extras.append(pytest_html.extras.url(rel_path, name=link_name)) + if report.failed: + props = dict(item.user_properties) + log_tail = props.get("training_log_tail") + if log_tail: + extras.append(pytest_html.extras.text(log_tail, name="Training Log (tail)")) + report.extras = extras + + +# def pytest_html_results_table_header(cells): +# cells.insert(-1, "Value") +# cells.insert(-1, "Unit") + + +# def pytest_html_results_table_row(report, cells): +# if not hasattr(report, 'user_properties'): +# return +# props = dict(report.user_properties) +# has = "metric_value" in props +# val = props.get("metric_value") +# unit = props.get("metric_unit", "") if has else "" +# if not has: +# shown = "" +# elif val is None: +# shown = "-" +# elif isinstance(val, float): +# shown = f"{val:.3f}" +# else: +# shown = str(val) +# cells.insert(-1, f"{shown}") +# cells.insert(-1, f"{unit}") diff --git a/cvs/tests/training/megatron/megatron_distributed.py b/cvs/tests/training/megatron/megatron_distributed.py new file mode 100644 index 000000000..6f2fcaad9 --- /dev/null +++ b/cvs/tests/training/megatron/megatron_distributed.py @@ -0,0 +1,375 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. +The year included in the foregoing notice is the year of creation of the work. +All code contained here is Property of Advanced Micro Devices, Inc. + +Unified Megatron training suite for distributed (multi-node) runs. +Topology is determined by the config file: + framework=megatron_distributed -> multi-node (distributed_training=True) + +Lifecycle (each stage is a separate test): + test_launch_container — launch the container once for all sweep combos + test_smoke — fixed small cell: model loads and runs N steps without error + test_training — parametrized: one test per sweep combo; kills GPU + processes in finally so VRAM is free for the next combo + test_metric — parametrized: threshold check per combo via evaluate_all + test_loss_curve — parametrized: slope-based loss decrease check with PNG render + test_teardown — tear down the container once after all combos +''' + +import json +import os +import time + +import pytest + +from cvs.lib import globals +from cvs.lib.training.megatron.megatron_lib import MegatronTrainingJob +from cvs.lib.training.megatron.utils.loss_curve import ( + parse_all_loss_points, + sample_loss_curve, + evaluate_loss_decreasing, +) +from cvs.lib.training.megatron.utils.loss_curve_plot import render_loss_curve_png +from cvs.lib.training.megatron.utils.scaling import compute_scaling_efficiency +from cvs.lib.utils.verdict import _check_one, ThresholdViolation +from cvs.lib.utils_lib import update_test_result + +log = globals.log + +# Smoke cell: smallest fixed parameters that confirm the model loads and trains. +_SMOKE_MBS = "1" +_SMOKE_GBS = "16" +_SMOKE_ITERS = "10" +_SMOKE_PRECISION = "BF16" + + +def pytest_generate_tests(metafunc): + """Parametrize test_training and test_metric from sweep.combinations filtered by sweep.runs. + + sweep.combinations is a dict of {run_id: {micro_batch_size, global_batch_size, ...}}. + sweep.runs is a list of run_ids to execute (subset or all). + One case is emitted per entry in sweep.runs — no cartesian product. + The pytest parametrize ID is the run_id so that request.node.callspec.id + can be passed directly to variant_config.cell_key(). + """ + config_file = metafunc.config.getoption("config_file") + if not config_file or not os.path.isfile(config_file): + return + with open(config_file) as fp: + raw = json.load(fp) + + sweep = raw.get("sweep", {}) + combinations = sweep.get("combinations", {}) + runs = sweep.get("runs", list(combinations.keys())) + + cases = [] + ids = [] + for run_id in runs: + if run_id not in combinations: + log.warning("sweep.runs entry '%s' not found in sweep.combinations; skipping", run_id) + continue + combo = combinations[run_id] + mbs = combo["micro_batch_size"] + gbs = combo["global_batch_size"] + precision = combo.get("precision", "") + cases.append((mbs, gbs, precision)) + ids.append(run_id) + + if "micro_batch_size" in metafunc.fixturenames and "global_batch_size" in metafunc.fixturenames and cases: + metafunc.parametrize("micro_batch_size,global_batch_size,precision", cases, ids=ids) + + +def test_launch_container(orch, variant_config, lifecycle, request): + """Stage 0: launch the container once for all sweep combos.""" + nodeid = request.node.nodeid + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + lifecycle.torn_down = False + + t = time.monotonic() + ok = orch.setup_containers() + lifecycle.record(nodeid, "container_launch", time.monotonic() - t) + if not ok: + lifecycle.failed = True + pytest.fail(f"setup_containers() returned False for {name}") + if not orch.verify_containers_running(name): + lifecycle.failed = True + pytest.fail(f"container {name} not running after setup_containers()") + + +def test_download_tokenizer(orch, variant_config, hf_token, lifecycle, request): + """Stage 1: download the tokenizer model once if the model family requires it.""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + mt_obj = MegatronTrainingJob( + orch, + variant_config, + hf_token=hf_token, + micro_batch_size="1", + global_batch_size="1", + precision="BF16", + distributed_training=True, + tune_model_params=False, + run_label="tokenizer_check", + ) + + if not mt_obj._needs_local_tokenizer(): + lifecycle.tokenizer_path = None + log.info( + "test_download_tokenizer: no local tokenizer needed for %s — skipping download", + variant_config.model_params["tokenizer_model"], + ) + return + + t = time.monotonic() + try: + mt_obj.download_tokenizer_model() + except Exception: + lifecycle.failed = True + raise + + lifecycle.tokenizer_path = mt_obj.local_tokenizer_path + lifecycle.record(request.node.nodeid, "tokenizer_download", time.monotonic() - t) + log.info("test_download_tokenizer: tokenizer ready at %s", lifecycle.tokenizer_path) + + +def test_smoke(orch, variant_config, hf_token, lifecycle, request): + """Stage 2: smoke-test — model loads and runs _SMOKE_ITERS steps without error. + + Passes if training reaches iteration _SMOKE_ITERS/_SMOKE_ITERS without error. + No metric assertions — completion without error is the only requirement. + """ + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + globals.error_list = [] + + mt_obj = MegatronTrainingJob( + orch, + variant_config, + hf_token=hf_token, + micro_batch_size=_SMOKE_MBS, + global_batch_size=_SMOKE_GBS, + precision=_SMOKE_PRECISION, + distributed_training=True, + tune_model_params=False, + run_label="smoke", + ) + mt_obj.iterations = int(_SMOKE_ITERS) + mt_obj.local_tokenizer_path = getattr(lifecycle, "tokenizer_path", None) + + t = time.monotonic() + try: + mt_obj.build_training_job_cmd() + mt_obj.start_training_job() + mt_obj.poll_for_training_completion() + except Exception: + lifecycle.failed = True + raise + finally: + mt_obj.stop_training_processes() + + if globals.error_list: + lifecycle.failed = True + update_test_result() + lifecycle.record(request.node.nodeid, "smoke", time.monotonic() - t) + log.info("smoke PASSED | iters=%s", _SMOKE_ITERS) + + +def test_training( + orch, variant_config, hf_token, micro_batch_size, global_batch_size, precision, train_res_dict, lifecycle, request +): + """Stage 3 (parametrized): run one sweep combo inside the shared container. + + stop_training_processes() runs in a finally block after every combo so GPU + memory is released before the next combo starts. + """ + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + nodeid = request.node.nodeid + combo_key = request.node.callspec.id + globals.error_list = [] + mt_obj = MegatronTrainingJob( + orch, + variant_config, + hf_token=hf_token, + micro_batch_size=micro_batch_size, + global_batch_size=global_batch_size, + precision=precision, + distributed_training=True, + tune_model_params=False, + run_label=combo_key, + ) + + mt_obj.local_tokenizer_path = getattr(lifecycle, "tokenizer_path", None) + + elapsed = 0 + try: + t = time.monotonic() + mt_obj.build_training_job_cmd() + mt_obj.start_training_job() + mt_obj.poll_for_training_completion() + mt_obj.verify_training_results() + elapsed = time.monotonic() - t + except Exception: + lifecycle.failed = True + raise + finally: + mt_obj.stop_training_processes() + + lifecycle.record(nodeid, "training", elapsed) + request.node.user_properties.append(("metric_value", elapsed)) + request.node.user_properties.append(("metric_unit", "s")) + + train_res_dict[combo_key] = mt_obj.training_results_dict + train_res_dict[combo_key]["_combo_log_dir"] = mt_obj.combo_log_dir + + tput_per_gpu = train_res_dict[combo_key].get("throughput_per_gpu", []) + if tput_per_gpu: + gpus_per_node = 8 + tokens_per_sec_total = float(tput_per_gpu[-1]) * int(mt_obj.nnodes) * gpus_per_node + baseline = variant_config.scaling_baseline + efficiency = compute_scaling_efficiency( + tokens_per_sec_total, + int(mt_obj.nnodes), + baseline.tokens_per_sec_total, + baseline.num_nodes, + ) + if efficiency is not None: + train_res_dict[combo_key]["scaling_efficiency_pct"] = [str(efficiency)] + try: + tail = mt_obj._read_last_node_log(tail_lines=50) + train_res_dict[combo_key]["_log_tail"] = tail + request.node.user_properties.append(("training_log_tail", tail)) + except Exception: + pass + update_test_result() + + +def test_metric(variant_config, micro_batch_size, global_batch_size, precision, train_res_dict, lifecycle, request): + """Stage 4 (parametrized): compare each combo's metrics against thresholds.""" + combo_key = request.node.callspec.id + if combo_key not in train_res_dict: + pytest.skip(f"no recorded results for combo '{combo_key}' (training did not run)") + + if not variant_config.enforce_thresholds: + log.info("enforce_thresholds=false; skipping verdict for combo '%s'", combo_key) + return + + cell = variant_config.cell_key(combo_key) + thresholds = variant_config.thresholds.get(cell) + if not thresholds: + log.warning("no thresholds defined for cell '%s'; skipping threshold checks", cell) + return + + actuals_raw = train_res_dict[combo_key] + request.node.user_properties.append(("training_log_tail", actuals_raw.get("_log_tail", ""))) + actuals = {f"training.{k}": float(v[-1]) for k, v in actuals_raw.items() if v and not k.startswith("_")} + + log.info("--- Threshold check for combo '%s' ---", combo_key) + violations = [] + for metric, spec in thresholds.items(): + if metric not in actuals: + msg = f"{metric}: missing from actuals" + log.error(" FAILED %s", msg) + violations.append(msg) + continue + if actuals[metric] is None: + msg = f"{metric}: value is None (metric unavailable for this run)" + log.error(" FAILED %s", msg) + violations.append(msg) + continue + spec_with_actuals = dict(spec) + if spec.get("kind") == "min_ratio": + spec_with_actuals["_actuals"] = actuals + v = _check_one(metric, actuals[metric], spec_with_actuals) + if v: + log.error(" FAILED %s", v) + violations.append(v) + else: + log.info(" PASSED %s: actual=%s threshold=%s", metric, actuals[metric], spec) + + if violations: + summary = "FAILED\n" + "\n".join(violations) + log.error("--- %d violation(s) for combo '%s' ---", len(violations), combo_key) + request.node.user_properties.append(("threshold_comparison", summary)) + raise ThresholdViolation(violations) + + log.info("--- All threshold checks PASSED for combo '%s' ---", combo_key) + request.node.user_properties.append(("threshold_comparison", "PASSED")) + + +def test_loss_curve( + orch, variant_config, micro_batch_size, global_batch_size, precision, train_res_dict, lifecycle, request +): + """Parametrized: slope-based loss curve check with PNG render.""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + combo_key = request.node.callspec.id + if combo_key not in train_res_dict: + pytest.skip(f"no recorded results for combo '{combo_key}' (training did not run)") + + combo_log_dir = train_res_dict[combo_key].get("_combo_log_dir") + if not combo_log_dir: + log.warning("no log dir recorded for combo '%s'; skipping loss curve check", combo_key) + pytest.skip(f"no log dir recorded for combo '{combo_key}'") + + n = len(orch.hosts) + last_host = orch.hosts[-1] + log_path = f"{combo_log_dir}/out-node{n - 1}/training.log" + out_dict = orch.exec(f"cat {log_path}", hosts=[last_host]) + log_text = out_dict.get(last_host) or "" + + lc = variant_config.loss_curve + step_metrics = parse_all_loss_points(log_text) + points = sample_loss_curve(step_metrics, lc.sample_every, lc.milestone_steps) + + log.info("--- Loss curve check for combo '%s' (%d points sampled) ---", combo_key, len(points)) + + mgr = getattr(request.config, "_html_report_manager", None) + mgr_enabled = mgr is not None and getattr(mgr, "is_enabled", False) + out_dir = mgr.log_dir if mgr_enabled else "/tmp" + try: + from pathlib import Path as _Path + import uuid as _uuid + + _Path(out_dir).mkdir(parents=True, exist_ok=True) + fname = f"loss_curve_{combo_key}_{str(_uuid.uuid4()).split('-')[-1]}.png" + png_path = _Path(out_dir) / fname + title = f"Training Loss Curve — {variant_config.model_params.get('model_name', '')} [{combo_key}]" + rendered = render_loss_curve_png(points, png_path, title=title) + if rendered and mgr_enabled: + rel_path = str(_Path(rendered).relative_to(mgr.htmlpath.parent)) + lifecycle.add_artifact(request.node.nodeid, f"Loss Curve [{combo_key}]", rel_path, rendered) + except Exception as e: + log.warning("loss curve: could not render PNG (%s)", e) + + verdict = evaluate_loss_decreasing(points, lc.max_slope) + if verdict is None: + pytest.skip(f"loss curve needs >= 2 sampled points (got {len(points)}); increase training_iterations") + + decreasing, slope, detail = verdict + log.info("loss curve: %s", detail) + + if points: + request.node.user_properties.append(("metric_value", points[-1][1])) + request.node.user_properties.append(("metric_unit", "lm_loss")) + + if lc.enforce and not decreasing: + pytest.fail(f"training loss is not decreasing for combo '{combo_key}': {detail}") + + +def test_teardown(orch, lifecycle, request): + """Stage 5: tear down the container once after all combos have run.""" + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + t = time.monotonic() + orch.teardown_containers() + lifecycle.record(request.node.nodeid, "teardown", time.monotonic() - t) + if orch.verify_containers_running(name): + log.error("container %s still running after teardown_containers()", name) + else: + lifecycle.torn_down = True diff --git a/cvs/tests/training/megatron/megatron_single.py b/cvs/tests/training/megatron/megatron_single.py new file mode 100644 index 000000000..bc27b1aae --- /dev/null +++ b/cvs/tests/training/megatron/megatron_single.py @@ -0,0 +1,373 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. +The year included in the foregoing notice is the year of creation of the work. +All code contained here is Property of Advanced Micro Devices, Inc. + +Unified Megatron training suite for single-node runs. +Topology is determined by the config file: + framework=megatron_single -> single-node (distributed_training=False) + +Lifecycle (each stage is a separate test): + test_launch_container — launch the container once for all sweep combos + test_smoke — fixed small cell: model loads and runs N steps without error + test_training — parametrized: one test per sweep combo; kills GPU + processes in finally so VRAM is free for the next combo + test_metric — parametrized: threshold check per combo via evaluate_all + test_loss_curve — parametrized: slope-based loss decrease check with PNG render + test_teardown — tear down the container once after all combos +''' + +import json +import os +import time + +import pytest + +from cvs.lib import globals +from cvs.lib.training.megatron.megatron_lib import MegatronTrainingJob +from cvs.lib.training.megatron.utils.loss_curve import ( + parse_all_loss_points, + sample_loss_curve, + evaluate_loss_decreasing, +) +from cvs.lib.training.megatron.utils.loss_curve_plot import render_loss_curve_png +from cvs.lib.training.megatron.utils.scaling import compute_scaling_efficiency +from cvs.lib.utils.verdict import _check_one, ThresholdViolation +from cvs.lib.utils_lib import update_test_result + +log = globals.log + +# Smoke cell: smallest fixed parameters that confirm the model loads and trains. +_SMOKE_MBS = "1" +_SMOKE_GBS = "8" +_SMOKE_ITERS = "10" +_SMOKE_PRECISION = "BF16" + + +def pytest_generate_tests(metafunc): + """Parametrize test_training and test_metric from sweep.combinations filtered by sweep.runs. + + sweep.combinations is a dict of {run_id: {micro_batch_size, global_batch_size, ...}}. + sweep.runs is a list of run_ids to execute (subset or all). + One case is emitted per entry in sweep.runs — no cartesian product. + The pytest parametrize ID is the run_id so that request.node.callspec.id + can be passed directly to variant_config.cell_key(). + """ + config_file = metafunc.config.getoption("config_file") + if not config_file or not os.path.isfile(config_file): + return + with open(config_file) as fp: + raw = json.load(fp) + + sweep = raw.get("sweep", {}) + combinations = sweep.get("combinations", {}) + runs = sweep.get("runs", list(combinations.keys())) + + cases = [] + ids = [] + for run_id in runs: + if run_id not in combinations: + log.warning("sweep.runs entry '%s' not found in sweep.combinations; skipping", run_id) + continue + combo = combinations[run_id] + mbs = combo["micro_batch_size"] + gbs = combo["global_batch_size"] + precision = combo.get("precision", "") + cases.append((mbs, gbs, precision)) + ids.append(run_id) + + if "micro_batch_size" in metafunc.fixturenames and "global_batch_size" in metafunc.fixturenames and cases: + metafunc.parametrize("micro_batch_size,global_batch_size,precision", cases, ids=ids) + + +def test_launch_container(orch, variant_config, lifecycle, request): + """Stage 0: launch the container once for all sweep combos.""" + nodeid = request.node.nodeid + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + lifecycle.torn_down = False + + t = time.monotonic() + ok = orch.setup_containers() + lifecycle.record(nodeid, "container_launch", time.monotonic() - t) + if not ok: + lifecycle.failed = True + pytest.fail(f"setup_containers() returned False for {name}") + if not orch.verify_containers_running(name): + lifecycle.failed = True + pytest.fail(f"container {name} not running after setup_containers()") + + +def test_download_tokenizer(orch, variant_config, hf_token, lifecycle, request): + """Stage 1: download the tokenizer model once if the model family requires it.""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + mt_obj = MegatronTrainingJob( + orch, + variant_config, + hf_token=hf_token, + micro_batch_size="1", + global_batch_size="1", + precision="BF16", + distributed_training=False, + tune_model_params=False, + run_label="tokenizer_check", + ) + + if not mt_obj._needs_local_tokenizer(): + lifecycle.tokenizer_path = None + log.info( + "test_download_tokenizer: no local tokenizer needed for %s — skipping download", + variant_config.model_params["tokenizer_model"], + ) + return + + t = time.monotonic() + try: + mt_obj.download_tokenizer_model() + except Exception: + lifecycle.failed = True + raise + + lifecycle.tokenizer_path = mt_obj.local_tokenizer_path + lifecycle.record(request.node.nodeid, "tokenizer_download", time.monotonic() - t) + log.info("test_download_tokenizer: tokenizer ready at %s", lifecycle.tokenizer_path) + + +def test_smoke(orch, variant_config, hf_token, lifecycle, request): + """Stage 2: smoke-test — model loads and runs _SMOKE_ITERS steps without error. + + Passes if training reaches iteration _SMOKE_ITERS/_SMOKE_ITERS without error. + No metric assertions — completion without error is the only requirement. + """ + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + globals.error_list = [] + + mt_obj = MegatronTrainingJob( + orch, + variant_config, + hf_token=hf_token, + micro_batch_size=_SMOKE_MBS, + global_batch_size=_SMOKE_GBS, + precision=_SMOKE_PRECISION, + distributed_training=False, + tune_model_params=False, + run_label="smoke", + ) + mt_obj.iterations = int(_SMOKE_ITERS) + mt_obj.local_tokenizer_path = getattr(lifecycle, "tokenizer_path", None) + + t = time.monotonic() + try: + mt_obj.build_training_job_cmd() + mt_obj.start_training_job() + mt_obj.poll_for_training_completion() + except Exception: + lifecycle.failed = True + raise + finally: + mt_obj.stop_training_processes() + + if globals.error_list: + lifecycle.failed = True + update_test_result() + lifecycle.record(request.node.nodeid, "smoke", time.monotonic() - t) + log.info("smoke PASSED | iters=%s", _SMOKE_ITERS) + + +def test_training( + orch, variant_config, hf_token, micro_batch_size, global_batch_size, precision, train_res_dict, lifecycle, request +): + """Stage 3 (parametrized): run one sweep combo inside the shared container. + + stop_training_processes() runs in a finally block after every combo so GPU + memory is released before the next combo starts. + """ + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + nodeid = request.node.nodeid + combo_key = request.node.callspec.id + globals.error_list = [] + mt_obj = MegatronTrainingJob( + orch, + variant_config, + hf_token=hf_token, + micro_batch_size=micro_batch_size, + global_batch_size=global_batch_size, + precision=precision, + distributed_training=False, + tune_model_params=False, + run_label=combo_key, + ) + + mt_obj.local_tokenizer_path = getattr(lifecycle, "tokenizer_path", None) + + elapsed = 0 + try: + t = time.monotonic() + mt_obj.build_training_job_cmd() + mt_obj.start_training_job() + mt_obj.poll_for_training_completion() + mt_obj.verify_training_results() + elapsed = time.monotonic() - t + except Exception: + lifecycle.failed = True + raise + finally: + mt_obj.stop_training_processes() + + lifecycle.record(nodeid, "training", elapsed) + request.node.user_properties.append(("metric_value", elapsed)) + request.node.user_properties.append(("metric_unit", "s")) + + train_res_dict[combo_key] = mt_obj.training_results_dict + train_res_dict[combo_key]["_combo_log_dir"] = mt_obj.combo_log_dir + + tput_per_gpu = train_res_dict[combo_key].get("throughput_per_gpu", []) + if tput_per_gpu: + gpus_per_node = 8 + tokens_per_sec_total = float(tput_per_gpu[-1]) * int(mt_obj.nnodes) * gpus_per_node + baseline = variant_config.scaling_baseline + efficiency = compute_scaling_efficiency( + tokens_per_sec_total, + int(mt_obj.nnodes), + baseline.tokens_per_sec_total, + baseline.num_nodes, + ) + if efficiency is not None: + train_res_dict[combo_key]["scaling_efficiency_pct"] = [str(efficiency)] + try: + tail = mt_obj._read_last_node_log(tail_lines=50) + train_res_dict[combo_key]["_log_tail"] = tail + request.node.user_properties.append(("training_log_tail", tail)) + except Exception: + pass + update_test_result() + + +def test_metric(variant_config, micro_batch_size, global_batch_size, precision, train_res_dict, lifecycle, request): + """Stage 4 (parametrized): compare each combo's metrics against thresholds.""" + combo_key = request.node.callspec.id + if combo_key not in train_res_dict: + pytest.skip(f"no recorded results for combo '{combo_key}' (training did not run)") + + if not variant_config.enforce_thresholds: + log.info("enforce_thresholds=false; skipping verdict for combo '%s'", combo_key) + return + + cell = variant_config.cell_key(combo_key) + thresholds = variant_config.thresholds.get(cell) + if not thresholds: + log.warning("no thresholds defined for cell '%s'; skipping threshold checks", cell) + return + + actuals_raw = train_res_dict[combo_key] + request.node.user_properties.append(("training_log_tail", actuals_raw.get("_log_tail", ""))) + actuals = {f"training.{k}": float(v[-1]) for k, v in actuals_raw.items() if v and not k.startswith("_")} + + log.info("--- Threshold check for combo '%s' ---", combo_key) + violations = [] + for metric, spec in thresholds.items(): + if metric not in actuals: + msg = f"{metric}: missing from actuals" + log.error(" FAILED %s", msg) + violations.append(msg) + continue + if actuals[metric] is None: + msg = f"{metric}: value is None (metric unavailable for this run)" + log.error(" FAILED %s", msg) + violations.append(msg) + continue + spec_with_actuals = dict(spec) + if spec.get("kind") == "min_ratio": + spec_with_actuals["_actuals"] = actuals + v = _check_one(metric, actuals[metric], spec_with_actuals) + if v: + log.error(" FAILED %s", v) + violations.append(v) + else: + log.info(" PASSED %s: actual=%s threshold=%s", metric, actuals[metric], spec) + + if violations: + summary = "FAILED\n" + "\n".join(violations) + log.error("--- %d violation(s) for combo '%s' ---", len(violations), combo_key) + request.node.user_properties.append(("threshold_comparison", summary)) + raise ThresholdViolation(violations) + + log.info("--- All threshold checks PASSED for combo '%s' ---", combo_key) + request.node.user_properties.append(("threshold_comparison", "PASSED")) + + +def test_loss_curve( + orch, variant_config, micro_batch_size, global_batch_size, precision, train_res_dict, lifecycle, request +): + """Parametrized: slope-based loss curve check with PNG render.""" + if lifecycle.failed: + pytest.skip("a prior lifecycle stage failed") + + combo_key = request.node.callspec.id + if combo_key not in train_res_dict: + pytest.skip(f"no recorded results for combo '{combo_key}' (training did not run)") + + combo_log_dir = train_res_dict[combo_key].get("_combo_log_dir") + if not combo_log_dir: + log.warning("no log dir recorded for combo '%s'; skipping loss curve check", combo_key) + pytest.skip(f"no log dir recorded for combo '{combo_key}'") + + log_path = f"{combo_log_dir}/out-node0/training.log" + out_dict = orch.exec(f"cat {log_path}") + log_text = list(out_dict.values())[-1] or "" + + lc = variant_config.loss_curve + step_metrics = parse_all_loss_points(log_text) + points = sample_loss_curve(step_metrics, lc.sample_every, lc.milestone_steps) + + log.info("--- Loss curve check for combo '%s' (%d points sampled) ---", combo_key, len(points)) + + mgr = getattr(request.config, "_html_report_manager", None) + mgr_enabled = mgr is not None and getattr(mgr, "is_enabled", False) + out_dir = mgr.log_dir if mgr_enabled else "/tmp" + try: + from pathlib import Path as _Path + import uuid as _uuid + + _Path(out_dir).mkdir(parents=True, exist_ok=True) + fname = f"loss_curve_{combo_key}_{str(_uuid.uuid4()).split('-')[-1]}.png" + png_path = _Path(out_dir) / fname + title = f"Training Loss Curve — {variant_config.model_params.get('model_name', '')} [{combo_key}]" + rendered = render_loss_curve_png(points, png_path, title=title) + if rendered and mgr_enabled: + rel_path = str(_Path(rendered).relative_to(mgr.htmlpath.parent)) + lifecycle.add_artifact(request.node.nodeid, f"Loss Curve [{combo_key}]", rel_path, rendered) + except Exception as e: + log.warning("loss curve: could not render PNG (%s)", e) + + verdict = evaluate_loss_decreasing(points, lc.max_slope) + if verdict is None: + pytest.skip(f"loss curve needs >= 2 sampled points (got {len(points)}); increase training_iterations") + + decreasing, slope, detail = verdict + log.info("loss curve: %s", detail) + + if points: + request.node.user_properties.append(("metric_value", points[-1][1])) + request.node.user_properties.append(("metric_unit", "lm_loss")) + + if lc.enforce and not decreasing: + pytest.fail(f"training loss is not decreasing for combo '{combo_key}': {detail}") + + +def test_teardown(orch, lifecycle, request): + """Stage 5: tear down the container once after all combos have run.""" + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + t = time.monotonic() + orch.teardown_containers() + lifecycle.record(request.node.nodeid, "teardown", time.monotonic() - t) + if orch.verify_containers_running(name): + log.error("container %s still running after teardown_containers()", name) + else: + lifecycle.torn_down = True diff --git a/cvs/tests/training/torchtitan/conftest.py b/cvs/tests/training/torchtitan/conftest.py new file mode 100644 index 000000000..8c323eef2 --- /dev/null +++ b/cvs/tests/training/torchtitan/conftest.py @@ -0,0 +1,164 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. +The year included in the foregoing notice is the year of creation of the work. +All code contained here is Property of Advanced Micro Devices, Inc. +''' + +import json +import os + +import pytest + +from cvs.core.orchestrators.factory import OrchestratorConfig, OrchestratorFactory +from cvs.lib import globals +from cvs.lib.utils_lib import resolve_cluster_config_placeholders +from cvs.lib.training.torchtitan.training_config_loader import load_training_variant + +log = globals.log + + +def _deep_merge(base, override): + """Recursively merge `override` onto `base` (dicts merged key-wise, scalars/lists replaced). + + Protects cluster-set scalar and dict container keys from being wiped by a + top-level replace: they survive unless the training block overrides that same + key. List keys (e.g. runtime.args, volumes) are replaced here and recombined + additively downstream in container.py's getters. + """ + if not (isinstance(base, dict) and isinstance(override, dict)): + return override + out = dict(base) + for k, v in override.items(): + out[k] = _deep_merge(base[k], v) if k in base else v + return out + + +@pytest.fixture(scope="module") +def cluster_dict(pytestconfig): + cluster_file = pytestconfig.getoption("cluster_file") + if not cluster_file: + pytest.fail("--cluster_file is required") + with open(cluster_file) as fp: + d = json.load(fp) + return resolve_cluster_config_placeholders(d) + + +@pytest.fixture(scope="module") +def variant_config(pytestconfig, cluster_dict): + config_file = pytestconfig.getoption("config_file") + if not config_file: + pytest.fail("--config_file is required") + return load_training_variant(config_file, cluster_dict) + + +@pytest.fixture(scope="module") +def hf_token(variant_config): + path = variant_config.config['hf_token_file'] + if not os.path.isfile(path): + pytest.skip(f"hf_token file missing: {path}") + with open(path) as fp: + return fp.read().strip() + + +class _Lifecycle: + """Cross-test state for the per-combo lifecycle model. + + Each combo's container launch, training, and teardown are timed sub-stages + of test_training. `report` maps each nodeid to its recorded (label, value, + unit) rows, which pytest_runtest_makereport renders into the HTML detail + panel. `torn_down` suppresses the orch fixture leak-guard: test_training + sets it True after its own teardown so the module-end finalizer does not + tear down a second time. + """ + + def __init__(self): + self.torn_down = False + self.report = {} # nodeid -> list[(label, value, unit)] + + def record(self, nodeid, label, value, unit="s"): + self.report.setdefault(nodeid, []).append((label, value, unit)) + + +@pytest.fixture(scope="module") +def lifecycle(): + return _Lifecycle() + + +@pytest.fixture(scope="module") +def train_res_dict(): + return {} + + +@pytest.fixture(scope="module") +def orch(cluster_dict, variant_config, lifecycle): + """Construct a ContainerOrchestrator and own a final teardown safety net. + + Each combo's test_training launches and tears down its own container in a + finally block, setting lifecycle.torn_down=True afterwards. This finalizer + only fires when torn_down is False -- i.e. a combo crashed hard before its + own teardown ran -- so nothing leaks past the module without double-tearing + down in the normal case. + """ + container_block = _deep_merge(cluster_dict.get("container", {}), variant_config.container.model_dump()) + testsuite_config = {"orchestrator": "container", "container": container_block} + cfg = OrchestratorConfig.from_configs(cluster_dict, testsuite_config) + o = OrchestratorFactory.create_orchestrator(log, cfg) + yield o + if not lifecycle.torn_down: + log.info("orch fixture leak-guard: tearing down container (per-combo teardown did not run)") + o.teardown_containers() + + +def pytest_collection_modifyitems(items): + """Pin test order: each combo's test_training (which owns the full container + lifecycle) runs before any test_throughput, which only reads saved results.""" + rank = { + "test_training": 0, + "test_throughput": 1, + } + items.sort(key=lambda it: rank.get(it.originalname or it.name.split("[")[0], 99)) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + """Attach this test's recorded timing rows to its HTML report detail panel.""" + outcome = yield + report = outcome.get_result() + if report.when != "call": + return + lc = item.funcargs.get("lifecycle") + rows = getattr(lc, "report", {}).get(item.nodeid) if lc else None + if not rows: + return + try: + import pytest_html + except ImportError: + return + body = "".join(f"{label}{value:.1f}{unit}" for label, value, unit in rows) + html = f"{body}
stagevalueunit
" + extras = getattr(report, "extras", []) + extras.append(pytest_html.extras.html(html)) + report.extras = extras + + +def pytest_html_results_table_header(cells): + cells.insert(-1, "Value") + cells.insert(-1, "Unit") + + +def pytest_html_results_table_row(report, cells): + props = dict(report.user_properties) + has = "metric_value" in props + val = props.get("metric_value") + unit = props.get("metric_unit", "") if has else "" + if not has: + shown = "" + elif val is None: + shown = "-" + elif isinstance(val, float): + shown = f"{val:.3f}" + else: + shown = str(val) + cells.insert(-1, f"{shown}") + cells.insert(-1, f"{unit}") diff --git a/cvs/tests/training/torchtitan/torchtitan_distributed.py b/cvs/tests/training/torchtitan/torchtitan_distributed.py new file mode 100644 index 000000000..aad0987cd --- /dev/null +++ b/cvs/tests/training/torchtitan/torchtitan_distributed.py @@ -0,0 +1,231 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. +The year included in the foregoing notice is the year of creation of the work. +All code contained here is Property of Advanced Micro Devices, Inc. + +Parametrized TorchTitan distributed (multi-node) training suite. +One config per model; sweep.combinations + sweep.runs drive parametrization. + +Each sweep combo runs in its OWN freshly-launched container set: launch -> train -> +verify -> save results -> teardown. Combos never share port 6000, log files, or +scripts dir, and each combo's dmesg/verify window is scoped to its own run. +The image is pulled only on the first launch (cached thereafter), so recycling +the containers per combo is cheap. +''' + +import json +import os +import re +import time + +import pytest + +from cvs.lib import globals +from cvs.lib.training.torchtitan import torchtitan_lib +from cvs.lib.utils.verdict import evaluate_all +from cvs.lib.utils_lib import update_test_result + +log = globals.log + + +def pytest_generate_tests(metafunc): + """Parametrize micro_batch_size and global_batch_size from sweep.combinations filtered by sweep.runs. + + sweep.combinations is a dict of {run_id: {micro_batch_size, global_batch_size, ...}}. + sweep.runs is a list of run_ids to execute (subset or all). + One case is emitted per entry in sweep.runs — no cartesian product. + """ + config_file = metafunc.config.getoption("config_file") + if not config_file or not os.path.isfile(config_file): + return + with open(config_file) as fp: + raw = json.load(fp) + + sweep = raw.get("sweep", {}) + combinations = sweep.get("combinations", {}) + runs = sweep.get("runs", list(combinations.keys())) + + cases = [] + ids = [] + for run_id in runs: + if run_id not in combinations: + log.warning("sweep.runs entry '%s' not found in sweep.combinations; skipping", run_id) + continue + combo = combinations[run_id] + mbs = combo["micro_batch_size"] + gbs = combo["global_batch_size"] + precision = combo.get("precision", "") + result_dict = combo.get("result_dict", {}) + cases.append((mbs, gbs, precision, result_dict)) + ids.append(combo.get("name", run_id)) + + if "micro_batch_size" in metafunc.fixturenames and "global_batch_size" in metafunc.fixturenames and cases: + metafunc.parametrize("micro_batch_size,global_batch_size,precision,result_dict", cases, ids=ids) + + +def test_training( + orch, + variant_config, + hf_token, + micro_batch_size, + global_batch_size, + precision, + result_dict, + train_res_dict, + lifecycle, + request, +): + """Run the full per-combo lifecycle in a dedicated container set. + + Launches fresh containers for this combo, runs distributed TorchTitan training + for the given micro_batch_size / global_batch_size across all nodes, verifies + and stores the results, then ALWAYS tears the containers down (finally) so the + next combo starts on a clean cluster — freeing port 6000, the training log, + and the scripts dir. The image is pulled only on the first launch (cached + afterwards), so relaunch per combo is cheap. + + Model-level params (tp, pp, precision, etc.) come from variant_config.model_params. + Each container-lifecycle sub-stage is timed via lifecycle.record so it shows + up in this test's HTML detail panel. + """ + nodeid = request.node.nodeid + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + + # A container set is about to exist; the orch leak-guard should own cleanup until + # this combo's own teardown (finally) confirms it is gone. + lifecycle.torn_down = False + + try: + # Stage 0: disable firewall — required for distributed runs to avoid + # inter-node MPI threads timing out against the Rendezvous endpoint. + # Runs on baremetal (orch.all) before containers are launched. + t = time.monotonic() + out_dict = orch.all.exec("sudo service ufw status") + for node, out in (out_dict or {}).items(): + if not re.search("inactive", out or "", re.I): + orch.all.exec("sudo service ufw stop") + out_dict = orch.all.exec("sudo ufw status") + for node, out in (out_dict or {}).items(): + if not re.search("inactive|disabled", out or "", re.I): + pytest.fail(f"failed to disable firewall on node {node}") + lifecycle.record(nodeid, "firewall_disable", time.monotonic() - t) + + # Stage 1: launch fresh containers for this combo. + t = time.monotonic() + ok = orch.setup_containers() + lifecycle.record(nodeid, "container_launch", time.monotonic() - t) + if not ok: + pytest.fail(f"setup_containers() returned False for {name}") + if not orch.verify_containers_running(name): + pytest.fail(f"container {name} not running after setup_containers()") + + # Stage 2: start sshd. TorchTitan uses torchrun with c10d (not MPI), + # but we set up sshd for consistency with other training frameworks. + t = time.monotonic() + ok = orch.setup_sshd() + lifecycle.record(nodeid, "sshd_setup", time.monotonic() - t) + if not ok: + pytest.fail("setup_sshd() returned False") + probe = orch.exec("bash -c 'ss -ltn 2>/dev/null | grep -q :2224 && echo OK || echo NO'") + if not any("OK" in (v or "") for v in (probe or {}).values()): + pytest.fail("sshd not listening on 2224 after setup_sshd()") + + # Stage 3: download HF model assets (TorchTitan-specific). + # Creates a temporary TorchTitanTrainingJob just for downloading. + # Idempotent - skips if already present. + globals.error_list = [] + tt_obj_download = torchtitan_lib.TorchTitanTrainingJob( + orch, + variant_config, + hf_token, + micro_batch_size=micro_batch_size, + global_batch_size=global_batch_size, + precision=precision, + result_dict=result_dict, + distributed_training=True, + tune_model_params=False, + run_label=request.node.callspec.id, + ) + + t = time.monotonic() + tt_obj_download.download_hf_assets() + lifecycle.record(nodeid, "model_download", time.monotonic() - t) + + # Stage 4: training. + globals.error_list = [] + tt_obj = torchtitan_lib.TorchTitanTrainingJob( + orch, + variant_config, + hf_token, + micro_batch_size=micro_batch_size, + global_batch_size=global_batch_size, + precision=precision, + result_dict=result_dict, + distributed_training=True, + tune_model_params=False, + run_label=request.node.callspec.id, + ) + + t = time.monotonic() + tt_obj.run_pretraining_tasks() + tt_obj.build_training_job_cmd() + tt_obj.start_training_job() + tt_obj.poll_for_training_completion() + tt_obj.verify_training_results() + elapsed = time.monotonic() - t + + lifecycle.record(nodeid, "training", elapsed) + request.node.user_properties.append(("metric_value", elapsed)) + request.node.user_properties.append(("metric_unit", "s")) + + combo_key = request.node.callspec.id + train_res_dict[combo_key] = tt_obj.training_results_dict + update_test_result() + finally: + # Teardown — always recycle the containers so the next combo starts on a + # clean cluster even if a stage above failed. + t = time.monotonic() + orch.teardown_containers() + lifecycle.record(nodeid, "teardown", time.monotonic() - t) + if orch.verify_containers_running(name): + log.error("container %s still running after teardown_containers()", name) + else: + lifecycle.torn_down = True + + +def test_throughput( + variant_config, micro_batch_size, global_batch_size, precision, result_dict, train_res_dict, lifecycle, request +): + """Threshold check using variant_config.cell_key() and thresholds. + + Uses cell_key() format: MBS=,GBS=,PRECISION= + Thresholds loaded from external *_threshold.json file via variant_config.thresholds + Supports both new thresholds (preferred) and legacy result_dict (backwards compat) + """ + combo_key = request.node.callspec.id + if combo_key not in train_res_dict: + pytest.skip(f"no recorded results for combo {combo_key} (training did not run)") + + if not variant_config.enforce_thresholds: + log.info("enforce_thresholds=false; recorded metrics for combo %s, skipping verdict", combo_key) + return + + # Use new cell_key format for threshold lookup + cell_key = variant_config.cell_key(combo_key) + + # Prefer external thresholds, fallback to legacy result_dict + if variant_config.thresholds: + if cell_key not in variant_config.thresholds: + log.warning("no threshold entry for cell %s; skipping", cell_key) + return + threshold_specs = variant_config.thresholds[cell_key] + elif result_dict: + # Legacy mode: inline result_dict - convert to threshold spec format + threshold_specs = {f"training.{k}": {"kind": "min", "value": v} for k, v in result_dict.items()} + else: + log.warning("no thresholds defined for combo %s; skipping threshold checks", combo_key) + return + + # Evaluate thresholds (raises ThresholdViolation on failure) + evaluate_all(train_res_dict[combo_key], threshold_specs) diff --git a/cvs/tests/training/torchtitan/torchtitan_single.py b/cvs/tests/training/torchtitan/torchtitan_single.py new file mode 100644 index 000000000..34363b133 --- /dev/null +++ b/cvs/tests/training/torchtitan/torchtitan_single.py @@ -0,0 +1,221 @@ +''' +Copyright 2025 Advanced Micro Devices, Inc. +All rights reserved. This notice is intended as a precaution against inadvertent publication and does not imply publication or any waiver of confidentiality. +The year included in the foregoing notice is the year of creation of the work. +All code contained here is Property of Advanced Micro Devices, Inc. + +Parametrized TorchTitan single-node training suite. +One config per model; sweep.combinations + sweep.runs drive parametrization. + +Each sweep combo runs in its OWN freshly-launched container: launch -> train -> +verify -> save results -> teardown. Combos never share port 6000, log files, or +scripts dir, and each combo's dmesg/verify window is scoped to its own run. +The image is pulled only on the first launch (cached thereafter), so recycling +the container per combo is cheap. +''' + +import json +import os +import time + +import pytest + +from cvs.lib import globals +from cvs.lib.training.torchtitan import torchtitan_lib +from cvs.lib.utils.verdict import evaluate_all +from cvs.lib.utils_lib import update_test_result + +log = globals.log + + +def pytest_generate_tests(metafunc): + """Parametrize micro_batch_size and global_batch_size from sweep.combinations filtered by sweep.runs. + + sweep.combinations is a dict of {run_id: {micro_batch_size, global_batch_size, ...}}. + sweep.runs is a list of run_ids to execute (subset or all). + One case is emitted per entry in sweep.runs — no cartesian product. + """ + config_file = metafunc.config.getoption("config_file") + if not config_file or not os.path.isfile(config_file): + return + with open(config_file) as fp: + raw = json.load(fp) + + sweep = raw.get("sweep", {}) + combinations = sweep.get("combinations", {}) + runs = sweep.get("runs", list(combinations.keys())) + + cases = [] + ids = [] + for run_id in runs: + if run_id not in combinations: + log.warning("sweep.runs entry '%s' not found in sweep.combinations; skipping", run_id) + continue + combo = combinations[run_id] + mbs = combo["micro_batch_size"] + gbs = combo["global_batch_size"] + precision = combo.get("precision", "") + result_dict = combo.get("result_dict", {}) + cases.append((mbs, gbs, precision, result_dict)) + ids.append(combo.get("name", run_id)) + + if "micro_batch_size" in metafunc.fixturenames and "global_batch_size" in metafunc.fixturenames and cases: + metafunc.parametrize("micro_batch_size,global_batch_size,precision,result_dict", cases, ids=ids) + + +def test_training( + orch, + variant_config, + hf_token, + micro_batch_size, + global_batch_size, + precision, + result_dict, + train_res_dict, + lifecycle, + request, +): + """Run the full per-combo lifecycle in a dedicated container. + + Launches a fresh container for this combo, runs single-node TorchTitan + training for the given micro_batch_size / global_batch_size, verifies and + stores the results, then ALWAYS tears the container down (finally) so the + next combo starts on a clean node — freeing port 6000, the training log, and + the scripts dir. The image is pulled only on the first launch (cached + afterwards), so relaunch per combo is cheap. + + Model-level params (tp, pp, precision, etc.) come from variant_config.model_params. + Each container-lifecycle sub-stage is timed via lifecycle.record so it shows + up in this test's HTML detail panel. + """ + nodeid = request.node.nodeid + name = orch.get_container_name(orch.container_config, orch.container_config["image"]) + + # A container is about to exist; the orch leak-guard should own cleanup until + # this combo's own teardown (finally) confirms it is gone. + lifecycle.torn_down = False + + try: + # Stage 1: launch a fresh container for this combo (was test_launch_container). + t = time.monotonic() + ok = orch.setup_containers() + lifecycle.record(nodeid, "container_launch", time.monotonic() - t) + if not ok: + pytest.fail(f"setup_containers() returned False for {name}") + if not orch.verify_containers_running(name): + pytest.fail(f"container {name} not running after setup_containers()") + + # Stage 2: start sshd (was test_setup_sshd). Single-node runs skip + # starting the in-container sshd (it exists only for inter-node MPI), so + # only probe 2224 when there is more than one host. + t = time.monotonic() + ok = orch.setup_sshd() + lifecycle.record(nodeid, "sshd_setup", time.monotonic() - t) + if not ok: + pytest.fail("setup_sshd() returned False") + if len(orch.hosts) > 1: + probe = orch.exec("bash -c 'ss -ltn 2>/dev/null | grep -q :2224 && echo OK || echo NO'") + if not any("OK" in (v or "") for v in (probe or {}).values()): + pytest.fail("sshd not listening on 2224 after setup_sshd()") + + # Stage 3: download HF model assets (TorchTitan-specific). + # Creates a temporary TorchTitanTrainingJob just for downloading. + # Idempotent - skips if already present. + globals.error_list = [] + tt_obj_download = torchtitan_lib.TorchTitanTrainingJob( + orch, + variant_config, + hf_token, + micro_batch_size=micro_batch_size, + global_batch_size=global_batch_size, + precision=precision, + result_dict=result_dict, + distributed_training=False, + tune_model_params=False, + run_label=request.node.callspec.id, + ) + + t = time.monotonic() + tt_obj_download.download_hf_assets() + lifecycle.record(nodeid, "model_download", time.monotonic() - t) + + # Stage 4: training. + globals.error_list = [] + tt_obj = torchtitan_lib.TorchTitanTrainingJob( + orch, + variant_config, + hf_token, + micro_batch_size=micro_batch_size, + global_batch_size=global_batch_size, + precision=precision, + result_dict=result_dict, + distributed_training=False, + tune_model_params=False, + run_label=request.node.callspec.id, + ) + + t = time.monotonic() + tt_obj.run_pretraining_tasks() + tt_obj.exec_nic_setup_scripts() + tt_obj.build_training_job_cmd() + tt_obj.start_training_job() + tt_obj.poll_for_training_completion() + tt_obj.verify_training_results() + elapsed = time.monotonic() - t + + lifecycle.record(nodeid, "training", elapsed) + request.node.user_properties.append(("metric_value", elapsed)) + request.node.user_properties.append(("metric_unit", "s")) + + combo_key = request.node.callspec.id + train_res_dict[combo_key] = tt_obj.training_results_dict + update_test_result() + finally: + # Teardown (was test_teardown) — always recycle the container so the next + # combo starts on a clean node even if a stage above failed. + t = time.monotonic() + orch.teardown_containers() + lifecycle.record(nodeid, "teardown", time.monotonic() - t) + if orch.verify_containers_running(name): + log.error("container %s still running after teardown_containers()", name) + else: + # This combo's container is gone; suppress the module-end leak-guard + # so it does not tear down a second time. + lifecycle.torn_down = True + + +def test_throughput( + variant_config, micro_batch_size, global_batch_size, precision, result_dict, train_res_dict, lifecycle, request +): + """Threshold check using variant_config.cell_key() and threshold_dict. + + Uses cell_key() format: MBS=,GBS=,PRECISION= + Thresholds loaded from external *_threshold.json file via variant_config.threshold_dict + Supports both new threshold_dict (preferred) and legacy result_dict (backwards compat) + """ + combo_key = request.node.callspec.id + if combo_key not in train_res_dict: + pytest.skip(f"no recorded results for combo {combo_key} (training did not run)") + + if not variant_config.enforce_thresholds: + log.info("enforce_thresholds=false; recorded metrics for combo %s, skipping verdict", combo_key) + return + + # Use new cell_key format for threshold lookup + cell_key = variant_config.cell_key(combo_key) + + # Prefer external thresholds, fallback to legacy result_dict + if variant_config.thresholds: + if cell_key not in variant_config.thresholds: + log.warning("no threshold entry for cell %s; skipping", cell_key) + return + threshold_specs = variant_config.thresholds[cell_key] + elif result_dict: + # Legacy mode: inline result_dict - convert to threshold spec format + threshold_specs = {f"training.{k}": {"kind": "min", "value": v} for k, v in result_dict.items()} + else: + log.warning("no thresholds defined for combo %s; skipping threshold checks", combo_key) + return + + # Evaluate thresholds (raises ThresholdViolation on failure) + evaluate_all(train_res_dict[combo_key], threshold_specs) diff --git a/docs/how-to/run-cvs-tests.rst b/docs/how-to/run-cvs-tests.rst index 34cbf83c2..962ea734b 100644 --- a/docs/how-to/run-cvs-tests.rst +++ b/docs/how-to/run-cvs-tests.rst @@ -41,8 +41,8 @@ You can list available tests using either `cvs run` (with no arguments) or `cvs • ib_perf_bw_test • install_ibperf_tools - cvs.tests.inference.inferencemax (1 test suite) - • inferencemax_gpt_oss_120b_single + cvs.tests.inference.atom (1 test suite) + • atom cvs.tests.inference.pytorch_xdit (2 test suites) • pytorch_xdit_flux1_dev_single @@ -52,11 +52,8 @@ You can list available tests using either `cvs run` (with no arguments) or `cvs • sglang_deepseek_r1_671b_distributed • sglang_llama_70b_distributed - cvs.tests.inference.vllm (4 test suites) - • vllm_deepseek31_685b_single - • vllm_gpt_oss_120b_single - • vllm_qwen3_235b_single - • vllm_qwen3_80b_single + cvs.tests.inference.vllm (1 test suite) + • vllm cvs.tests.mori (1 test suite) • mori_benchmark_test @@ -80,7 +77,7 @@ You can list available tests using either `cvs run` (with no arguments) or `cvs • megatron_llama3_1_8b_single ================================================================================ - Total: 34 test suites across 1 package(s) + Total: 29 test suites across 1 package(s) Run all tests in a file: @@ -628,27 +625,39 @@ Use these scripts to run the Mori tests. cvs run mori_benchmark_test --cluster_file input/cluster_file/cluster.json --config_file input/config_file/mori/mi35x_mori_config.json --html=/var/www/html/cvs/mori.html --capture=tee-sys --self-contained-html --log-file=/tmp/mori.log -vvv -s -Inferencemax test scripts +ATOM test scripts ------------------------------ -You can list all available Inferencemax test cases using the CLI: +You can list all available ATOM test cases using the CLI: .. code:: bash - cvs list inferencemax_gpt_oss_120b_single + cvs list atom .. code:: text - Available tests in inferencemax_gpt_oss_120b_single: - - test_cleanup_stale_containers - - test_gpt_oss_120_single_node - - test_launch_inference_containers + Available tests in atom: + - test_launch_container + - test_atom_inference + - test_print_results_table + - test_teardown -Use these scripts to run the Inferencemax tests. +Use these scripts to run the ATOM tests. Supply your own suite JSON +(``schema_version: 1`` variant config); see :doc:`../reference/configuration-files/atom`. +After ``cvs copy-config``, keep **one** ``*threshold.json`` in the same directory as the +``--config_file`` you pass (per-variant subdirs under ``~/input/.../atom/``). +Copy-paste lab commands: ``cvs/input/config_file/inference/atom/README.md``. .. code:: bash - cvs run inferencemax_gpt_oss_120b_single --cluster_file input/cluster_file/cluster.json --config_file input/config_file/inference/inferencemax/mi300x_inferencemax_gpt_oss_120b_single.json --html=/var/www/html/cvs/inferencemax.html --capture=tee-sys --self-contained-html --log-file=/tmp/inferencemax.log -vvv -s + TS=$(date +%Y%m%d_%H%M%S) + cvs run atom \ + --cluster_file ~/input/cluster_file/atom_cluster.json \ + --config_file ~/input/config_file/inference/atom/single/mi300x_atom_deepseek-r1_fp8_single.json \ + --html=~/cvs_results/${TS}_atom-single_mi300x.html \ + --self-contained-html \ + --log-file=~/cvs_results/${TS}_atom-single_mi300x.log \ + -vvv -s Pytorch xdit test scripts @@ -748,73 +757,45 @@ Use these scripts to run the Sglang tests. VLLM test scripts ------------------------------ -You can list all available VLLM test cases using the CLI: +vLLM benchmarks use one parametrized suite, ``vllm``, covering both single-node and +multinode runs — the topology comes from the configuration file, not from the suite name. +Configuration files live in ``cvs/input/config_file/inference/vllm/``, each with a sibling +threshold file (see :func:`cvs.lib.inference.utils.vllm_config_loader.load_variant`, or +:func:`cvs.lib.inference.atom.atom_config_loader.load_variant` for ATOM). Point +``--config_file`` at one of them and ``--cluster_file`` at a cluster JSON that matches +your hardware. .. code:: bash - cvs list vllm_deepseek31_685b_single + cvs list vllm .. code:: text - Available tests in vllm_deepseek31_685b_single: - - test_cleanup_stale_containers - - test_launch_inference_containers - - test_print_results_table - - test_vllm_inference + Available tests in vllm: + • test_accuracy_eval + • test_discover_topology + • test_gpu_metric + • test_launch_container + • test_metric + • test_model_fetch + • test_openai_compatible_smoke + • test_print_results_table + • test_prom_metric + • test_setup_sshd + • test_teardown + • test_vllm_inference -.. code:: bash - - cvs list vllm_gpt_oss_120b_single +At run time, ``test_vllm_inference`` and the metric tests are parametrized per sweep cell, +producing names such as ``test_vllm_inference[balanced-conc64]`` from the ``sweep`` block in +your configuration file. -.. code:: text - - Available tests in vllm_gpt_oss_120b_single: - - test_cleanup_stale_containers - - test_launch_inference_containers - - test_print_results_table - - test_vllm_inference - .. code:: bash - cvs list vllm_qwen3_235b_single - -.. code:: text - - Available tests in vllm_qwen3_235b_single: - - test_cleanup_stale_containers - - test_launch_inference_containers - - test_print_results_table - - test_vllm_inference - -.. code:: bash - - cvs list vllm_qwen3_80b_single - -.. code:: text - - Available tests in vllm_qwen3_80b_single: - - test_cleanup_stale_containers - - test_launch_inference_containers - - test_print_results_table - - test_vllm_inference - -Use these scripts to run the VLLM tests. - -.. code:: bash - - cvs run vllm_deepseek31_685b_single --cluster_file input/cluster_file/cluster.json --config_file input/config_file/inference/vllm/mi355x_vllm_single.json --html=/var/www/html/cvs/deepseek.html --capture=tee-sys --self-contained-html --log-file=/tmp/deepseek.log -vvv -s - -.. code:: bash - - cvs run vllm_gpt_oss_120b_single --cluster_file input/cluster_file/cluster.json --config_file input/config_file/inference/vllm/mi355x_vllm_single.json --html=/var/www/html/cvs/gpt.html --capture=tee-sys --self-contained-html --log-file=/tmp/gpt.log -vvv -s - -.. code:: bash - - cvs run vllm_qwen3_235b_single --cluster_file input/cluster_file/cluster.json --config_file input/config_file/inference/vllm/mi355x_vllm_single.json --html=/var/www/html/cvs/qwen235.html --capture=tee-sys --self-contained-html --log-file=/tmp/qwen235.log -vvv -s - -.. code:: bash + cvs run vllm --cluster_file input/cluster_file/cluster_container.json --config_file input/config_file/inference/vllm/mi300x_vllm_llama31-70b_fp8_single.json --html=/var/www/html/cvs/vllm.html --capture=tee-sys --self-contained-html --log-file=/tmp/vllm.log -vvv -s - cvs run vllm_qwen3_80b_single --cluster_file input/cluster_file/cluster.json --config_file input/config_file/inference/vllm/mi355x_vllm_single.json --html=/var/www/html/cvs/qwen80.html --capture=tee-sys --self-contained-html --log-file=/tmp/qwen80.log -vvv -s +For the full configuration schema, metrics, and thresholds see +:doc:`/reference/configuration-files/vllm`; for a step-by-step first run including multinode, +see :doc:`/how-to/run-vllm-benchmarks`. Test results diff --git a/docs/how-to/run-vllm-benchmarks.rst b/docs/how-to/run-vllm-benchmarks.rst new file mode 100644 index 000000000..b652e6c1b --- /dev/null +++ b/docs/how-to/run-vllm-benchmarks.rst @@ -0,0 +1,222 @@ +.. meta:: + :description: Run vLLM inference benchmarks with CVS, single-node and multinode + :keywords: CVS, vLLM, inference, benchmark, multinode, ray, LLM, ROCm + +***************************** +Run vLLM inference benchmarks +***************************** + +The vLLM suite measures LLM serving throughput, latency, and accuracy on AMD Instinct GPUs. One parametrized suite covers both single-node and multinode runs — you select the topology in the configuration file, not by choosing a different suite. + +This page walks through a first run. For the full schema, every metric, and the threshold grammar, see :doc:`/reference/configuration-files/vllm`. + +Prerequisites +============= + +On every cluster node: + +- **Docker** installed, with the SSH user able to run it (passwordless ``sudo docker`` or membership in the ``docker`` group). +- **Host driver** loaded, so ``/dev/kfd``, ``/dev/dri/*``, and ``/dev/infiniband/*`` are present for passthrough. +- **A vLLM image** either already loaded or pullable from a reachable registry. It must contain ``vllm`` on the path — the suite invokes ``vllm serve`` and ``vllm bench serve`` inside the container. +- **Model weights** staged on a shared filesystem that every node mounts at the same path. Remote model download is not implemented, so weights must be present before the run. +- **A Hugging Face token file**, if the model needs one. A pre-staged model can run without it. + +On the head node where you launch ``cvs run``: + +- CVS installed (see :doc:`/install/cvs-install`). +- SSH key-based access to every cluster node. + +For a multinode run, additionally have on hand: + +- The **head node address** reachable from every worker. +- The **network interface name** used for inter-node traffic, for example ``ens51f1np1``. Find it with ``ip -br addr`` on a node. CVS cannot derive this automatically. + +Step 1: Copy a configuration file +================================= + +CVS ships single-node and distributed vLLM configurations. List them: + +.. code:: bash + + cvs copy-config inference/vllm + +Copy the one that matches your topology: + +.. code:: bash + + # Single node + cvs copy-config inference/vllm/mi300x_vllm_llama31-70b_fp8_single.json \ + --output /tmp/cvs/vllm_singlenode_config.json + + # Multiple nodes + cvs copy-config inference/vllm/mi300x_vllm_llama31-70b_fp8_distributed.json \ + --output /tmp/cvs/vllm_multinode_config.json + +You also need a cluster file describing your nodes. Use the container template, since the vLLM suite always runs inside a container: + +.. code:: bash + + cvs copy-config cluster_container.json --output /tmp/cvs/cluster.json + +Step 2: Fill in the placeholders +================================ + +Every value marked ```` must be replaced before the run. + +In the **cluster file**, set your SSH user, private key path, and node addresses. See :doc:`/how-to/run-with-containers` for a walkthrough. + +In the **configuration file**, set: + +- ``container.image`` — your vLLM image. Cite the full tag; do not abbreviate it. +- ``paths.shared_fs`` — the shared filesystem root. The other paths derive from it by default. +- ``paths.models_dir`` — where the weights live. Make sure this path is also mounted into the container by ``container.runtime.args.volumes``. +- ``paths.hf_token_file`` — path to your token file. +- ``model.id`` — the model to serve. + +For a **multinode** configuration, also set: + +- ``params.master_addr`` — the head node address. +- ``roles.server.ib_netdev`` — the interface name you looked up in the prerequisites. + +.. tip:: + + Leave ``enforce_thresholds`` set to ``false`` for your first run on new hardware. The run then measures and records everything without failing on thresholds you have not calibrated yet. Set it to ``true`` once you know what good looks like. + +Step 3: Run the suite +===================== + +.. code:: bash + + cvs run vllm \ + --cluster_file /tmp/cvs/cluster.json \ + --config_file /tmp/cvs/vllm_singlenode_config.json \ + --html /tmp/cvs/vllm.html --self-contained-html \ + --log-file /tmp/cvs/cvs.log + +The suite name is ``vllm`` for every topology. There is no separate distributed suite — a multinode run is the same command with a multinode configuration file: + +.. code:: bash + + cvs run vllm \ + --cluster_file /tmp/cvs/cluster.json \ + --config_file /tmp/cvs/vllm_multinode_config.json \ + --html /tmp/cvs/vllm.html --self-contained-html \ + --log-file /tmp/cvs/cvs.log + +.. note:: + + ``--self-contained-html`` only takes effect together with ``--html``. Always pass both, so the report is a single file you can attach or copy off the cluster. + + Any flag CVS does not recognize is passed straight through to pytest, so options such as ``-vvv`` and ``--capture=tee-sys`` work as usual. + +Step 4: Read the results +======================== + +Open the HTML report. Each lifecycle stage and each metric is its own row: + +- **Lifecycle rows** — container launch, topology discovery, model fetch, the OpenAI-compatible smoke test, then teardown. These tell you *how far* the run got. +- **Inference rows** — one per sweep cell, labelled ``-conc``. +- **Metric rows** — one per metric per cell. A metric that could not be measured is skipped rather than failed. +- **Results table** — the summary near the end, also printed to the console. This is where you read the measured numbers. + +Per-cell logs land under your configured ``log_dir``:: + + /vllm/out-node/isl_osl_conc/ + vllm_serve_server.log <- the server's own log + client.log <- the load generator + results <- raw benchmark JSON + +.. important:: + + When a run fails, read ``vllm_serve_server.log`` on the node, not just the CVS client log. Some faults — GPU exceptions, weight-loading failures, out-of-memory kills — appear only in the server log. + +A skipped ``test_setup_sshd`` row is expected. vLLM communicates over the host network and needs no inter-container sshd. + +Going multinode +=============== + +Three settings turn a single-node configuration into a multinode one: ``params.nnodes``, ``params.pipeline_parallel_size``, and ``roles.server.ib_netdev``. Which combination is valid depends on the distributed executor backend. + +Using the default backend (mp) +------------------------------ + +If you set nothing else, the suite uses ``mp``. It requires pipeline parallelism across the nodes: + +.. code:: json + + { + "params": { + "tensor_parallelism": "8", + "pipeline_parallel_size": "2", + "nnodes": "2", + "master_addr": "10.0.0.1" + }, + "roles": { + "server": { + "ib_netdev": "ens51f1np1" + } + } + } + +CVS launches ``vllm serve`` on every node with the correct rank, adding ``--headless`` to every rank above 0. + +Using ray +--------- + +Ray is opt-in. Add it to ``serve_args``: + +.. code:: json + + { + "roles": { + "server": { + "serve_args": { + "distributed-executor-backend": "ray" + }, + "ib_netdev": "ens51f1np1" + } + }, + "params": { + "tensor_parallelism": "8", + "pipeline_parallel_size": "1", + "nnodes": "2", + "master_addr": "10.0.0.1" + } + } + +CVS then bootstraps a Ray cluster before serving: ``ray start --head`` on rank 0, ``ray start --address=...`` on each worker, and ``ray stop`` at teardown. Only the head node runs ``vllm serve``, so worker nodes produce no server log. + +.. note:: + + Ray is not required for multinode — ``mp`` is the default and works across nodes. What ray changes is that it **removes the pipeline-parallelism requirement**, so ``pipeline_parallel_size`` of 1 becomes valid. Use ray when you want pure tensor-parallel serving across nodes; use the default otherwise. + + Only the exact lowercase string ``"ray"`` selects it. ``"Ray"`` silently falls back to ``mp`` and then fails validation if ``pipeline_parallel_size`` is 1. + +Common pitfalls +=============== + +**The run fails immediately with a validation error.** Configuration files are validated before anything launches, and every block except ``container`` rejects unknown keys — so a misspelled key is a hard error rather than a silently ignored setting. Read the message: it names the offending key. + +**"nnodes=2 > 1 requires pipeline_parallel_size > 1".** You configured multiple nodes on the default ``mp`` backend without pipeline parallelism. Either raise ``pipeline_parallel_size``, or switch to ray. + +**"ib_netdev is required in roles.server when nnodes > 1".** Set it to the interface name. There is deliberately no ``"auto"`` value — it cannot be derived reliably from HCA names. + +**"Container image not specified in config".** ``container.image`` is empty. Watch for this specific trap: if your configuration file has a ``container`` block that omits ``image``, it overwrites the cluster file's image with an empty string. Set ``image`` in whichever file defines the block. + +**Container launch crashes with "too many values to unpack".** You placed ``env`` under ``container.runtime.args``. It belongs at the ``container`` top level. + +**A threshold fails with "missing from actuals".** The threshold gates a metric this run did not produce. The usual cause is ``params.metric_percentiles`` omitting a percentile that a threshold references — the default ``"50,90,95,99"`` covers all gated latency metrics. + +**Every metric row skips.** The benchmark produced no parseable results. Check ``client.log`` and the server log for the cell. + +**The sweep is slower than expected.** Cells that differ only in concurrency reuse the running server; changing ISL, OSL, TP, PP, or any server argument forces a restart and a weight reload. Ordering ``runs`` so concurrency varies fastest avoids needless reloads. + +**Using** ``runtime.name: "enroot"``. Enroot is registered but not implemented, and the run fails at container launch. Use ``docker``. + +See also +======== + +- :doc:`/reference/configuration-files/vllm` — full configuration schema, metrics, and thresholds +- :doc:`/reference/configuration-files/cluster-file` — cluster file schema +- :doc:`/how-to/run-with-containers` — container backend in depth +- :doc:`/how-to/run-cvs-tests` — running other CVS suites diff --git a/docs/install/cvs-install.rst b/docs/install/cvs-install.rst index a806ccf5f..14f2e2f1b 100644 --- a/docs/install/cvs-install.rst +++ b/docs/install/cvs-install.rst @@ -373,32 +373,35 @@ Inference CVS provides comprehensive inference testing configurations for various LLM serving frameworks and models. -**InferenceMAX (vLLM Benchmarking)** +**ATOM (vLLM Benchmarking)** -1. Copy the InferenceMAX configuration file: +1. Copy the ATOM configuration files (main ``*.json`` and optional sibling ``*_threshold.json``): - .. code:: bash +.. code:: bash - cvs copy-config inference/mi300x_singlenode_inferencemax.json --output ~/my_inferencemax_config.json + cvs copy-config inference/atom/mi300x_atom_gpt-oss-120b_bf16.json --output ~/my_atom_config.json + cvs copy-config inference/atom/mi300x_atom_gpt-oss-120b_bf16_threshold.json --output ~/my_atom_threshold.json -2. Edit the file and modify these parameters: +2. Edit the files and modify these parameters: - ``container_image``: Docker image with vLLM - ``nnodes``: Number of nodes in the cluster -**vLLM Single-Node (MI355X)** +**vLLM Inference** -1. Copy the vLLM single-node configuration file: +1. Copy the vLLM configuration file matching your topology: .. code:: bash - cvs copy-config inference/mi355x_singlenode_vllm.json --output ~/my_vllm_config.json + cvs copy-config inference/vllm/mi300x_vllm_llama31-70b_fp8_single.json --output ~/my_vllm_config.json + cvs copy-config inference/vllm/mi300x_vllm_llama31-70b_fp8_distributed.json --output ~/my_vllm_multinode_config.json 2. Edit the file and configure: - - ``container_image``: vLLM container for MI355X - - ``nnodes``: Number of nodes in the cluster - - ``data_cache_dir``: Model cache directory + - ``container.image``: Docker image with vLLM + - ``paths.shared_fs``: Shared filesystem root + - ``paths.models_dir``: Model weights directory + - ``params.nnodes``: Number of nodes in the cluster **SGLang Disaggregated Prefill-Decode** diff --git a/docs/reference/configuration-files/atom.rst b/docs/reference/configuration-files/atom.rst new file mode 100644 index 000000000..5374a5c3c --- /dev/null +++ b/docs/reference/configuration-files/atom.rst @@ -0,0 +1,192 @@ +.. meta:: :description: Configure the variables in the ATOM configuration files + :keywords: inference, ROCm, install, cvs, ATOM, ATOM + +*************************************** +ATOM inference configuration file +*************************************** + +ATOM tests validate LLM serving on AMD GPU clusters using the **ATOM** stack +(``atom.entrypoints.openai_server`` + ``atom.benchmarks.benchmark_serving``). W1 workloads +use ``params.driver: atom``. Multinode **pipeline parallel** (``PP=2``) uses a framework +coordinator: ``params.driver: vllm_atom`` (vLLM + ATOM ROCm kernels) or ``params.driver: sglang``. +A legacy ``params.driver: vllm`` path remains for GPT-OSS uplift only. + +The suite checks: + +- **Container orchestration**: Docker with ROCm; cluster + variant container blocks merged +- **Model serving**: ATOM OpenAI-compatible server, health + warmup probes +- **Performance metrics**: Throughput, per-GPU throughput, TTFT/TPOT (including p99/p95 tails) +- **Benchmarking**: Named ISL/OSL combos with explicit concurrency sweep cells +- **Result verification**: Tiered ``client.*`` thresholds when ``enforce_thresholds`` is true + +Configs use flat sibling pairs under +``cvs/input/config_file/inference/atom/``, matching ``inference/vllm/`` naming: +``{gpu}_atom_{model}_{precision}[_{mode}].json`` plus optional +``…_threshold.json``. Pass ``--config_file`` to the main JSON; +:func:`cvs.lib.utils.config_loader.substitute_config` discovers the sole sibling ``*threshold.json`` in the **config file's parent directory** when +``threshold_json`` is omitted. If that directory contains more than one ``*threshold.json``, +loading fails with an ambiguous-threshold ``ValueError``. + +**Lab ``~/input`` layout:** the repo keeps every variant flat in one tree, but after +``cvs copy-config`` you should place each run's config + threshold pair in a dedicated +subdirectory (for example ``~/input/.../atom/single/``) so only one +threshold file sits beside the config you pass to ``--config_file``. Alternatively set +``"threshold_json"`` in the config to an explicit path. See the in-tree README at +``cvs/input/config_file/inference/atom/README.md`` for copy-paste commands. + +**Cluster file:** use ``cvs/input/cluster_file/atom_cluster.json``. Edit ``node_dict`` so host count matches variant ``params.nnodes`` (one host for single-node sweeps; two for multinode). +Container ``name`` must match the variant (``atom_mi300x`` / ``atom_mi355x``); +the suite deep-merges variant ``container`` over the cluster file. + +**Launcher vs GPU node:** pytest and HTML/log output run on the host where you invoke +``cvs run``. :class:`cvs.core.orchestrators.container.ContainerOrchestrator` SSHes to +cluster nodes (``cluster_file`` ``mgmt_ip`` / ``node_dict``) and runs ``sudo docker`` there. +``paths.models_dir`` and the ATOM image must exist on the GPU node; ``priv_key_file`` and +``paths.hf_token_file`` are read on the launcher. Local Docker on the launcher is not required. + +**ATOM server CLI:** set ``roles.server.atom_args`` inline in the config (vLLM-style, analogous to +``roles.server.serve_args`` on ``vllm_single``). When ``params.driver`` is ``atom``, ``atom_args`` +is required. MTP3 variants also set ``params.bench_extra_args`` (for example ``--use-chat-template``). + +Pytest and HTML layout (atom) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. list-table:: + :widths: 10 35 55 + :header-rows: 1 + + * - Stage + - Test + - Notes + * - 1 + - ``test_launch_container`` + - Host Docker launch; records ``container_launch``. + * - 2 + - ``test_setup_sshd`` + - Multinode only; single-node skips sshd probe. + * - 3 + - ``test_model_fetch`` + - Ensures model bytes under ``paths.models_dir``. + * - 4 + - ``test_atom_inference`` + - One parametrized cell; server start (or reuse), bench, parse ``results.json``. + * - 5 + - ``test_cell_metrics`` + - One HTML row per **metric tier** per cell (throughput, ttft, tpot, health, record). + * - 6 + - ``test_print_results_table`` + - Session results grid from ``inf_res_dict``. + * - 7 + - ``test_teardown`` + - Explicit teardown; sets ``lifecycle.torn_down``. + +Example variant layout +====================== + +Each stem has ``.json`` (``schema_version: 1``, ``framework: atom``) +and sibling ``_threshold.json``. In the CVS source tree many stems share one directory; +on a lab machine, copy only the pair you need into a per-variant subdirectory (or set +``threshold_json``). See ``mi300x_atom_deepseek-r1_fp8_single.json`` +for the W1 MI300X reference. + +.. dropdown:: Example ``mi300x_atom_deepseek-r1_fp8_single_threshold.json`` (excerpt) + + .. code:: json + + { + "ISL=1024,OSL=1024,TP=8,CONC=128": { + "client.output_throughput": {"kind": "min_tok_s", "value": 2590.98}, + "client.per_gpu_throughput": {"kind": "min_tok_s", "value": 648.65}, + "client.p99_ttft_ms": {"kind": "max_ms", "value": 1834.3}, + "client.p95_tpot_ms": {"kind": "max_ms", "value": 53.76}, + "client.success_rate": {"kind": "min", "value": 1}, + "client.failed": {"kind": "max", "value": 0} + } + } + + Every member of :data:`cvs.lib.inference.atom.atom_parsing.GATED_METRICS` needs a + spec in each cell when ``enforce_thresholds`` is true. W1 perf gates include + ``per_gpu_throughput``, ``output_tput_per_gpu``, ``p99_ttft_ms``, and ``p95_tpot_ms``. + +Parameters +========== + +Top-level blocks follow the DTNI variant schema. ATOM-specific keys: + +.. list-table:: + :widths: 3 3 5 + :header-rows: 1 + + * - Block / key + - Example + - Description + * - ``framework`` + - ``atom`` + - Suite identifier for :func:`load_variant`. + * - ``gpu_arch`` + - ``mi300x`` + - Hardware profile for the variant. + * - ``roles.server.atom_args`` + - ``["-tp", "8", "--kv_cache_dtype", "fp8"]`` + - Inline ATOM ``openai_server`` CLI tokens after ``--model`` / ``--server-port``. + * - ``roles.server.serve_args`` + - ``{"kv-cache-dtype": "fp8", "enforce-eager": true}`` + - vLLM / vLLM-ATOM path (``params.driver: vllm`` or ``vllm_atom``); merged into ``vllm serve`` argv. + * - ``roles.server.sglang_args`` + - ``["--trust-remote-code", "--disable-cuda-graph"]`` + - Extra tokens appended to ``sglang.launch_server`` when ``params.driver: sglang``. + * - ``roles.server.ib_netdev`` + - ``ens51f1np1`` + - **Required** when ``nnodes > 1`` and ``driver`` is ``vllm_atom`` or ``sglang``; sets ``NCCL_SOCKET_IFNAME``. + * - ``enforce_thresholds`` + - ``true`` / ``false`` + - When true, ``test_cell_metrics`` asserts via :func:`cvs.lib.utils.verdict.evaluate_all`. + * - ``paths.*`` + - ``shared_fs``, ``models_dir`` (``/home/models``), ``log_dir``, ``hf_token_file`` + - ``models_dir`` is an absolute HF hub cache path on GPU nodes; variant configs also bind-mount ``/home/models`` into the container. + * - ``model.id`` + - ``deepseek-ai/DeepSeek-R1-0528`` + - HuggingFace model id for ATOM server and bench. + * - ``container.image`` / ``container.name`` + - ``rocm/atom-dev:latest``, ``atom_mi300x`` + - Docker image and container name (override cluster file defaults). + * - ``roles.server.atom_args`` + - ``-tp``, ``--kv_cache_dtype`` + - Extra CLI tokens after ``--model`` / ``--server-port`` (ATOM driver). + * - ``roles.server.env`` + - ``ATOM_DISABLE_MMAP`` + - Merged into ``/tmp/server_env_script.sh`` before server launch. + * - ``params.driver`` + - ``atom`` / ``vllm`` / ``vllm_atom`` / ``sglang`` + - ``atom`` = standalone ATOM server + ``benchmark_serving`` (no native PP). ``vllm_atom`` = vLLM multinode PP coordinator + ATOM kernels. ``sglang`` = SGLang PP coordinator. ``vllm`` = interim uplift. + * - ``params.tensor_parallelism`` + - ``8`` + - TP size; appears in threshold cell keys as ``TP``. + * - ``params.reuse_server_across_sweep`` + - ``true`` + - Skip server restart when only concurrency changes between sweep cells. + * - ``params.nnodes`` / ``params.pipeline_parallel_size`` + - ``2`` / ``2`` (multinode PP) + - True multinode pipeline parallel: set ``driver=vllm_atom`` or ``sglang`` with ``nnodes=2`` and ``pipeline_parallel_size=2`` (cell keys use ``PP=2``). Requires ``roles.server.ib_netdev``. Standalone ``driver=atom`` multinode uses SPMD data parallel (``DP`` in cell keys), not PP. + * - ``params.master_addr`` / ``params.master_port`` + - head VPC IP / ``29501`` + - Rendezvous for vLLM/SGLang distributed executor (``dist-init-addr`` for SGLang). + * - ``params.scaling_baseline_output_throughput`` + - ``1500`` + - Single-node reference ``output_throughput`` for ``scaling.efficiency_pct`` (record-only). + * - ``params.server_warmup_wait_s`` / ``client_initial_wait_s`` + - ``330`` / ``120`` + - Config-driven server warmup and client poll floor (use `-k` for a one-cell smoke). + * - ``params.metric_percentiles`` + - ``95,99`` + - Tail percentiles for W1 gates (p95 TPOT, p99 TTFT). + * - ``params.bench_max_failed_requests`` + - ``0`` + - Runtime cap on bench ``Failed requests``; pair with threshold ``client.failed``. + * - ``sweep.sequence_combinations`` / ``sweep.runs`` + - named ISL/OSL + ``{combo, concurrency}`` + - Explicit cell list (not a cartesian product). + +Metric tiers and parsing live in :mod:`cvs.lib.inference.atom.atom_parsing` +(see ``cvs/lib/inference/utils/docs/atom-parsing.md``). Legacy monolithic JSON +(``config`` + ``benchmark_params``) and the deprecated ``inferencemax`` suite are not used. diff --git a/docs/reference/configuration-files/configure-config.rst b/docs/reference/configuration-files/configure-config.rst index 51b10c4c6..362a6baa9 100644 --- a/docs/reference/configuration-files/configure-config.rst +++ b/docs/reference/configuration-files/configure-config.rst @@ -32,8 +32,8 @@ The following list provides a link to code snippets and the parameters for each - :doc:`Megatron ` - :doc:`MORI (RDMA Performance) ` - :doc:`Aorta (Distributed Training) ` -- :doc:`InferenceMAX (vLLM Benchmarking) ` -- :doc:`vLLM Single-Node (MI355X) ` +- :doc:`ATOM (vLLM Benchmarking) ` +- :doc:`vLLM Inference ` - :doc:`SGLang Disaggregated Prefill-Decode ` - :doc:`Flux.1 Text-to-Image ` - :doc:`WAN 2.2 Image-to-Video ` diff --git a/docs/reference/configuration-files/inferencemax.rst b/docs/reference/configuration-files/inferencemax.rst deleted file mode 100644 index 480e6962d..000000000 --- a/docs/reference/configuration-files/inferencemax.rst +++ /dev/null @@ -1,210 +0,0 @@ -.. meta:: - :description: Configure the variables in the InferenceMAX configuration files - :keywords: inference, ROCm, install, cvs, InferenceMAX, vLLM - -******************************************* -InferenceMAX inference configuration file -******************************************* - -InferenceMAX tests validate inference performance for large language models (LLMs) using vLLM backend on AMD GPU clusters. These tests ensure optimal inference throughput, latency, and token generation performance for AI serving workloads. - -The InferenceMAX tests check: - -- **Container orchestration**: Docker setup with ROCm for inference workloads -- **Model serving**: vLLM backend initialization and model loading -- **Performance metrics**: Output throughput, Time to First Token (TTFT), and Time Per Output Token (TPOT) -- **Benchmarking**: Load testing with various concurrency levels and sequence lengths -- **Result verification**: Expected throughput and latency metrics - -Change the parameters as needed in the InferenceMAX configuration file: ``mi300x_singlenode_inferencemax.json`` for single node inference configurations. - -.. note:: - - - Parameters with the ```` value must have that value modified to your specifications. - - ``{user-id}`` will be resolved to the current username in the runtime. You can also manually change this value to your username. - -``mi300x_singlenode_inferencemax.json`` -======================================== - -Here's a code snippet of the ``mi300x_singlenode_inferencemax.json`` file for reference: - -.. dropdown:: ``mi300x_singlenode_inferencemax.json`` - - .. code:: json - - { - "config": { - "container_image": "rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1", - "container_name": "inference_max_rocm", - "_example_nnodes": "4", - "nnodes": "4", - "inferencemax_repo": "https://github.com/InferenceMAX/InferenceMAX.git", - "benchmark_script_repo": "https://github.com/kimbochen/bench_serving.git", - "hf_token_file": "/home/{user-id}/.hf_token", - "shm_size": "128G", - "log_dir": "/home/{user-id}/LOGS", - "container_config": { - "device_list": [ - "/dev/dri", - "/dev/kfd" - ], - "volume_dict": { - "/home/{user-id}": "/home/{user-id}" - }, - "env_dict": {} - } - }, - "benchmark_params": { - "gpt-oss-120b": { - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8000", - "_example_dataset_name": "sharegpt|hf|random|sonnet|burstgpt", - "dataset_name": "random", - "max_concurrency": "64", - "model": "openai/gpt-oss-120b", - "num_prompts": "1000", - "input_sequence_length": "8192", - "output_sequence_length": "1024", - "burstiness": "1.0", - "seed": "0", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "random_prefix_len": "0", - "tensor_parallelism": "8", - "_example_tokenizer_mode": "auto|slow|mistral|custom", - "tokenizer_mode": "auto", - "percentiles_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "server_script": "gptoss_fp4_mi300x_docker.sh", - "bench_serv_script": "benchmark_serving.py", - "result_dict": { - "output_throughput_per_sec": "4200", - "mean_ttft_ms": "500", - "mean_tpot_ms": "15" - } - } - } - } - -Parameters -========== - -Use the parameters in this table to configure the InferenceMAX configuration file. - -.. |br| raw:: html - -
- -.. list-table:: - :widths: 3 3 5 - :header-rows: 1 - - * - Configuration parameters - - Default values - - Description - * - ``container_image`` - - rocm/7.0:rocm7.0_ubuntu_22.04_ |br| vllm_0.10.1_instinct_20250927_rc1 - - Docker container image with ROCm and vLLM for inference - * - ``container_name`` - - inference_max_rocm - - Name of the Docker container instance - * - ``nnodes`` - - 4 - - Number of nodes in the cluster - * - ``inferencemax_repo`` - - https://github.com/InferenceMAX/ |br| InferenceMAX.git - - Git repository URL for InferenceMAX framework - * - ``benchmark_script_repo`` - - https://github.com/kimbochen/ |br| bench_serving.git - - Git repository URL for benchmarking scripts - * - ``hf_token_file`` - - ``/home/{user-id}/`` |br| ``.hf_token`` - - Path to HuggingFace authentication token file for model access - * - ``shm_size`` - - 128G - - Shared memory size allocated to the container - * - ``log_dir`` - - ``/home/{user-id}/LOGS`` - - Directory where inference logs are stored - * - ``container_config.`` |br| ``device_list`` - - Values: |br| - ``"/dev/dri"`` |br| - ``"/dev/kfd"`` - - List of device paths to mount in the container for GPU access - * - ``container_config.`` |br| ``volume_dict`` - - ``{"/home/{user-id}": "/home/{user-id}"}`` - - Dictionary mapping host paths to container paths for volume mounts - * - ``/home/{user-id}`` - - ``/home/{user-id}`` - - User home directory mount - * - ``container_config.`` |br| ``env_dict`` - - Empty - - Dictionary of environment variables to set in the container - * - ``benchmark_params.`` |br| ``gpt-oss-120b.backend`` - - vllm - - Inference backend to use (vLLM) - * - ``benchmark_params.`` |br| ``gpt-oss-120b.base_url`` - - http://0.0.0.0 - - Base URL for the inference server - * - ``benchmark_params.`` |br| ``gpt-oss-120b.port_no`` - - 8000 - - Port number for the inference server - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``dataset_name`` - - random - - Dataset type for benchmarking (sharegpt, hf, random, sonnet, burstgpt) - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``max_concurrency`` - - 64 - - Maximum number of concurrent requests during benchmarking - * - ``benchmark_params.`` |br| ``gpt-oss-120b.model`` - - openai/gpt-oss-120b - - HuggingFace model identifier or path - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``num_prompts`` - - 1000 - - Total number of prompts to send during the benchmark - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``input_sequence_length`` - - 8192 - - Length of input sequences in tokens - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``output_sequence_`` |br| ``length`` - - 1024 - - Expected length of output sequences in tokens - * - ``benchmark_params.`` |br| ``gpt-oss-120b.burstiness`` - - 1.0 - - Request burstiness factor (1.0 = uniform distribution) - * - ``benchmark_params.`` |br| ``gpt-oss-120b.seed`` - - 0 - - Random seed for reproducible benchmark results - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``max_model_length`` - - 9216 - - Maximum total sequence length the model can handle - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``random_range_ratio`` - - 0.8 - - Range ratio for random dataset generation - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``random_prefix_len`` - - 0 - - Prefix length for random dataset generation - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``tensor_parallelism`` - - 8 - - Number of GPUs to use for tensor parallelism - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``tokenizer_mode`` - - auto - - Tokenizer mode (auto, slow, mistral, custom) - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``percentiles_metrics`` - - ttft,tpot,itl,e2el - - Comma-separated list of metrics to compute percentiles for (ttft: Time to First Token, tpot: Time Per Output Token, itl: Inter-Token Latency, e2el: End-to-End Latency) - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``metric_percentiles`` - - 99 - - Percentile values to compute for metrics (e.g., 99 for 99th percentile) - * - ``benchmark_params.`` |br| ``gpt-oss-120b.server_script`` - - gptoss_fp4_mi300x_docker.sh - - Script to launch the inference server - * - ``benchmark_params.`` |br| ``gpt-oss-120b.`` |br| ``bench_serv_script`` - - benchmark_serving.py - - Script to run the benchmarking client - * - ``benchmark_params.`` |br| ``gpt-oss-120b.result_dict.`` |br| ``output_throughput_`` |br| ``per_sec`` - - 4200 - - Expected number of output tokens generated per second - * - ``benchmark_params.`` |br| ``gpt-oss-120b.result_dict.`` |br| ``mean_ttft_ms`` - - 500 - - Expected mean Time to First Token in milliseconds - * - ``benchmark_params.`` |br| ``gpt-oss-120b.result_dict.`` |br| ``mean_tpot_ms`` - - 15 - - Expected mean Time Per Output Token in milliseconds diff --git a/docs/reference/configuration-files/vllm.rst b/docs/reference/configuration-files/vllm.rst new file mode 100644 index 000000000..b0c6d2103 --- /dev/null +++ b/docs/reference/configuration-files/vllm.rst @@ -0,0 +1,1211 @@ +.. meta:: + :description: Configure the vLLM inference benchmark suite in CVS + :keywords: inference, ROCm, cvs, vLLM, LLM, benchmark, multinode, thresholds, metrics, accuracy + +********************************** +vLLM inference configuration file +********************************** + +The vLLM suite benchmarks LLM serving throughput, latency, and accuracy on AMD Instinct GPUs. It is a **single parametrized suite**: the same test file covers single-node and multinode pipeline-parallel runs, and the topology is determined entirely by the configuration file. There is no separate "single-node" and "distributed" suite to choose between. + +Run it with: + +.. code:: bash + + cvs run vllm --cluster_file --config_file + +For a step-by-step walkthrough of a first run, see :doc:`/how-to/run-vllm-benchmarks`. This page is the schema and metric reference. + +Lifecycle +========= + +Each stage of the run is an independent test, so every stage becomes its own timed, pass/fail row in the HTML report. The suite pins this order explicitly rather than relying on definition order: + +.. list-table:: + :widths: 1 3 6 + :header-rows: 1 + + * - Order + - Test + - Purpose + * - 0 + - ``test_launch_container`` + - Pull/load the image and start the container on every node + * - 1 + - ``test_setup_sshd`` + - Always skipped for vLLM (see note below) + * - 2 + - ``test_discover_topology`` + - Resolve IB HCA devices; no-op when ``nnodes`` is 1 + * - 3 + - ``test_model_fetch`` + - Stage model weights + * - 4 + - ``test_openai_compatible_smoke`` + - Short-lived server; verifies the OpenAI-compatible API answers + * - 5 + - ``test_vllm_inference`` + - Run one benchmark cell (parametrized per sweep run) + * - 6 + - ``test_metric``, ``test_gpu_metric``, ``test_prom_metric`` + - One row per metric, per cell + * - 7 + - ``test_accuracy_eval`` + - lm-eval accuracy tasks, if any are configured + * - 8 + - ``test_print_results_table`` + - Console + report summary table + * - 9 + - ``test_teardown`` + - Stop the server and tear down the container + +.. note:: + + ``test_setup_sshd`` always skips in this suite. vLLM uses ``--distributed-executor-backend mp`` with NCCL over the host network, so no inter-container sshd is needed. A skipped row here is expected, not a problem. + +If a stage fails, later stages are skipped rather than cascading into confusing downstream errors. The container is still torn down by a leak-guard even when a mid-sweep test fails. + +Configuration file structure +============================ + +A vLLM configuration file has these top-level keys: + +.. list-table:: + :widths: 3 2 5 + :header-rows: 1 + + * - Key + - Required + - Description + * - ``schema_version`` + - yes + - Must be ``1`` + * - ``framework`` + - yes + - Must be ``"vllm"`` + * - ``gpu_arch`` + - yes + - GPU architecture label, for example ``"mi300x"``. Reported, not enforced + * - ``enforce_thresholds`` + - no (default ``true``) + - When ``false``, threshold failures and coverage gaps become warnings + * - ``threshold_json`` + - no + - Explicit path to the threshold file. See :ref:`vllm-threshold-discovery` + * - ``container`` + - no + - Container/Docker settings. See :ref:`vllm-container` + * - ``paths`` + - yes + - Filesystem locations. See :ref:`vllm-paths` + * - ``model`` + - yes + - Model identifier. See :ref:`vllm-model` + * - ``roles`` + - yes + - Server arguments and environment. See :ref:`vllm-roles` + * - ``params`` + - yes + - Client and topology parameters. See :ref:`vllm-params` + * - ``sweep`` + - yes + - Sequence combinations and runs. See :ref:`vllm-sweep` + * - ``thresholds`` + - no + - Per-cell pass/fail specs. See :ref:`vllm-thresholds` + * - ``accuracy`` + - no + - lm-eval task selection. See :ref:`vllm-accuracy` + +.. important:: + + Every block except ``container`` **forbids unknown keys**. A misspelled key is a hard validation error at load time, not a silently ignored setting. The ``container`` block is permissive because it passes ``runtime`` and other keys through to the orchestrator untouched. + +Placeholder substitution +------------------------ + +Values are resolved in three passes, so later forms can reference earlier ones: + +1. **Cluster placeholders** — ``{user-id}`` resolves to the current username. +2. **Self-reference within** ``paths`` — ``{shared_fs}`` expands to the already-resolved ``paths.shared_fs``. +3. **Cross-block** — ``{paths.models_dir}`` expands anywhere else in the file, such as in a volume mount. + +.. code:: json + + { + "paths": { + "shared_fs": "/mnt/dtni/{user-id}", + "models_dir": "{shared_fs}/models", + "log_dir": "{shared_fs}/LOGS", + "hf_token_file": "{shared_fs}/.cache/huggingface/token" + }, + "container": { + "runtime": { + "args": { + "volumes": ["{paths.models_dir}:/models"] + } + } + } + } + +.. _vllm-backends: + +Execution backends +================== + +Four different things in this stack are called a "backend". They are unrelated, and confusing them is the most common configuration mistake. + +.. list-table:: + :widths: 3 2 5 + :header-rows: 1 + + * - Setting + - Values + - What it selects + * - ``params.backend`` + - ``"vllm"`` (default) + - The **client** backend passed to ``vllm bench serve --backend``. Nothing to do with distribution + * - ``roles.server.serve_args.``\ ``distributed-executor-backend`` + - ``"mp"`` (default), ``"ray"`` + - How vLLM distributes the model across nodes. This is the multinode setting + * - ``container.runtime.name`` + - ``"docker"`` (default), ``"enroot"`` + - The container runtime + * - Cluster file ``orchestrator`` + - ``"baremetal"``, ``"container"`` + - Whether CVS runs commands on the host or inside a container. See :doc:`/reference/configuration-files/cluster-file` + +.. warning:: + + ``enroot`` is registered but **not implemented**. Every method is a stub that returns failure, so a run with ``runtime.name: "enroot"`` fails at ``test_launch_container``. Podman is not supported. Use ``docker``. + +Distributed executor: mp and ray +-------------------------------- + +Multinode runs support **two** executor backends. ``mp`` is the default and requires no configuration key at all. + +**mp (default).** Used whenever ``distributed-executor-backend`` is absent from ``serve_args``. The suite injects the full distributed block into each rank's ``vllm serve`` command: + +.. code:: bash + + vllm serve --tensor-parallel-size --port \ + --node-rank --master-addr --master-port \ + --nnodes --pipeline-parallel-size \ + --distributed-executor-backend mp + +Every rank above 0 additionally gets ``--headless``. This path **requires pipeline parallelism** (``pipeline_parallel_size`` greater than 1). + +**ray (opt-in).** Selected by setting ``distributed-executor-backend`` to the exact lowercase string ``"ray"``. Any other spelling, including ``"Ray"``, falls back to the mp path. Ray takes a completely different route: + +1. Bootstrap the cluster head: ``ray start --head --port=`` +2. Bootstrap each worker: ``ray start --address=:`` +3. Launch ``vllm serve`` on the **head node only** — workers run no serve process +4. On teardown, broadcast ``ray stop`` after the process kill + +Under ray, none of the mp distributed flags are emitted; the backend flag reaches vLLM through normal ``serve_args`` flattening. ``--pipeline-parallel-size`` is added only when ``pipeline_parallel_size`` is greater than 1. + +.. note:: + + Ray does not *enable* multinode — it **relaxes** the pipeline-parallelism requirement. With ray, ``pipeline_parallel_size`` of 1 is legal and is the expected configuration for pure tensor-parallel multinode serving. With mp, pipeline parallelism is mandatory. + +Because only the head node serves under ray, worker ranks produce no per-rank server log. That is expected. + +Topology validation rules +------------------------- + +These rules are enforced when the configuration file loads, before anything starts: + +.. list-table:: + :widths: 4 6 + :header-rows: 1 + + * - Condition + - Rule + * - ``nnodes`` > 1, backend is not ray + - ``pipeline_parallel_size`` **must** be greater than 1 + * - ``nnodes`` > 1, backend is ray + - ``pipeline_parallel_size`` of 1 is valid + * - ``pipeline_parallel_size`` > 1 + - ``nnodes`` **must** be greater than 1 + * - ``nnodes`` > 1, either backend + - ``roles.server.ib_netdev`` is **required** + +The corresponding error messages are: + +.. code:: text + + nnodes=2 > 1 requires pipeline_parallel_size > 1 (got pp=1) + pipeline_parallel_size=2 > 1 requires nnodes > 1 (got nnodes=1) + ib_netdev is required in roles.server when nnodes > 1. Set it to the Linux + network interface name for NCCL_SOCKET_IFNAME (e.g. "ens51f1np1"). Cannot be + auto-derived from HCA names. + +Multinode prerequisites +----------------------- + +Beyond the validation rules, a multinode run needs: + +- ``params.master_addr`` — the head node's address, reachable from every worker. +- ``params.master_port`` — default ``"29501"``. +- ``roles.server.ib_netdev`` — the Linux interface name. There is deliberately no ``"auto"`` value; it cannot be derived reliably from HCA names. This value populates ``NCCL_SOCKET_IFNAME``, ``GLOO_SOCKET_IFNAME``, and ``TP_SOCKET_IFNAME``. +- ``roles.server.ib_hca_devices`` — ``"auto"``, an explicit list, or ``null``. When set, populates ``NCCL_IB_HCA``. + +.. _vllm-container: + +Container and Docker configuration +================================== + +The container block controls image selection, lifetime, and the ``docker run`` flags. + +.. code:: json + + { + "container": { + "lifetime": "per_run", + "name": "vllm_perf_inference_rocm", + "image": "rocm/vllm:latest", + "runtime": { + "name": "docker", + "args": { + "network": "host", + "ipc": "host", + "privileged": true, + "volumes": [ + "/home/{user-id}:/home/{user-id}", + "{paths.models_dir}:/models" + ] + } + } + } + } + +Container block keys +-------------------- + +.. list-table:: + :widths: 3 2 5 + :header-rows: 1 + + * - Key + - Default + - Description + * - ``image`` + - none + - Container image. **Required** — launch fails with ``Container image not specified in config`` + * - ``name`` + - ``_`` + - Container name + * - ``lifetime`` + - ``"per_run"`` + - One of ``no_launch``, ``per_run``, ``persistent`` + * - ``runtime.name`` + - ``"docker"`` + - Container runtime + * - ``runtime.args`` + - ``{}`` + - Docker flags; see the table below + * - ``env`` + - ``{}`` + - Container-level environment variables. **Top level, not under** ``runtime.args`` + * - ``image_tar`` + - absent + - Path on each host to a saved image tar to ``docker load`` instead of pulling. **Top level** + +.. warning:: + + Put ``env`` at the **container top level**. Placing it under ``runtime.args`` crashes container launch: the code iterates that value as a sequence of pairs, which raises ``ValueError: too many values to unpack`` for any key longer than two characters. + +Runtime arguments +----------------- + +All keys under ``runtime.args`` are optional. **List-valued keys append to the defaults; scalar keys override them.** There is no way to remove a default device or capability. + +.. list-table:: + :widths: 2 1 3 4 + :header-rows: 1 + + * - Key + - Merge + - Default + - Emitted flag + * - ``volumes`` + - append + - ``/home/$USER/.ssh:/host_ssh`` (always added) + - ``-v :[:ro]`` + * - ``devices`` + - append + - ``/dev/kfd``, ``/dev/dri``, ``/dev/infiniband`` + - ``--device `` + * - ``cap_add`` + - append + - ``SYS_PTRACE``, ``IPC_LOCK``, ``SYS_ADMIN`` + - ``--cap-add `` + * - ``security_opt`` + - append + - ``seccomp=unconfined``, ``apparmor=unconfined`` + - ``--security-opt `` + * - ``group_add`` + - append + - ``video`` + - ``--group-add `` + * - ``ulimit`` + - append + - ``memlock=-1`` + - ``--ulimit `` + * - ``network`` + - override + - ``host`` + - ``--network `` + * - ``ipc`` + - override + - ``host`` + - ``--ipc `` + * - ``privileged`` + - override + - ``true`` + - ``--privileged`` + * - ``registry`` + - n/a + - none + - Triggers ``docker login``; see below + +The assembled command is: + +.. code:: bash + + docker run -d --name sleep infinity + +The container is a long-lived sidecar; every workload command runs through ``docker exec`` inside it. InfiniBand devices are additionally passed through by per-host shell expansion at launch time, so each node mounts the devices it actually has. + +.. note:: + + ``--gpus`` is deliberately never emitted — GPU access on AMD hardware comes from the ``/dev/kfd`` and ``/dev/dri`` device mounts plus the ``video`` group. + + ``shm_size`` is **not supported** on this path. Setting ``runtime.args.shm_size`` is silently ignored; ``--shm-size`` is never emitted. + +Container lifetime +------------------ + +.. list-table:: + :widths: 2 4 4 + :header-rows: 1 + + * - ``lifetime`` + - Setup behavior + - Teardown behavior + * - ``no_launch`` + - Verifies a container of that name is already running; never starts one + - No-op + * - ``per_run`` + - Force-removes any stale container of the same name, then launches + - ``docker rm -f`` + * - ``persistent`` + - Attaches if running on all hosts; cold-starts if absent on all hosts; **refuses** on partial or failed probe + - No-op + +.. tip:: + + With ``persistent``, always pin ``container.name`` explicitly. The default name is derived from the image, so bumping an image tag silently abandons the old container and starts a new one. + +Registry authentication +----------------------- + +Set ``runtime.args.registry`` to log in before pulling: + +.. code:: json + + { + "registry": { + "username": "myuser", + "password_file": "/path/on/each/host/to/token", + "server": "registry.example.com" + } + } + +``username`` and ``password_file`` are required; ``server`` defaults to Docker Hub. The password is read from a **file path on each remote host** — there is no inline password or token key, and the login is kept out of the logs. Login is skipped entirely when ``image_tar`` is set, since a tar load never pulls. + +Image resolution order at launch: + +1. If ``image_tar`` is set and the image is absent, ``docker load`` it. +2. Otherwise, if ``registry`` is set, log in. +3. Check whether the image exists on **all** hosts. +4. If not, ``docker pull`` it, with no retry or fallback. + +.. note:: + + The image-exists check matches ``Repository:Tag`` exactly, so an image referenced without a tag or by digest never matches and is pulled on every run. + +Cluster file merge +------------------ + +The variant's ``container`` block is deep-merged **onto** the cluster file's block: dictionaries merge key-wise, while scalars and lists are replaced. Cluster-set values survive unless the variant sets the same key. + +.. warning:: + + A ``container`` block in the variant always contributes ``lifetime``, ``name``, and ``image`` — including empty defaults. If your variant defines ``container`` but omits ``image``, it overwrites a cluster-file ``image`` with an empty string and the launch fails. Set ``image`` in whichever file defines the block. + +.. _vllm-paths: + +Paths +===== + +All four keys are required. + +.. list-table:: + :widths: 3 7 + :header-rows: 1 + + * - Key + - Description + * - ``shared_fs`` + - Root of the shared filesystem, typically the anchor other paths reference + * - ``models_dir`` + - Model weight cache; exported into the server as ``HF_HUB_CACHE`` + * - ``log_dir`` + - Root for run artifacts + * - ``hf_token_file`` + - Path to a file containing the Hugging Face token + +If ``hf_token_file`` does not exist and the model is pre-staged (``model.remote`` of 0), the run continues with an empty token and the server sets ``HF_HUB_OFFLINE=1``. If the model is remote, the suite skips instead. + +Per-cell artifacts land in:: + + /vllm/out-node/isl_osl_conc/ + vllm_serve_server.log + client.log + results + +.. _vllm-model: + +Model +===== + +.. list-table:: + :widths: 3 2 5 + :header-rows: 1 + + * - Key + - Default + - Description + * - ``id`` + - none + - Hugging Face model ID or local path, for example ``amd/Llama-3.1-70B-Instruct-FP8-KV`` + * - ``remote`` + - none + - ``0`` for a pre-staged model + +.. important:: + + ``remote: 1`` is **not implemented** and raises ``NotImplementedError`` at load time. Stage weights under ``paths.models_dir`` and use ``remote: 0``. + +.. _vllm-roles: + +Server role +=========== + +``roles.server`` controls the ``vllm serve`` process. + +.. list-table:: + :widths: 3 2 5 + :header-rows: 1 + + * - Key + - Default + - Description + * - ``serve_args`` + - ``{}`` + - Flags passed through to ``vllm serve`` + * - ``env`` + - ``{}`` + - Environment for the server process and the benchmark client + * - ``ib_hca_devices`` + - ``null`` + - ``"auto"``, an explicit list, or ``null``; sets ``NCCL_IB_HCA`` + * - ``ib_netdev`` + - ``null`` + - Interface name; required when ``nnodes`` is greater than 1 + +How serve_args are flattened +---------------------------- + +.. list-table:: + :widths: 3 3 4 + :header-rows: 1 + + * - JSON value + - Emitted + - Example + * - Scalar + - ``--flag value`` + - ``"kv-cache-dtype": "fp8"`` → ``--kv-cache-dtype fp8`` + * - ``true`` + - ``--flag`` (bare) + - ``"enforce-eager": true`` → ``--enforce-eager`` + * - ``false`` + - nothing + - ``"enforce-eager": false`` → omitted entirely + * - List + - flag repeated per element + - ``"x": ["a","b"]`` → ``--x a --x b`` + +``serve_args.log-level``, if set, must be one of ``debug``, ``info``, ``warning``, ``error``, ``critical``. + +Derived max-model-len +--------------------- + +``--max-model-len`` is computed and emitted **only when** ``serve_args`` does not already set ``max-model-len``: + +.. code:: text + + ceil((isl + osl) * (1 + random_range_ratio)) + random_prefix_len + 8 + +Setting ``max-model-len`` explicitly in ``serve_args`` suppresses the derived value, so the flag never appears twice. + +Environment variables: two mechanisms +------------------------------------- + +These are separate and are frequently confused. + +.. list-table:: + :widths: 2 4 4 + :header-rows: 1 + + * - + - ``container.env`` + - ``roles.server.env`` + * - Applied by + - ``docker run -e`` + - A sourced shell script inside the container + * - Scope + - Every command in the container, for its whole lifetime + - The ``vllm serve`` processes and the benchmark client + * - Changing it + - Requires recreating the container + - Takes effect on the next run + * - Defaults + - ``GPUS=8``, ``MULTINODE=true`` + - See below + +The server environment script always exports: + +.. code:: bash + + export HF_TOKEN= + export HF_HUB_CACHE= + export VLLM_USE_AITER_UNIFIED_ATTENTION=1 + export VLLM_ROCM_USE_AITER_MHA=0 + export VLLM_ROCM_USE_AITER_FUSED_MOE_A16W4=1 + +then, conditionally, ``NCCL_IB_HCA`` (from ``ib_hca_devices``) and ``NCCL_SOCKET_IFNAME`` / ``GLOO_SOCKET_IFNAME`` / ``TP_SOCKET_IFNAME`` (from ``ib_netdev``). Entries from ``roles.server.env`` are appended **last**, so they override any of the above. + +.. _vllm-params: + +Parameters +========== + +``params`` holds the client knobs and the topology. + +.. list-table:: + :widths: 3 2 5 + :header-rows: 1 + + * - Key + - Default + - Description + * - ``backend`` + - ``"vllm"`` + - Client backend for ``vllm bench serve`` + * - ``base_url`` + - ``"http://0.0.0.0"`` + - Server base URL + * - ``port_no`` + - ``"8888"`` + - Server port + * - ``dataset_name`` + - ``"random"`` + - Dataset for the load generator + * - ``num_prompts`` + - ``"3200"`` + - Total prompts per cell + * - ``burstiness`` + - ``"1.0"`` + - 1.0 is a uniform arrival process; lower is burstier + * - ``seed`` + - ``"0"`` + - Random seed + * - ``request_rate`` + - ``"inf"`` + - Arrival rate; ``inf`` sends as fast as concurrency allows + * - ``random_range_ratio`` + - ``"0.8"`` + - Length jitter around ISL/OSL; also feeds the derived max-model-len + * - ``random_prefix_len`` + - ``"0"`` + - Shared prefix length + * - ``tensor_parallelism`` + - ``"8"`` + - TP degree + * - ``pipeline_parallel_size`` + - ``"1"`` + - PP degree; see :ref:`vllm-backends` + * - ``nnodes`` + - ``"1"`` + - Node count + * - ``master_addr`` + - ``"localhost"`` + - Head node address for multinode + * - ``master_port`` + - ``"29501"`` + - Rendezvous port + * - ``tokenizer_mode`` + - ``"auto"`` + - Tokenizer mode + * - ``percentile_metrics`` + - ``"ttft,tpot,itl,e2el"`` + - Metric families to compute percentiles for + * - ``metric_percentiles`` + - ``"50,90,95,99"`` + - Percentiles to emit + * - ``client_poll_count`` + - ``"20"`` + - Client completion polls before giving up + +.. tip:: + + ``metric_percentiles`` must emit every percentile your thresholds gate. The default ``"50,90,95,99"`` covers all gated latency metrics. Narrowing it to ``"99"`` makes p50/p90/p95 unavailable, and any threshold on them then fails loudly. + +.. _vllm-sweep: + +Sweep +===== + +The sweep is an explicit list of runs, not a cartesian product. Named sequence combinations are declared once, then referenced by the runs list. + +.. code:: json + + { + "sweep": { + "sequence_combinations": [ + { + "name": "balanced", + "isl": "1000", + "osl": "1000", + "goodput_slo": { "ttft_ms": 2000.0, "tpot_ms": 50.0, "e2el_ms": 60000.0 } + } + ], + "runs": [ + { "combo": "balanced", "concurrency": 16 }, + { "combo": "balanced", "concurrency": 32 } + ] + } + } + +.. list-table:: + :widths: 3 7 + :header-rows: 1 + + * - Key + - Description + * - ``sequence_combinations[].name`` + - Unique label; duplicates are rejected + * - ``sequence_combinations[].isl`` + - Input sequence length + * - ``sequence_combinations[].osl`` + - Output sequence length + * - ``sequence_combinations[].goodput_slo`` + - Optional; ``ttft_ms``, ``tpot_ms``, ``e2el_ms``, all required together + * - ``runs[].combo`` + - Must name a declared combination + * - ``runs[].concurrency`` + - Integer max concurrency for this cell + +A ``combo`` that names no declared combination is a load-time error listing the known names. + +When ``goodput_slo`` is set, the client is invoked with ``--goodput ttft: tpot: e2el:`` and ``client.goodput`` becomes meaningful. + +Cell keys +--------- + +Each run is one **cell**, identified by a canonical key used to look up thresholds: + +.. code:: text + + Single-node: ISL=,OSL=,TP=,CONC= + Distributed: ISL=,OSL=,TP=,PP=,CONC= + +The ``PP=`` segment appears **only** when ``pipeline_parallel_size`` is greater than 1, which keeps single-node keys backward compatible. Examples:: + + ISL=1000,OSL=1000,TP=8,CONC=16 + ISL=1000,OSL=1000,TP=8,PP=2,CONC=16 + +Server reuse +------------ + +Cells that differ **only** in concurrency share a server identity, so the suite reuses the running server instead of stopping it, restarting, and reloading weights. Changing ISL, OSL, TP, PP, or any server argument forces a restart. Ordering runs so that concurrency varies fastest therefore makes a sweep substantially quicker. + +.. _vllm-thresholds: + +Thresholds +========== + +Thresholds turn measurements into pass/fail results. They are keyed by cell key, then by fully-qualified metric name: + +.. code:: json + + { + "ISL=1000,OSL=1000,TP=8,CONC=16": { + "client.total_token_throughput": { "kind": "min_tok_s", "value": 4000 }, + "client.mean_ttft_ms": { "kind": "max_ms", "value": 500 }, + "client.failed": { "kind": "max", "value": 0 }, + "client.success_rate": { "kind": "min", "value": 0.99 }, + "gpu.gpu_compute_util_pct": { "kind": "within", "value": 90, "tolerance_pct": 10 }, + "client.output_throughput": { "kind": "min_ratio", "value": 0.8, + "reference": "client.total_token_throughput" } + } + } + +Threshold kinds +--------------- + +.. list-table:: + :widths: 2 2 6 + :header-rows: 1 + + * - ``kind`` + - Extra keys + - Fails when + * - ``min`` + - — + - ``actual < value`` + * - ``max`` + - — + - ``actual > value``. Unit-agnostic upper bound, for counts such as ``failed`` + * - ``max_ms`` + - — + - ``actual > value``. Identical comparison to ``max``, but the message says "ms" + * - ``min_tok_s`` + - — + - ``actual < value``. Identical comparison to ``min``, but the message says "tok/s" + * - ``within`` + - ``tolerance_pct`` + - ``actual`` falls outside ``value ± tolerance_pct`` percent + * - ``min_ratio`` + - ``reference`` + - ``actual / `` is less than ``value`` + +An unrecognized ``kind`` is a violation, not a silent skip. A metric that is missing from the results, or whose value is ``None``, is also a loud violation rather than a pass. + +For ``min_ratio``, the ``reference`` names another metric in the same cell. If that reference is missing, ``None``, or zero, the check fails with a message naming the reason. + +.. _vllm-threshold-coverage: + +Coverage checking +----------------- + +At load time the threshold file is checked on one axis: **cell coverage**. Every sweep cell must have a threshold entry, and no threshold key may name a cell that the sweep does not produce. This catches keys left behind after a sweep edit. + +There is no per-metric coverage requirement. A cell's entry may spec a single metric or two dozen — a threshold file is free to gate only the metrics you care about rather than every member of every family. + +The ``accuracy`` key is exempt from cell-coverage checking, since it is keyed by task rather than by cell. + +Setting ``enforce_thresholds`` to ``false`` downgrades coverage problems to warnings and stops threshold violations from failing tests. The run still measures and records everything, which makes it the right setting for a first calibration run on new hardware. + +Which metrics are asserted is then decided per metric at evaluation time, not at load time. A metric is checked only when its cell carries a spec for it; with no spec it is measured and reported but never asserted. A spec of ``null`` is the explicit way to say the same thing. + +.. _vllm-threshold-discovery: + +Threshold file discovery +------------------------ + +The threshold file is located in one of two ways: + +- **Explicit** — set ``threshold_json`` to a path. A relative path resolves against the configuration file's directory. +- **Implicit** — if ``threshold_json`` is absent, the loader looks for exactly one file matching ``*threshold.json`` beside the configuration file. Finding more than one is an error, so add ``threshold_json`` when several coexist in a directory. + +Metrics +======= + +Metrics live in namespaces. Each numeric metric becomes one test, and therefore one row in the HTML report. A metric that could not be measured skips rather than failing. Read the measured values from the results table and the per-cell logs; the report's Value and Unit columns are currently disabled. + +Client metrics +-------------- + +Measured by the load generator (``vllm bench serve``) and namespaced ``client.*``. **Gated** marks the metrics designated as pass/fail criteria: they populate the report's gate matrix and they are the ones a threshold file normally specs. The mark does not make a metric mandatory — any metric, gated or not, is asserted only when its cell carries a spec for it (see :ref:`vllm-threshold-coverage`). + +.. list-table:: + :widths: 4 1 1 4 + :header-rows: 1 + + * - Metric + - Unit + - Gated + - Notes + * - ``client.total_token_throughput`` + - tok/s + - yes + - Input plus output tokens per second + * - ``client.output_throughput`` + - tok/s + - yes + - Generated tokens per second + * - ``client.mean_ttft_ms`` + - ms + - yes + - Time to first token + * - ``client.median_ttft_ms`` + - ms + - yes + - + * - ``client.p90_ttft_ms`` + - ms + - yes + - + * - ``client.p95_ttft_ms`` + - ms + - yes + - + * - ``client.p99_ttft_ms`` + - ms + - yes + - + * - ``client.mean_tpot_ms`` + - ms + - yes + - Time per output token + * - ``client.median_tpot_ms`` + - ms + - yes + - + * - ``client.p90_tpot_ms`` + - ms + - yes + - + * - ``client.p95_tpot_ms`` + - ms + - yes + - + * - ``client.p99_tpot_ms`` + - ms + - yes + - + * - ``client.mean_itl_ms`` + - ms + - yes + - Inter-token latency + * - ``client.median_itl_ms`` + - ms + - yes + - + * - ``client.p95_itl_ms`` + - ms + - yes + - + * - ``client.p99_itl_ms`` + - ms + - yes + - ITL has no p90 producer + * - ``client.mean_e2el_ms`` + - ms + - yes + - End-to-end latency + * - ``client.median_e2el_ms`` + - ms + - yes + - + * - ``client.p90_e2el_ms`` + - ms + - yes + - + * - ``client.p95_e2el_ms`` + - ms + - yes + - + * - ``client.p99_e2el_ms`` + - ms + - yes + - + * - ``client.success_rate`` + - \- + - yes + - Derived; see below + * - ``client.failed`` + - \- + - yes + - Failed request count + * - ``client.max_concurrency`` + - \- + - no + - + * - ``client.max_concurrent_requests`` + - \- + - no + - + * - ``client.num_prompts`` + - \- + - no + - + * - ``client.completed`` + - \- + - no + - + * - ``client.duration`` + - s + - no + - + * - ``client.request_throughput`` + - req/s + - no + - + * - ``client.goodput`` + - req/s + - no + - Alias of the stock ``request_goodput``; meaningful only with ``goodput_slo`` + * - ``client.per_gpu_throughput`` + - tok/s + - no + - Derived; see below + * - ``client.decode_throughput_p50`` + - tok/s + - no + - Derived; see below + * - ``client.max_output_tokens_per_s`` + - tok/s + - no + - + * - ``client.total_input_tokens`` + - \- + - no + - + * - ``client.total_output_tokens`` + - \- + - no + - + * - ``client.normalized_ttft_ms_per_tok`` + - ms/tok + - no + - Derived; see below + * - ``client.decode_latency_ratio`` + - \- + - no + - Derived; see below + +Derived client metrics +~~~~~~~~~~~~~~~~~~~~~~ + +.. code:: text + + per_gpu_throughput = total_token_throughput / (tp * pp) + normalized_ttft_ms_per_tok = mean_ttft_ms / isl + decode_latency_ratio = p99_itl_ms / p50_itl_ms + decode_throughput_p50 = 1000 / median_tpot_ms + success_rate = completed / (completed + failed) + +Every division is guarded: a missing, ``None``, or zero divisor yields ``None`` — reported as ``-`` — rather than a bogus zero or a crash. + +.. note:: + + ``client.request_rate`` is not surfaced as a metric row, because the stock benchmark emits the string ``inf`` rather than a number. + + A new metric is **record-only by default**. Adding its name to the gated set marks it as a pass/fail criterion and files it under one of the report's gate-matrix tiers; it still only fails a run in those cells whose threshold entry specs it. + +GPU metrics +----------- + +Sampled from ``amd-smi`` during the run and namespaced ``gpu.*``. None are gated by default. + +.. list-table:: + :widths: 4 1 5 + :header-rows: 1 + + * - Metric + - Unit + - Description + * - ``gpu.peak_gpu_memory_mb`` + - MB + - Peak VRAM observed + * - ``gpu.model_load_memory_mb`` + - MB + - VRAM attributable to loading weights + * - ``gpu.model_load_s`` + - s + - Weight load duration + * - ``gpu.gpu_bandwidth_util_pct`` + - % + - Memory bandwidth utilization + * - ``gpu.gpu_compute_util_pct`` + - % + - Compute utilization + +Server metrics +-------------- + +Scraped from the vLLM ``/metrics`` Prometheus endpoint and namespaced ``prom.*``. + +.. list-table:: + :widths: 4 1 5 + :header-rows: 1 + + * - Metric + - Unit + - Source histogram + * - ``prom.queue_time_p50_ms`` + - ms + - ``vllm:request_queue_time_seconds`` + * - ``prom.queue_time_p95_ms`` + - ms + - ``vllm:request_queue_time_seconds`` + * - ``prom.prefill_time_p50_ms`` + - ms + - ``vllm:request_prefill_time_seconds`` + * - ``prom.prefill_time_p95_ms`` + - ms + - ``vllm:request_prefill_time_seconds`` + +vLLM's Prometheus counters are cumulative over the server process lifetime, so a raw scrape after cell three would include cells one and two. The suite therefore scrapes **before and after each cell** and diffs the histogram buckets, giving per-cell quantiles. Quantiles are computed with the same interpolation PromQL's ``histogram_quantile`` uses. + +If the endpoint cannot be reached, all four report ``-`` and skip rather than failing the run. + +Results table +------------- + +The summary table emits seven fixed columns — Model, GPU, ISL, OSL, Policy, Conc, Host — followed by Req/s, Total tok/s, Mean TTFT, P95 TTFT, Mean TPOT, P95 TPOT, P99 ITL, and Goodput. + +.. _vllm-accuracy: + +Accuracy tests +============== + +Accuracy evaluation runs `lm-evaluation-harness `_ against the live server after the performance sweep. Task **selection** lives in the configuration file; **gating values** live in the threshold file. + +.. code:: json + + { + "accuracy": { + "tasks": [ + { + "id": "gsm8k_strict", + "task": "gsm8k", + "num_fewshot": 5, + "num_concurrent": 8, + "apply_chat_template": false + } + ] + } + } + +.. list-table:: + :widths: 3 2 5 + :header-rows: 1 + + * - Key + - Default + - Description + * - ``id`` + - none + - Unique label for this entry; duplicates are rejected + * - ``task`` + - none + - lm-eval task name + * - ``num_fewshot`` + - ``0`` + - Few-shot example count + * - ``num_concurrent`` + - ``8`` + - Concurrent requests + * - ``apply_chat_template`` + - ``false`` + - Selects the endpoint; see below + * - ``metadata`` + - ``{}`` + - Passed through to lm-eval + * - ``include_path`` + - ``""`` + - Directory of custom task definitions + * - ``gen_kwargs`` + - ``{}`` + - Generation arguments + +``apply_chat_template`` selects the API surface: + +.. list-table:: + :widths: 2 3 4 + :header-rows: 1 + + * - Value + - lm-eval model + - Endpoint + * - ``false`` + - ``local-completions`` + - ``/v1/completions`` + * - ``true`` + - ``local-chat-completions`` + - ``/v1/chat/completions`` + +lm-eval is probed for at run time and installed into the container if absent. Each task has a four-hour timeout. Results land under ``/accuracy``. + +Accuracy metric keys +-------------------- + +Accuracy metrics are keyed ``.``, with any comma in the name replaced by a double underscore. For example, gsm8k's ``exact_match,strict-match`` becomes: + +.. code:: text + + gsm8k.exact_match__strict-match + +Gate them in the threshold file's top-level ``accuracy`` block, keyed by the task ``id``: + +.. code:: json + + { + "accuracy": { + "gsm8k_strict": { + "gsm8k.exact_match__strict-match": { "kind": "min", "value": 0.75 } + } + } + } + +.. note:: + + An accuracy failure does not mark the shared lifecycle as failed, so the remaining stages — the results table and teardown — still run normally. + +Troubleshooting +=============== + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - Message + - Cause and fix + * - ``nnodes=N > 1 requires pipeline_parallel_size > 1`` + - Multinode on the mp backend needs pipeline parallelism. Either raise ``pipeline_parallel_size``, or set ``distributed-executor-backend`` to ``"ray"`` + * - ``pipeline_parallel_size=N > 1 requires nnodes > 1`` + - Pipeline parallelism spans nodes. Raise ``nnodes`` or reset ``pipeline_parallel_size`` to 1 + * - ``ib_netdev is required in roles.server when nnodes > 1`` + - Set ``roles.server.ib_netdev`` to the interface name. There is no ``"auto"`` + * - ``Container image not specified in config`` + - ``container.image`` is empty. Note that a variant ``container`` block with no ``image`` overwrites the cluster file's value + * - ``duplicate sequence_combination names`` + - Two entries in ``sequence_combinations`` share a ``name`` + * - ``run.combo names no sequence_combination`` + - A ``runs[].combo`` does not match any declared name; the message lists the valid ones + * - ``duplicate task id(s)`` + - Two ``accuracy.tasks`` entries share an ``id`` + * - ``: unknown threshold kind`` + - Typo in ``kind``. Valid values are ``min``, ``max``, ``max_ms``, ``min_tok_s``, ``within``, ``min_ratio`` + * - ``: missing from actuals`` + - A threshold gates a metric this run did not produce. Common cause: ``metric_percentiles`` omits the gated percentile + * - ``NotImplementedError: model.remote=1`` + - Remote model download is unimplemented. Pre-stage weights and set ``remote: 0`` + * - ``ValueError: too many values to unpack`` + - ``env`` was placed under ``runtime.args``. Move it to the ``container`` top level + * - Extra-key validation error + - A misspelled key. Every block except ``container`` forbids unknown keys + +See also +======== + +- :doc:`/how-to/run-vllm-benchmarks` — step-by-step first run +- :doc:`/reference/configuration-files/cluster-file` — cluster file and orchestrator backends +- :doc:`/how-to/run-with-containers` — container backend walkthrough +- :doc:`/how-to/run-cvs-tests` — running other CVS suites diff --git a/docs/reference/configuration-files/vllm_singlenode_mi355x.rst b/docs/reference/configuration-files/vllm_singlenode_mi355x.rst deleted file mode 100644 index 73b09d0fa..000000000 --- a/docs/reference/configuration-files/vllm_singlenode_mi355x.rst +++ /dev/null @@ -1,213 +0,0 @@ -.. meta:: - :description: Configure the variables in the MI355X Single-Node vLLM configuration file - :keywords: inference, ROCm, install, cvs, vLLM, MI355X, LLM, single-node - -**************************************************** -MI355X single-node vLLM inference configuration file -**************************************************** - -MI355X single-node vLLM tests validate LLM inference performance using vLLM on AMD MI355X GPU systems. These tests ensure optimal throughput, latency, and scalability for large language model serving workloads on single-node configurations. - -The MI355X vLLM tests check: - -- **Container orchestration**: Docker setup with vLLM for single-node inference -- **Model serving**: LLM deployment with PagedAttention and continuous batching -- **Performance metrics**: Throughput, TTFT, TPOT, ITL, and E2EL -- **Multiple models**: GPT-OSS-120B, Qwen3-235B, Qwen3-80B, DeepSeek-V3.1 -- **Workload scenarios**: Balanced, long generation, and long context -- **Result verification**: Expected throughput and latency metrics - -Change the parameters as needed in the MI355X vLLM configuration file: ``mi355x_singlenode_vllm.json`` for single-node LLM serving. - -.. note:: - - - ``{user-id}`` will be resolved to the current username in the runtime. You can also manually change this value to your username. - -``mi355x_singlenode_vllm.json`` -================================ - -Here's a code snippet of the ``mi355x_singlenode_vllm.json`` file for reference: - -.. dropdown:: ``mi355x_singlenode_vllm.json`` - - .. code:: json - - { - "config": { - "container_image": "rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1", - "container_name": "vllm_inference_rocm", - "nnodes": "1", - "benchmark_server_script_path": "/home/{user-id}/benchmark_server_scripts/", - "benchmark_script_repo": "https://github.com/kimbochen/bench_serving.git", - "hf_token_file": "/home/{user-id}/.hf_token", - "shm_size": "16G", - "log_dir": "/home/{user-id}/LOGS", - "data_cache_dir": "/it-share/models/", - "container_config": { - "device_list": [ - "/dev/dri", - "/dev/kfd", - "/dev/mem" - ], - "volume_dict": { - "/home/{user-id}": "/home/{user-id}", - "/it-share/models/": "/models" - }, - "env_dict": { - "HF_HUB_CACHE": "/models/huggingface-cache" - } - } - }, - "benchmark_params": { - "gpt-oss-120b": { - "container_image": "rocm/7.0:rocm7.0_ubuntu_22.04_vllm_0.10.1_instinct_20250927_rc1", - "backend": "vllm", - "base_url": "http://0.0.0.0", - "port_no": "8888", - "dataset_name": "random", - "concurrency_levels": [16, 32, 64], - "model": "openai/gpt-oss-120b", - "num_prompts": "3200", - "sequence_combinations": [ - {"isl": "1024", "osl": "1024", "name": "balanced"}, - {"isl": "1024", "osl": "8192", "name": "long_generation"}, - {"isl": "8192", "osl": "1024", "name": "long_context"} - ], - "burstiness": "1.0", - "seed": "0", - "request_rate": "inf", - "max_model_length": "9216", - "random_range_ratio": "0.8", - "tensor_parallelism": "1", - "tokenizer_mode": "auto", - "percentile_metrics": "ttft,tpot,itl,e2el", - "metric_percentiles": "99", - "result_dict": { - "ISL=1024,OSL=1024,TP=1,CONC=16": { - "total_throughput_per_sec": "4651", - "mean_ttft_ms": "70", - "mean_tpot_ms": "8" - } - } - } - } - } - -Parameters -========== - -Use the parameters in this table to configure the MI355X vLLM configuration file. - -.. |br| raw:: html - -
- -.. list-table:: - :widths: 3 3 5 - :header-rows: 1 - - * - Configuration parameters - - Default values - - Description - * - ``container_image`` - - rocm/7.0:rocm7.0_ubuntu_22.04_ |br| vllm_0.10.1_instinct_20250927_rc1 - - Docker container image with vLLM for MI355X GPUs - * - ``container_name`` - - vllm_inference_rocm - - Name of the Docker container instance - * - ``nnodes`` - - 1 - - Number of nodes (single-node configuration) - * - ``benchmark_server_`` |br| ``script_path`` - - ``/home/{user-id}/`` |br| ``benchmark_server_scripts/`` - - Path to benchmark server scripts - * - ``benchmark_script_repo`` - - https://github.com/kimbochen/ |br| bench_serving.git - - GitHub repository for benchmark scripts - * - ``hf_token_file`` - - ``/home/{user-id}/`` |br| ``.hf_token`` - - Path to HuggingFace authentication token file - * - ``shm_size`` - - 16G - - Shared memory size for the container - * - ``log_dir`` - - ``/home/{user-id}/LOGS`` - - Directory for vLLM logs - * - ``data_cache_dir`` - - /it-share/models/ - - Directory for model cache - * - ``container_config.`` |br| ``device_list`` - - Values: |br| - ``"/dev/dri"`` |br| - ``"/dev/kfd"`` |br| - ``"/dev/mem"`` - - List of device paths to mount in the container for GPU access - * - ``container_config.`` |br| ``volume_dict`` - - ``{"/home/{user-id}":`` |br| ``"/home/{user-id}",`` |br| ``"/it-share/models/": "/models"}`` - - Dictionary mapping host paths to container paths for volume mounts - * - ``container_config.`` |br| ``env_dict.HF_HUB_CACHE`` - - /models/huggingface-cache - - HuggingFace model cache directory - * - ``benchmark_params.`` |br| ``.container_image`` - - Model-specific container image - - Container image for specific model benchmarks (overrides global container_image if set) - * - ``benchmark_params.`` |br| ``.backend`` - - vllm - - Inference backend to use (vLLM) - * - ``benchmark_params.`` |br| ``.base_url`` - - http://0.0.0.0 - - Base URL for the vLLM server - * - ``benchmark_params.`` |br| ``.port_no`` - - 8888 - - Port number for the vLLM server - * - ``benchmark_params.`` |br| ``.dataset_name`` - - random - - Dataset type for benchmarking (sharegpt, hf, random, sonnet, burstgpt) - * - ``benchmark_params.`` |br| ``.`` |br| ``concurrency_levels`` - - [16, 32, 64] - - List of concurrent request levels to test - * - ``benchmark_params.`` |br| ``.model`` - - Model identifier - - HuggingFace model identifier or local path (e.g., openai/gpt-oss-120b, Qwen/Qwen3-235B-A22B-Instruct-2507, Qwen/Qwen3-Next-80B-A3B-Instruct, deepseek-ai/DeepSeek-V3.1) - * - ``benchmark_params.`` |br| ``.num_prompts`` - - 3200 - - Total number of prompts to send during benchmarking - * - ``benchmark_params.`` |br| ``.`` |br| ``sequence_combinations`` - - Three scenarios - - List of input/output sequence length combinations with scenario names: balanced (ISL=1024, OSL=1024), long_generation (ISL=1024, OSL=8192), long_context (ISL=8192, OSL=1024) - * - ``benchmark_params.`` |br| ``.burstiness`` - - 1.0 - - Request burstiness factor (1.0 = uniform distribution, higher values create more bursty traffic) - * - ``benchmark_params.`` |br| ``.seed`` - - 0 - - Random seed for reproducible benchmark results - * - ``benchmark_params.`` |br| ``.request_rate`` - - inf - - Maximum request rate (inf = unlimited, or specify QPS) - * - ``benchmark_params.`` |br| ``.`` |br| ``max_model_length`` - - 9216 - - Maximum total sequence length the model can handle - * - ``benchmark_params.`` |br| ``.`` |br| ``random_range_ratio`` - - 0.8 - - Ratio for randomizing input/output lengths around specified values - * - ``benchmark_params.`` |br| ``.`` |br| ``random_prefix_len`` - - 0 - - Length of random prefix for shared prefix testing - * - ``benchmark_params.`` |br| ``.`` |br| ``tensor_parallelism`` - - Varies per model - - Number of GPUs to use for tensor parallelism (1 for GPT-OSS-120B and Qwen3-80B, 8 for Qwen3-235B and DeepSeek-V3.1) - * - ``benchmark_params.`` |br| ``.tokenizer_mode`` - - auto - - Tokenizer mode (auto, slow, mistral, custom) - * - ``benchmark_params.`` |br| ``.`` |br| ``percentile_metrics`` - - ttft,tpot,itl,e2el - - Comma-separated list of metrics to compute percentiles for (ttft: Time to First Token, tpot: Time Per Output Token, itl: Inter-Token Latency, e2el: End-to-End Latency) - * - ``benchmark_params.`` |br| ``.`` |br| ``metric_percentiles`` - - 99 - - Percentile values to compute for metrics (e.g., 99 for 99th percentile) - * - ``benchmark_params.`` |br| ``.server_script`` - - Model-specific script - - Shell script to launch vLLM server with model-specific configuration - * - ``benchmark_params.`` |br| ``.`` |br| ``bench_serv_script`` - - benchmark_serving.py - - Python script to run the benchmarking client - * - ``benchmark_params.`` |br| ``.result_dict`` - - Model-specific baselines - - Dictionary of expected performance results for each workload scenario, specifying total_throughput_per_sec, mean_ttft_ms, and mean_tpot_ms for different combinations of ISL (Input Sequence Length), OSL (Output Sequence Length), TP (Tensor Parallelism), and CONC (Concurrency) diff --git a/docs/sphinx/_toc.yml.in b/docs/sphinx/_toc.yml.in index 6a1aa4887..10373bbcd 100644 --- a/docs/sphinx/_toc.yml.in +++ b/docs/sphinx/_toc.yml.in @@ -19,6 +19,8 @@ subtrees: title: Run tests - file: how-to/run-with-containers title: Run tests with the container backend + - file: how-to/run-vllm-benchmarks + title: Run vLLM inference benchmarks - file: how-to/run-cluster title: Monitor the health of GPU clusters @@ -46,10 +48,10 @@ subtrees: title: MORI - file: reference/configuration-files/aorta title: Aorta - - file: reference/configuration-files/inferencemax - title: InferenceMAX - - file: reference/configuration-files/vllm_singlenode_mi355x - title: vLLM Single-Node MI355X + - file: reference/configuration-files/atom + title: ATOM + - file: reference/configuration-files/vllm + title: vLLM Inference - file: reference/configuration-files/sglang title: SGLang Disaggregated - file: reference/configuration-files/flux1_t2i diff --git a/docs/what-is-cvs.rst b/docs/what-is-cvs.rst index 1e03b45f9..73a0aef7a 100644 --- a/docs/what-is-cvs.rst +++ b/docs/what-is-cvs.rst @@ -23,7 +23,7 @@ Here are the tests available in the CVS: - **RDMA performance tests**: Validate RDMA (Remote Direct Memory Access) bandwidth and latency with MORI for high-speed inter-node communication using AMD Pensando AINIC and other RDMA-capable devices. - **Inference tests**: Validate LLM serving performance and generative AI workloads across AMD GPU clusters. - - InferenceMAX benchmarks vLLM inference performance for models like GPT-OSS-120B, measuring throughput, TTFT (Time to First Token), and TPOT (Time Per Output Token). + - ATOM benchmarks vLLM inference performance for models like GPT-OSS-120B, measuring throughput, TTFT (Time to First Token), and TPOT (Time Per Output Token). - vLLM single-node tests support multiple models (GPT-OSS-120B, Qwen3-235B, Qwen3-80B, DeepSeek-V3.1) with various workload scenarios on MI355X GPUs. - SGLang disaggregated prefill-decode architecture tests optimize LLM serving by separating prefill and decode phases across different nodes. - Flux.1 text-to-image generation tests validate distributed image generation using xDiT with Ulysses and Ring parallelization. diff --git a/plans/building-a-cvs-test-suite.md b/plans/building-a-cvs-test-suite.md new file mode 100644 index 000000000..e97f70cb8 --- /dev/null +++ b/plans/building-a-cvs-test-suite.md @@ -0,0 +1,270 @@ +**Building a CVS test suite — a reference guide** + +This guide documents the base infrastructure for writing a new **inference** or +**training** test suite in CVS, using the `vllm_single` suite (PR against +`dev/dtni`) as the worked reference implementation. If you are adding a serving +framework, a training benchmark, or any new workload, base your structure on the +patterns here rather than on the older `InferenceBaseJob` / `if_dict` style. + +> **Status note.** CVS is experimental. Flags and config keys change, and the +> built-in pass/fail is not a trustworthy oracle on its own — always verify a +> run's artifacts independently. This guide describes the structure, not a frozen +> API. + +> **Training is not ported yet.** Only `vllm_single` (inference) exists today. +> The library layout, the generic↔domain seam, and the suite skeleton below are +> deliberately framework-neutral so a training suite drops into the same shape — +> this guide is the blueprint for that port, not a record of it. + +### What changed in the restructure + +This PR reorganizes where shared vs. workload-specific code lives, so the next +suite reuses machinery instead of copy-pasting it. + +**`cvs/lib/dtni` → `cvs/lib/utils` (renamed).** `dtni` was a leftover project +codename; the directory actually holds pure, framework-agnostic helpers. The new +name says what it is: utilities any suite (inference, training, …) can call. + +**`cvs/lib/utils/` — the shared/common layer.** Everything here is generic: no +suite knows about vLLM, Megatron, ISL/OSL, or goodput. It holds: +- `config_loader.py` — the generic config skeleton (`BaseVariantConfig`), the + `paths`/`model`/`image`/`container` schema, the 3-pass placeholder + substitution engine, and `substitute_config()` (read a config + its sibling + threshold file, resolve placeholders). +- `verdict.py` — `evaluate_all(actuals, thresholds)`, a metric-name-agnostic + threshold checker (`min`, `max_ms`, `within`, `min_tok_s`, `min_ratio`). + +**`cvs/lib/inference/utils/` — the domain-specific layer.** Helpers only an +*inference* suite needs live here, one level down from the shared dir. It holds: +- `inferencing_config_loader.py` — the inference config schema: the named-combo + sweep selector, `GoodputSlo`, the vLLM bench `Params`, and + `VariantConfig(BaseVariantConfig)` with the ISL/OSL/TP/CONC `cell_key`. It + *imports* `BaseVariantConfig` + `substitute_config` from `cvs/lib/utils` rather + than re-implementing them. +- `vllm_parsing.py` — the `client.*` metric vocabulary (`to_client_metrics`, + `CLIENT_METRICS`): pure transforms from a vLLM benchmark artifact to a + namespaced metric dict. + +**The rule for where a helper goes.** If every workload could use it (config +plumbing, threshold math) → `cvs/lib/utils/`. If it only makes sense for one +domain (sweep shapes, serving metrics) → `cvs/lib//utils/`. A training +port adds `cvs/lib/training/utils/` the same way inference did — subclass +`BaseVariantConfig`, reuse `substitute_config` and `evaluate_all`, add its own +schema and metric vocabulary. The shared layer never grows a dependency on a +specific framework. + +``` +cvs/lib/ + utils/ # shared, framework-agnostic (was: dtni) + config_loader.py # BaseVariantConfig, substitute_config, placeholders + verdict.py # evaluate_all (generic threshold kinds) + inference/ + utils/ # inference-only helpers + inferencing_config_loader.py # sweep selector, Params, VariantConfig, cell_key + vllm_parsing.py # client.* metrics, to_client_metrics + vllm_single.py # the driver (VllmJob) + training/ # (future) same shape: + utils/ # training_config_loader.py, _parsing.py +``` + +### The layered architecture + +A suite is built from six layers. Each has one job; the seams between them are +the reusable surface a second suite plugs into. + +``` + cluster file (node IP + user/key/orchestrator) <- per-environment, OUT of repo + | + variant config (*_config.json + *threshold.json) <- per-workload, IN repo + | + config loader ── cvs/lib/utils/config_loader.py (generic: schema, substitution) + | └ cvs/lib/inference/utils/inferencing_config_loader.py (domain schema + sweep) + | + orchestrator ── cvs/core/orchestrators (ContainerOrchestrator; provided) + | + job / driver ── cvs/lib/inference/vllm_single.py (launch server, run client, fetch artifact) + | + suite / pytest ── cvs/tests/inference/vllm/ (lifecycle-as-tests, parametrization, HTML) + | + parsing + verdict ── vllm_parsing.to_client_metrics + cvs/lib/utils/verdict.evaluate_all +``` + +The **generic↔domain seam** is the key idea: anything every workload shares lives +in `cvs/lib/utils/`; anything specific to *your* workload lives in your own +`cvs/lib//utils/` module that subclasses/reuses it. The config loader +docstring literally calls this the "generalization seam," and this PR executes +the split. + +### The two config files (per workload) + +A workload is described by a pair of JSON files in the same directory: + +- `*_config.json` — the variant: paths, model, container (with `container.image`), + params, sweep. +- `*threshold.json` — the per-cell pass/fail thresholds. + +The loader finds the threshold by **sibling glob** (exactly one `*threshold.json` +next to the config), so the two share a descriptive prefix +(`llama31_70b_fp8_config.json` / `llama31_70b_fp8_threshold.json`). + +#### Placeholders + +Config values use `{...}` placeholders resolved in three passes: +`{user-id}` (from the cluster file) → `{shared_fs}` (self-reference within +`paths`) → `{paths.models_dir}` (cross-block). An unknown placeholder is left +literal — there is no error, so check for a stray brace in a resolved path if +something doesn't mount. + +#### enforce_thresholds + +`enforce_thresholds: false` makes the suite **record-only**: it captures every +metric and skips assertions, and a threshold/sweep mismatch warns instead of +failing the load. Use it for un-calibrated workloads (e.g. throughput +characterization where the published numbers are curves, not tabulated cells). +Flip to `true` once you have real numbers — the coverage check then guarantees +both that every sweep cell has a threshold entry AND that every cell carries a +spec for every gated metric (`GATED_METRICS`), so a green run can't have +silently skipped its verdict. + +### The sweep selector (named combos + runs) + +The sweep enumerates exactly the `(sequence-shape, concurrency)` cells to run. It +replaces the old `sequence_combinations × concurrency_levels` cartesian: + +```json +"sweep": { + "sequence_combinations": [ + { "name": "w1_isl=128_osl=2048", "isl": "128", "osl": "2048", + "goodput_slo": { "ttft_ms": 1000000000.0, "tpot_ms": 1000000000.0, "e2el_ms": 1000000000.0 } } + ], + "runs": [ + { "combo": "w1_isl=128_osl=2048", "concurrency": 16 } + ] +} +``` + +- Combos are named once; `runs` cherry-picks `(combo, concurrency)` pairs. No NxM + explosion — you list precisely the cells you want. +- Load-time validation rejects duplicate combo names and any `run.combo` that + names no combo. (This is mirrored in the suite's `pytest_generate_tests`, which + reads raw JSON at collection time before the typed loader runs.) +- `goodput_slo` is an **input** to the run (passed to `vllm bench serve + --goodput`), not a threshold — that's why it lives in the sweep. + +Each cell's threshold key is produced by `VariantConfig.cell_key()` +(`ISL=…,OSL=…,TP=…,CONC=…`). It is the single source of truth shared by the +coverage check and the verdict lookup, so threshold.json keys must match it +exactly. + +### The job/driver (self-contained, no external .sh) + +`vllm_single.VllmJob` is the reference driver. It talks only to an injected +orchestrator (`orch.exec`, which routes into the running container) and a typed +`VariantConfig`. Lifecycle highlights to copy: + +- **The server command is built in Python** (`_server_argv`), not cloned from an + external `.sh` repo. A run is self-contained. Per-model quirks come from + `roles.server.serve_args` (a `{flag: value}` map) / `roles.server.env` in + config. +- **`/tmp/server_env_script.sh`** is written by `build_server_cmd` and **sourced + by both server and client**, so the two share one environment (HF token, cache + pin, AITER flags). Each value is `shlex.quote`d. +- **`--max-model-len` is derived per cell** from isl/osl/random_range_ratio so a + sweep change stays self-consistent. +- **Readiness/completion are detected by scanning the whole log**, with narrow + failure markers (don't match bare `error:` — ROCm/vLLM logs benign ones). +- **Results come from the stock `results` artifact**, not console-regex. + `parse_results` fetches the extensionless JSON `vllm bench serve` writes to + `--result-dir` and hands it to the pure `to_client_metrics`. Missing/empty/ + unparseable → hard-fail the cell (never a silently-green empty row). + +The fetch lives in the job (artifact layout is job-specific); the transform lives +in `inference/utils` (so distributed/disagg/InferenceMax reuse it). + +### The suite (lifecycle-as-tests) + +In `cvs/tests/inference/vllm/`, each lifecycle stage is its own pytest test so it +shows up as a timed, pass/fail row in the HTML report: + +``` +test_launch_container → test_setup_sshd → test_model_fetch + → test_vllm_inference (per cell) → test_metric (per metric per cell) + → test_print_results_table → test_teardown +``` + +Patterns to copy: + +- **`pytest_generate_tests`** (in the suite module, not conftest) parametrizes + from the sweep selector. It runs at collection time and reads raw JSON, so it + re-validates combos by hand (mirroring the typed loader). +- **`test_vllm_inference` runs the benchmark once per cell** and stashes results + in a module-scoped `inf_res_dict`. **`test_metric` reads one cached metric** and + is one HTML row per metric per cell — no GPU work, asserts only when + `enforce_thresholds` is true and a spec exists. +- **`_Lifecycle`** (`conftest`) carries cross-test state: `failed` lets a broken + stage skip the rest instead of cascading; `torn_down` lets explicit teardown + suppress the fixture leak-guard finalizer. +- **The `orch` fixture owns only the teardown safety net** — launch/sshd happen + in tests so they're timed rows. A mid-sweep failure still tears the container + down via the finalizer. +- **Single-node guards**: `test_setup_sshd` only probes port 2224 when + `len(orch.hosts) > 1` (in-container sshd exists only for inter-node MPI). +- **HTML Value/Unit columns** come from `pytest_html_results_table_header` / + `_row` hooks scoped to this conftest, populated from + `metric_value`/`metric_unit` user-properties. + +### Parsing + verdict + +- **`to_client_metrics(raw, *, tp, isl)`** — pure: stock keys namespaced + `client.*` 1:1, plus derived metrics (`per_gpu_throughput`, + `decode_throughput_p50`, `success_rate`, …) via `_safe_div` (degrades to + `None`, never crashes). `CLIENT_METRICS` is the ordered display surface. +- **`evaluate_all(actuals, thresholds)`** — generic, framework-neutral. Kinds: + `min`, `max`, `max_ms`, `within`, `min_tok_s`, `min_ratio`. Raises + `ThresholdViolation` listing every failure. A `None` actual is a loud + violation, not a TypeError. +- **`GATED_METRICS`** (in `vllm_parsing`) — the asserted SLO subset of + `CLIENT_METRICS`. The loader's coverage check requires a threshold spec for + every gated metric in every present cell, so a gated metric can't silently + fall through to a zero-assertion record-only row. A new metric is record-only + until added to the set. + +### To add your own suite — checklist + +1. **Schema.** If serving: reuse `inferencing_config_loader` (subclass `Params` + for a new framework's flags). If a new domain (training): create + `cvs/lib//utils/_config_loader.py`, subclass + `BaseVariantConfig`, reuse `substitute_config` + `evaluate_all` from + `cvs/lib/utils/`. Define your own `cell_key` + coverage check. +2. **Config pair.** Write `*_config.json` + `*threshold.json` under + `cvs/input/config_file////`. Start + `enforce_thresholds: false` until calibrated. +3. **Driver.** Write a `Job` class that takes an `orch` + typed config and owns + launch → run → fetch-artifact → `parse`. Build commands in Python; keep the + pure transform in your `utils`. Hard-fail on missing artifacts. +4. **Suite.** Under `cvs/tests///`: `conftest.py` (fixtures + + lifecycle + HTML hooks), the suite module (`pytest_generate_tests` + + lifecycle-as-tests), `_shared.py` (the results table). Pin test order in + `pytest_collection_modifyitems`. +5. **Metrics.** Define your metric vocabulary + units list once in your `utils`, + the way `CLIENT_METRICS` does — don't re-list rows per suite. +6. **Cluster file.** Keep it minimal and OUT of the repo: node IP + user + key + + orchestrator. The variant config supplies the container block; don't ship a + bespoke per-suite cluster file. +7. **Verify independently.** After a run, read the artifact + the HTML cells + yourself. CVS's PASS is not a trustworthy oracle. + +### Reference files (this PR) + +| Layer | File | +|---|---| +| generic schema + substitution | `cvs/lib/utils/config_loader.py` | +| generic verdict | `cvs/lib/utils/verdict.py` | +| inference schema + sweep | `cvs/lib/inference/utils/inferencing_config_loader.py` | +| client metric vocabulary | `cvs/lib/inference/utils/vllm_parsing.py` | +| driver | `cvs/lib/inference/vllm_single.py` | +| suite | `cvs/tests/inference/vllm/{vllm_single,conftest,_shared}.py` | +| config pair | `cvs/input/config_file/inference/vllm_single/w1_llama31_70b_fp8kv/` | + +Agent-facing summaries (entry points + gotchas) live in each package's +`AGENTS.md`. diff --git a/plans/dtni-dev-guide.md b/plans/dtni-dev-guide.md new file mode 100644 index 000000000..62e8008ed --- /dev/null +++ b/plans/dtni-dev-guide.md @@ -0,0 +1,352 @@ +# DTNI suite developer guide + +> **⚠️ SUPERSEDED — design doc, not current state.** This guide predates the +> restructure that shipped. It still references the old `cvs/lib/dtni/` and +> `cvs/input/dtni/` paths (now `cvs/lib/utils/` + `cvs/lib/inference/utils/`), +> the `concurrency_levels × sequence_combinations` cartesian (now named combos + +> a `runs[]` selector), the top-level `image` block (now `container.image`), +> `OrchestratorConfig.from_dicts` / `container.enabled|launch`, and other +> pre-merge shapes. For the as-built architecture, read +> `plans/building-a-cvs-test-suite.md` and each package's `AGENTS.md`. Kept for +> the design rationale (the §5–§7 *why* behind the seam, the Job split, and the +> config/threshold split), which is still accurate. + +## 1. Intro and scope + +This guide is for CVS developers who already run `cvs run` regularly and have edited a test wrapper or a lib helper, but who haven't worked under the new DTNI (data-center training and inference) layout. The goal is to give you the mental model and the concrete skeleton needed to port an existing suite or author a new one. + +The framing: today every suite is a hand-written pytest module that ships its own container lifecycle, its own config parsing, and its own threshold checks inline. Under DTNI, those concerns move out of the test module into shared machinery — a typed config loader, an `orch` (orchestrator) fixture that owns the container, and a per-framework Job class that bundles the framework-specific verbs — so the test module shrinks to a few phases: load → setup → generated tests → custom tests. + +`vllm_single` (inference) and `megatron_*` (training) appear as running examples. The same shape applies to sglang, atom, pytorch_xdit, jax. + +## 2. Old lifecycle: `cvs run` to HTML report + +```mermaid +flowchart TD + A[cvs run suite --cluster_file --config_file] --> B[cvs/cli_plugins/run_plugin.py] + B --> C[pytest invocation] + C --> D[wrapper module imports] + D --> E[6 module fixtures
cluster_file, *_dict, hf_token, phdl, gpu_type] + E --> F[test_cleanup_stale_containers
docker_lib.kill + delete_all] + F --> G[test_launch_*_containers
docker_lib.launch_docker_container] + G --> H[parametrized workload test
Job.start/poll/verify] + H --> I[test_print_results_table
tabulate inf_res_dict] + I --> J[pytest HTML report + exit code] +``` + +Concrete trace using `cvs/tests/inference/vllm/vllm_qwen3_80b_single.py` (480 LOC) and `cvs/tests/training/megatron/megatron_llama3_1_8b_single.py` (315 LOC): + +1. **CLI entry.** `cvs/cli_plugins/run_plugin.py` parses args and invokes pytest against the resolved test module. `cvs list ` uses `cli_plugins/list_plugin.py` (which calls `pytest --collect-only -q`). +2. **Fixtures.** Each wrapper declares ~6 module-scoped fixtures that re-implement the same shape: `cluster_file`, `_config_file`, `cluster_dict`, `_dict`, helper dicts (`benchmark_params_dict` for inference, `model_params_dict` for training), `hf_token`, and a Pssh handle (`s_phdl`/`c_phdl` for inference, `phdl` for training). Dicts are loaded via raw `json.load` and run through `resolve_test_config_placeholders`. The training wrapper also probes `rocm-smi -a` live to derive `gpu_type`. +3. **Container lifecycle as ordered pytest functions.** `test_cleanup_stale_containers` calls `cvs/lib/docker_lib.py` (`kill_docker_container`, `delete_all_containers_and_volumes`). `test_launch__containers` calls `docker_lib.launch_docker_container` with device/volume/env/shm-size pulled from the loaded dict. Distributed training wrappers add a third lifecycle test (`test_disable_firewall`) that shells out via `phdl.exec('sudo service ufw stop')` to work around torchrun rendezvous timeouts. Inference wrappers add an autouse `cleanup_on_exit` fixture that kills the container again on module teardown. +4. **Workload.** Inference: parametrized `test__inference[-conc]` builds a `VllmJob` (`cvs/lib/inference/vllm.py`, subclass of `InferenceBaseJob` in `cvs/lib/inference/base.py`, 711 LOC), mutates a shared `benchmark_params_dict` with the current cell's params, calls `build_server_inference_job_cmd → start_inference_server_job → wait_for_health → start_inference_client_job → verify_inference_results`, parses into module-level `inf_res_dict`. Training: a single non-parametrized test builds a `MegatronLlamaTrainingJob` (`cvs/lib/megatron_training_lib.py`, 834 LOC, no shared base) and calls `exec_nic_setup_scripts → build_training_job_cmd → start_training_job → poll_for_training_completion → verify_training_results`. Both flows accumulate errors into the global `globals.error_list` and call `update_test_result()` at the end. +5. **Output.** Inference: `test_print_results_table` tabulates `inf_res_dict`. Training: no separate printer; verification is inline. Both emit the pytest HTML report and exit with the pytest exit code. + +The seams that hurt: + +- 4 wrappers per suite, byte-similar with one knob different (model id for inference, `distributed_training=False/True` for training). Any cluster-config schema change must be reapplied 4 times per suite. +- Container lifecycle encoded as ordered pytest functions plus an autouse fixture. Test ordering matters; shared state in `inf_res_dict`/`globals.error_list` is implicit. +- 700-800 LOC Job classes that mix command building, remote execution, output parsing, and pass/fail thresholds. `InferenceBaseJob` has vllm-shaped env vars leaked into the base, a dead distributed branch, the `random_range_ration` typo, and a silent skip in `verify_inference_results`. New suites that subclass it inherit the bugs. +- Cluster probes (`rocm-smi`, firewall status) live inside fixtures or workaround tests, called directly through `phdl.exec`. No consistent way to "ask the cluster something." +- Configuration is a single JSON blob mixing "what to run" with "did it pass." No typing, no schema, no separable lifecycle. + +## 3. New lifecycle under DTNI + +The CLI surface and pytest invocation are unchanged. What changes is what the test module looks like and where the work lives. + +```mermaid +flowchart TD + A[cvs run suite --cluster_file --config_file] --> B[cvs/cli_plugins/run_plugin.py] + B --> C[pytest invocation] + C --> D[wrapper module imports] + D --> E[Phase 1: load
config_loader.load_variant
returns typed config + thresholds] + E --> F[Phase 2: setup
orch fixture builds OrchestratorConfig
orch.setup_containers] + F --> G[Phase 3: generated tests
pytest_generate_tests parametrizes
over benchmark_params / model_params] + G --> H[Phase 4: custom tests
suite-specific assertions
e.g. firewall, NIC setup, print_results] + H --> I[orch.teardown_containers] + I --> J[pytest HTML report + exit code] +``` + +Phase-by-phase: + +1. **Load.** `cvs/lib/dtni/config_loader.py` reads `cvs/input/dtni///config.json` and `threshold.json`, validates through Pydantic models (`extra="forbid"` everywhere except the runtime args passthrough), runs placeholder substitution in fixed order (cluster → self-reference → cross-block), and returns typed objects. One wrapper per suite, parametrized across all variant directories. +2. **Setup.** A pytest fixture builds an `OrchestratorConfig` (`cvs/core/orchestrators/factory.py`) from the loaded config and yields an `orch`. The fixture calls `orch.setup_containers()` on entry and `orch.teardown_containers()` on exit. No `test_cleanup_stale_containers` or `test_launch_*_containers` in suite code — those concerns are gone. +3. **Generated tests.** `pytest_generate_tests` (in the suite's `conftest.py`) reads sweep dimensions from the typed config — `benchmark_params.concurrency_levels × sequence_combinations` for inference, `model_params` presets for training — and parametrizes the workload test. Each parametrize cell constructs a Job, calls its verbs, gets back a flat `actuals` dict, and runs `evaluate_all(actuals, thresholds)`. +4. **Custom tests.** Suite-specific assertions that aren't part of the sweep grid: firewall disable, NIC setup probe, results-table printer, smoke checks. These live in the wrapper as plain `def test_*` functions and use `orch.exec`/`orch.exec_on_head` to talk to the cluster — never `phdl.exec` or `docker_lib` directly. + +Output: same pytest HTML report, same exit code. Reportable rows are built from `actuals`, not from a global dict mutated by ordered tests. + +## 4. Test file skeleton (the phases, concretely) + +This is the base layout. Copy and replace the framework-specific bits. + +```python +# cvs/tests///.py +""" — DTNI layout. Phases: load → setup → generated → custom.""" + +import pytest +from cvs.lib.dtni.config_loader import load_variant, enumerate_variants +from cvs.lib.dtni.verdict import evaluate_all +from cvs.lib.._orch import Job + + +# -------- Phase 1: load (delegated to conftest.py fixtures) -------- +# variant_config + thresholds come from the conftest's load_variant fixture. +# This module does not call json.load. + +# -------- Phase 2: setup (delegated to conftest.py fixtures) -------- +# orch comes from the conftest's orch fixture. Container lifecycle is +# owned by the fixture (setup_containers on entry, teardown on exit). + +# -------- Phase 3: generated tests -------- +# Parametrization is driven by pytest_generate_tests in conftest.py, +# walking variant_config.benchmark_params (or model_params for training). + +def test_(orch, variant_config, hf_token, cell, inf_res_dict): + job = Job(orch=orch, config=variant_config, hf_token=hf_token) + job.stop() # idempotent pre-clean + job.start(cell) # framework verbs + job.wait_ready() + job.run(cell) + job.wait_complete() + actuals = job.parse_results() + inf_res_dict[cell.id] = actuals + evaluate_all(actuals, variant_config.thresholds, prefix=cell.id) + + +# -------- Phase 4: custom tests -------- +# Suite-specific assertions outside the sweep grid. Use orch, not phdl. + +def test_print_results_table(inf_res_dict): + from cvs.tests..._shared import print_table + print_table(inf_res_dict) + +# Training-flavored example (distributed quirk): +# def test_disable_firewall(orch): +# out = orch.exec("sudo service ufw stop || true") +# out = orch.exec("sudo ufw status") +# for node, text in out.items(): +# assert "inactive" in text.lower() or "disabled" in text.lower(), node +``` + +And the conftest that backs it: + +```python +# cvs/tests///conftest.py +import pytest +from cvs.lib.dtni.config_loader import load_variant, enumerate_variants +from cvs.core.orchestrators.factory import OrchestratorConfig, build_orchestrator + +def pytest_generate_tests(metafunc): + if "variant_config" in metafunc.fixturenames: + variants = enumerate_variants("cvs/input/dtni/") + metafunc.parametrize("variant_config", variants, ids=[v.id for v in variants], indirect=True) + if "cell" in metafunc.fixturenames: + # Cross-product of sweep dims from the already-resolved variant_config. + ... + +@pytest.fixture(scope="module") +def cluster_dict(pytestconfig): ... + +@pytest.fixture +def variant_config(request, cluster_dict): + return load_variant(request.param, cluster_dict) + +@pytest.fixture +def orch(variant_config, cluster_dict): + oc = OrchestratorConfig.from_dicts(cluster_dict, variant_config.container.dict()) + o = build_orchestrator(oc) + o.setup_containers() + try: + yield o + finally: + o.teardown_containers() + +@pytest.fixture(scope="session") +def inf_res_dict(): + return {} +``` + +Conventions baked into this skeleton: + +- The wrapper does not import `docker_lib`, `parallel_ssh_lib`, or `globals`. Anything those modules did is now reachable through `orch` or `evaluate_all`. +- The wrapper does not `json.load` anything. Config IO lives in `config_loader`. +- Job verbs are framework-specific but consistent in *shape*: a small constructor, a few verbs that drive remote work through `orch`, and a `parse_results()` that returns a flat dict. The exact verbs differ by domain (inference: `start_server/run_client`; training: `start_training/poll_for_completion`). +- `pytest_generate_tests` is the only place parametrization lives. No ad-hoc `@pytest.mark.parametrize` on the workload test. + +## 5. The three new concepts + +### `orch` (the orchestrator fixture) + +**What.** An object from `cvs/core/orchestrators/factory.py` that abstracts "run a command on cluster nodes" and, when `container.enabled=true`, "run that command inside a container managed by me." Methods you'll touch: `setup_containers`, `teardown_containers`, `exec`, `exec_on_head`. Routing between baremetal and container is automatic. + +**Why.** Today every suite re-implements container launch and cleanup as ordered pytest functions plus an autouse fixture, calling `docker_lib` directly, with bonus shell workarounds (firewall, NIC scripts) reaching past it into `phdl.exec`. That couples test order to lifecycle order, duplicates teardown, and forces every wrapper to know about Docker flags. `orch` collapses this to one fixture: the test sees a ready environment when it starts and a clean one when it ends. + +### The Job class + +**What.** A standalone Python class, one per framework, under `cvs/lib//_orch.py`. It bundles the framework-specific verbs and uses an injected `orch` for all remote execution. Domain shapes verb names: inference Jobs expose `build_server_cmd / start_server / wait_ready / run_client / parse_results`; training Jobs expose `build_training_cmd / start_training / poll_for_completion / parse_results`; pre-workload shell workarounds (NIC setup, firewall) become methods on the Job rather than ordered tests. + +**Why.** `InferenceBaseJob` (711 LOC) tried to be a shared base for every inference framework and ended up tangling vllm-shaped env vars into the base, with a dead distributed branch and silent skips in result verification. `MegatronLlamaTrainingJob` (834 LOC) avoided the base-class trap but mixed config parsing, command building, remote execution, output parsing, and pass/fail thresholds in one file. The new shape: small, flat, framework-specific Job; cluster talk via `orch`; thresholds via `evaluate_all`. No inheritance, no `globals.error_list`. + +### The config / threshold split + +**What.** The single suite config JSON splits into two files per variant directory: `config.json` (what to run — identity, paths, model, image, container, framework params, sweep dimensions) and `threshold.json` (did it pass — flat map of `.` to typed predicates). + +**Why.** Different churn rates (config flips with new models or images; thresholds drift with hardware/kernel/version moves), different ownership (suite author vs perf/release), different lifecycles (re-baseline thresholds without re-reviewing the whole suite). Splitting also makes `cvs list` granular at the variant level and removes v1's anti-pattern of encoding non-metric checks ("did the server start") as a numeric threshold. + +## 6. What the current lib/Job files try to do — and what to lift out + +Read this as a refactor map for the legacy files, not a critique. Use it to decide what lands in the new Job, what lands in shared machinery, and what stays in the old file for legacy suites. + +### `cvs/lib/inference/base.py` — `InferenceBaseJob` (711 LOC) + +Concerns currently mixed: + +| Concern | Current home | DTNI home | +|---|---|---| +| Container launch flags | base init pulls from `inference_dict['container_config']` | **orch** (passed through `container` block) | +| Server command build | `build_server_inference_job_cmd` | **Job** (framework-specific) | +| Remote process start | `start_inference_server_job` → `phdl.exec` | **Job** uses `orch.exec_on_head` | +| Health wait | `wait_for_inference_server_health` | **Job** (verb on the Job) | +| Client command build | `build_client_inference_job_cmd` | **Job** | +| Result parsing | `parse_inference_results` → `inf_res_dict` | **Job.parse_results** returns a flat dict | +| Threshold check | `verify_inference_results` (silent-skip bug) | **evaluate_all** (shared, predicate-typed) | +| Error accumulation | `globals.error_list` | plain `assert` + `evaluate_all` | + +What to lift out: container concerns (to `orch`), threshold checks (to `evaluate_all`), the global error list (delete entirely). What stays on the Job: framework verbs. Net: the new `VllmJob` is in the 200–300 LOC range instead of 711. + +### `cvs/lib/megatron_training_lib.py` — `MegatronLlamaTrainingJob` (834 LOC) + +Same split, training-flavored: + +| Concern | Current home | DTNI home | +|---|---|---| +| Per-model presets (`single_node`/`multi_node` × `gpu_type`) | `model_params_dict` indexed inside Job | **variant config** (one variant per preset) | +| NIC setup script execution | `exec_nic_setup_scripts` | **Job method**, calling `orch.exec` | +| Training command build | `build_training_job_cmd` | **Job** | +| Launch + poll | `start_training_job` + `poll_for_training_completion` | **Job** (drives via `orch`) | +| Log scan for errors | `scan_for_training_errors` | **Job.parse_results** returns flat metrics dict; **evaluate_all** gates pass/fail | +| Threshold check | `verify_training_results` | **evaluate_all** | +| Distributed-only workarounds (firewall) | extra ordered pytest function in distributed wrappers | **Job method or custom Phase-4 test using orch** | + +Optimizations specific to training: + +- The "single vs distributed" axis is a config dimension, not a separate suite. With one wrapper parametrized by variant directory, `llama3.1_8b_fp8_single` and `llama3.1_8b_fp8_distributed` are sibling variant dirs sharing one wrapper. +- `gpu_type` derived by `rocm-smi` probe in a fixture today; in DTNI it's either declared in the variant (`gpu_arch: mi300x`) or queried once via `orch.exec_on_head` and cached on `orch`. + +### `cvs/lib/docker_lib.py` and `cvs/lib/parallel_ssh_lib.py` + +Both are reachable from DTNI suites via `orch` indirection, but DTNI Job code should never import them directly. `parallel_ssh_lib` is a deprecated shim around `cvs/lib/parallel/pssh.py`; the orch already uses the new path. New suites referencing either of these by name should fail review. + +## 7. Config vs threshold: philosophy and concrete shape + +**Why split.** + +- Different churn rates. Configs change when you bring up a new model, image, or container layout. Thresholds change when hardware, kernels, or framework versions move performance characteristics. +- Different ownership. Suite author owns config; perf or release engineering often owns thresholds. +- Different lifecycle. Re-baselining thresholds for a new MI generation should not require touching what the run does. +- Granular `cvs list`. One variant directory = one collected suite row, regardless of how many metrics it gates. +- Rejects v1's anti-pattern of encoding "did the thing start" as `min: 1` against some token counter. Liveness belongs in `assert` inside the test; thresholds are for numeric pass/fail on real measurements. + +### Example: `config.json` for a `vllm_single` variant + +```json +{ + "schema_version": 1, + "framework": "vllm_single", + "gpu_arch": "mi355x", + "paths": { + "shared_fs": "/mnt/dtni/{user-id}/cvs", + "models_dir": "/mnt/dtni/{user-id}/models", + "datasets_dir": "{shared_fs}/datasets", + "artifacts_dir": "{shared_fs}/artifacts" + }, + "model": {"id": "Qwen3-Next-80B-A3B-Instruct", "remote": 0, "precision": "bf16"}, + "image": {"tag": "rocm/vllm:latest", "remote": 1}, + "container": { + "enabled": true, + "launch": true, + "name": "vllm_inference_rocm", + "runtime": { + "name": "docker", + "args": { + "volumes": {"{paths.models_dir}": "/models"}, + "env": {"VLLM_USE_TRITON_FLASH_ATTN": "0"}, + "shm_size": "64G" + } + } + }, + "params": {"tensor_parallelism": 8, "max_model_len": 8192, "gpu_memory_utilization": 0.85}, + "benchmark_params": { + "concurrency_levels": [16], + "sequence_combinations": [{"name": "balanced", "isl": 1024, "osl": 1024}] + } +} +``` + +Block walkthrough: + +- `schema_version`, `framework`, `gpu_arch` — identity. Loader picks the right Pydantic model and Job class. +- `paths` — substituted in fixed order: cluster (`{user-id}`) → self-reference (`{shared_fs}`) → cross-block (`{paths.X}` used elsewhere). +- `model`, `image` — typed objects with explicit `remote` flags. +- `container` — passed through to `OrchestratorConfig.container`. `launch: true` hands lifecycle to orch. +- `params` — framework server flags. Passthrough; the framework's Pydantic model decides what's allowed. +- `benchmark_params` — sweep dimensions. `pytest_generate_tests` reads these to parametrize. + +### Example: `threshold.json` for the same variant + +```json +{ + "smoke_request_latency_ms": {"kind": "max_ms", "value": 600000}, + "smoke_completion_tokens": {"kind": "min", "value": 1}, + + "balanced_conc16.request_throughput": {"kind": "min", "value": 0.5}, + "balanced_conc16.output_throughput": {"kind": "min_tok_s", "value": 50.0}, + "balanced_conc16.ttft_p95_ms": {"kind": "max_ms", "value": 60000}, + "balanced_conc16.tpot_p95_ms": {"kind": "max_ms", "value": 5000} +} +``` + +Walkthrough: + +- Flat namespace. Keys are `.` where `` matches the parametrize id or a synthetic name like `smoke`. +- Five predicate kinds: `min`, `max_ms`, `within`, `min_tok_s`, `min_ratio`. Each entry is `{kind, value, tolerance?}`. +- Missing entries are logged by `evaluate_all` but do not gate the run — adding new metrics stays cheap. + +### Anti-patterns to reject in review + +- A `config.json` key whose value is a pass/fail threshold (move it). +- A `threshold.json` entry that is actually a config flag (move it). +- A threshold that branches on hardware. Split into separate variant directories per `gpu_arch`. +- Placeholder substitution across files (a threshold value pulled from a config block). They have different lifecycles; do not couple them. +- A "smoke" threshold of `min: 1` standing in for "did the thing start." Liveness belongs in `assert` inside the test. +- Globbing multiple models into one config with `models: [...]`. Drops `cvs list` granularity. One model per variant directory. + +## 8. Porting checklist (suite-agnostic) + +1. **Inventory the source wrapper.** List every fixture, every test function, every key read from the config JSON, and every reach into `docker_lib`/`parallel_ssh_lib`/`globals`. Note which keys gate behavior vs gate pass/fail. +2. **Identify framework verbs.** Pull out the framework-specific calls in the legacy lib (`cvs/lib/inference/.py`, `cvs/lib/_training_lib.py`). These become the new Job's public surface. +3. **Classify each config key.** Each key is "what we run" (config), "did it pass" (threshold), or "cluster scaffolding" (already in the cluster file). If undecided, default to config; thresholds are only for numeric measurements with predicates. +4. **Lay out variant directories.** One directory per `(model, purpose)` or `(model, mode, purpose)` under `cvs/input/dtni//__/`. Full model IDs, no abbreviations. Single-vs-distributed is a *mode*, not a separate wrapper. +5. **Write the Job.** Standalone class under `cvs/lib//_orch.py`. Constructor takes the typed config and an `orch`. Do not inherit `InferenceBaseJob`. Do not import `globals`. +6. **Write the wrapper + conftest** following the skeleton in §4. One wrapper per suite. +7. **Verify against the pre-port baseline.** Run the old wrapper and the new one on the same hardware with the same model and confirm metrics are within noise. + +## 9. Verification template + +Every port PR should include: + +- `pytest --collect-only` on the new wrapper, expected count matches the variant × sweep grid. +- `cvs list ` showing the variant rows (run with `--config_file=dummy` if needed, mirroring `list_plugin.py`). +- An end-to-end hardware run that produces the HTML report and exits 0. +- Container lifecycle observation: `docker ps` before, during, and after the run, confirming the container is created by `orch.setup_containers()` and removed by `orch.teardown_containers()` — no stale containers remain. +- A negative test: deliberately tighten one threshold so it fails; confirm the HTML report flags the right cell and the exit code is non-zero. +- A diff against the pre-port baseline metrics, attached to the PR. + +## 10. Out of scope for DTNI v-PoC + +- Refactoring or fixing `cvs/lib/inference/base.py` (`InferenceBaseJob`) or `cvs/lib/megatron_training_lib.py`. Older suites still depend on them; leave them alone. +- `model.remote=1` HuggingFace download path. Only `remote=0` (pre-staged on shared FS) is wired up. +- Accuracy variants. Only `_perf` is in scope; `_accuracy` directories may exist but their evaluation pipeline is a follow-up. +- Sweep rework. The current `pytest_generate_tests`-style parametrization is preserved; a declarative sweep grammar is a follow-up. +- `globals.error_list` deletion. The new suites stop using it; the legacy import stays until the last suite ports. diff --git a/requirements.txt b/requirements.txt index 83663c028..dc0114e06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,6 @@ orjson openpyxl netmiko jinja2 + +# Headless plotting for training loss-curve PNGs (Agg backend; see cvs/lib/training/jaxmaxtext/utils/loss_curve.py) +matplotlib