diff --git a/nodescraper/cli/__init__.py b/nodescraper/cli/__init__.py index 44e9b02b..f0bea74e 100644 --- a/nodescraper/cli/__init__.py +++ b/nodescraper/cli/__init__.py @@ -26,7 +26,7 @@ from .cli import get_cli_top_level_subcommands from .cli import main as cli_entry -from .embed import CLI_TOP_LEVEL_SUBCOMMANDS, run_cli_return_code, run_main_return_code +from .embed import run_cli_return_code, run_main_return_code from .invocation import ( PluginRunInvocation, get_plugin_run_invocation, @@ -35,7 +35,6 @@ ) __all__ = [ - "CLI_TOP_LEVEL_SUBCOMMANDS", "cli_entry", "get_cli_top_level_subcommands", "run_cli_return_code", diff --git a/nodescraper/cli/cli.py b/nodescraper/cli/cli.py index 30dc8792..606dd5e8 100644 --- a/nodescraper/cli/cli.py +++ b/nodescraper/cli/cli.py @@ -37,7 +37,11 @@ import nodescraper from nodescraper.cli.compare_runs import run_compare_runs -from nodescraper.cli.constants import DEFAULT_CONFIG, META_VAR_MAP +from nodescraper.cli.constants import ( + DEFAULT_CONFIG, + KEYBOARD_INTERRUPT_EXIT_CODE, + META_VAR_MAP, +) from nodescraper.cli.dynamicparserbuilder import DynamicParserBuilder from nodescraper.cli.helper import ( dump_results_to_csv, @@ -459,6 +463,14 @@ def setup_logger( return logger +def _handle_keyboard_interrupt(logger: Optional[logging.Logger] = None) -> None: + if logger is not None: + logger.info("Received Ctrl+C. Shutting down...") + else: + sys.stderr.write("Interrupted.\n") + sys.exit(KEYBOARD_INTERRUPT_EXIT_CODE) + + def main( arg_input: Optional[list[str]] = None, *, @@ -474,169 +486,175 @@ def main( plugin_run_result_hooks: Optional callbacks invoked with each plugin's :class:`PluginResult` after ``run()`` completes (used by embedded hosts such as error-scraper). """ - if arg_input is None: - arg_input = sys.argv[1:] - - plugin_reg = PluginRegistry() - config_reg = _default_config_registry(plugin_reg) - parser, plugin_subparser_map = build_parser(plugin_reg, config_reg) - + logger: Optional[logging.Logger] = None try: - top_level_args, plugin_arg_map, invalid_plugins = process_args( - arg_input, list(plugin_subparser_map.keys()) - ) + if arg_input is None: + arg_input = sys.argv[1:] - parsed_args = parser.parse_args(top_level_args) - apply_host_cli_args_to_parsed_args(parsed_args, host_cli_args) - merge_plugin_connection_config_from_host_ns(parsed_args, host_cli_args) - system_info = get_system_info(parsed_args) - sname = system_info.name.lower().replace("-", "_").replace(".", "_") - timestamp = datetime.datetime.now().strftime("%Y_%m_%d-%I_%M_%S_%p") - - if parsed_args.log_path: - log_path = os.path.join( - parsed_args.log_path, - f"scraper_logs_{sname}_{timestamp}", - ) - os.makedirs(log_path) - else: - log_path = None + plugin_reg = PluginRegistry() + config_reg = _default_config_registry(plugin_reg) + parser, plugin_subparser_map = build_parser(plugin_reg, config_reg) - if parsed_args.no_console_log and not log_path: - base_dir = parsed_args.log_path if parsed_args.log_path else "." - log_path = os.path.join(base_dir, f"scraper_logs_{sname}_{timestamp}") - os.makedirs(log_path, exist_ok=True) - - logger = setup_logger( - parsed_args.log_level, - log_path, - console=not parsed_args.no_console_log, - ) - if log_path: - logger.info("Log path: %s", log_path) - - # Log warning if invalid plugin names were provided - if invalid_plugins: - logger.warning( - "Invalid plugin name(s) ignored: %s. Use 'describe plugin' to list available plugins.", - ", ".join(invalid_plugins), + try: + top_level_args, plugin_arg_map, invalid_plugins = process_args( + arg_input, list(plugin_subparser_map.keys()) ) - if parsed_args.subcmd == "summary": - generate_summary( - parsed_args.search_path, - parsed_args.output_path, - logger, - artifact_dir=log_path, - ) - sys.exit(0) + parsed_args = parser.parse_args(top_level_args) + apply_host_cli_args_to_parsed_args(parsed_args, host_cli_args) + merge_plugin_connection_config_from_host_ns(parsed_args, host_cli_args) + system_info = get_system_info(parsed_args) + sname = system_info.name.lower().replace("-", "_").replace(".", "_") + timestamp = datetime.datetime.now().strftime("%Y_%m_%d-%I_%M_%S_%p") + + if parsed_args.log_path: + log_path = os.path.join( + parsed_args.log_path, + f"scraper_logs_{sname}_{timestamp}", + ) + os.makedirs(log_path) + else: + log_path = None - if parsed_args.subcmd == "describe": - parse_describe(parsed_args, plugin_reg, config_reg, logger) - - if parsed_args.subcmd == "compare-runs": - run_compare_runs( - parsed_args.path1, - parsed_args.path2, - plugin_reg, - logger, - skip_plugins=getattr(parsed_args, "skip_plugins", None) or [], - include_plugins=getattr(parsed_args, "include_plugins", None), - truncate_message=not getattr(parsed_args, "dont_truncate", False), - artifact_dir=log_path, - ) - sys.exit(0) + if parsed_args.no_console_log and not log_path: + base_dir = parsed_args.log_path if parsed_args.log_path else "." + log_path = os.path.join(base_dir, f"scraper_logs_{sname}_{timestamp}") + os.makedirs(log_path, exist_ok=True) - if parsed_args.subcmd == "show-redfish-oem-allowable": - if not parsed_args.connection_config: - parser.error("show-redfish-oem-allowable requires --connection-config") - raw = parsed_args.connection_config.get("RedfishConnectionManager") - if not raw: - logger.error("Connection config must contain RedfishConnectionManager") - sys.exit(1) - params = RedfishConnectionParams.model_validate(raw) - password = params.password.get_secret_value() if params.password else None - base_url = f"{'https' if params.use_https else 'http'}://{params.host}" + ( - f":{params.port}" if params.port else "" - ) - conn = RedfishConnection( - base_url=base_url, - username=params.username, - password=password, - timeout=params.timeout_seconds, - use_session_auth=params.use_session_auth, - verify_ssl=params.verify_ssl, - api_root=params.api_root, + logger = setup_logger( + parsed_args.log_level, + log_path, + console=not parsed_args.no_console_log, ) - try: - conn._ensure_session() - allowable = get_oem_diagnostic_allowable_values(conn, parsed_args.log_service_path) - if allowable is None: - logger.warning( - "Could not read OEMDiagnosticDataType@Redfish.AllowableValues from LogService" - ) - sys.exit(1) - logger.info("%s", json.dumps(allowable, indent=2)) - finally: - conn.close() - sys.exit(0) + if log_path: + logger.info("Log path: %s", log_path) + + # Log warning if invalid plugin names were provided + if invalid_plugins: + logger.warning( + "Invalid plugin name(s) ignored: %s. Use 'describe plugin' to list available plugins.", + ", ".join(invalid_plugins), + ) + + if parsed_args.subcmd == "summary": + generate_summary( + parsed_args.search_path, + parsed_args.output_path, + logger, + artifact_dir=log_path, + ) + sys.exit(0) - if parsed_args.subcmd == "gen-plugin-config": + if parsed_args.subcmd == "describe": + parse_describe(parsed_args, plugin_reg, config_reg, logger) + + if parsed_args.subcmd == "compare-runs": + run_compare_runs( + parsed_args.path1, + parsed_args.path2, + plugin_reg, + logger, + skip_plugins=getattr(parsed_args, "skip_plugins", None) or [], + include_plugins=getattr(parsed_args, "include_plugins", None), + truncate_message=not getattr(parsed_args, "dont_truncate", False), + artifact_dir=log_path, + ) + sys.exit(0) - if parsed_args.reference_config_from_logs: - ref_config = generate_reference_config_from_logs( - parsed_args.reference_config_from_logs, plugin_reg, logger + if parsed_args.subcmd == "show-redfish-oem-allowable": + if not parsed_args.connection_config: + parser.error("show-redfish-oem-allowable requires --connection-config") + raw = parsed_args.connection_config.get("RedfishConnectionManager") + if not raw: + logger.error("Connection config must contain RedfishConnectionManager") + sys.exit(1) + params = RedfishConnectionParams.model_validate(raw) + password = params.password.get_secret_value() if params.password else None + base_url = f"{'https' if params.use_https else 'http'}://{params.host}" + ( + f":{params.port}" if params.port else "" + ) + conn = RedfishConnection( + base_url=base_url, + username=params.username, + password=password, + timeout=params.timeout_seconds, + use_session_auth=params.use_session_auth, + verify_ssl=params.verify_ssl, + api_root=params.api_root, ) - out_dir = log_path if log_path else parsed_args.output_path - path = os.path.join(out_dir, "reference_config.json") try: - with open(path, "w") as f: - json.dump( - ref_config.model_dump(mode="json", exclude_none=True), - f, - indent=2, + conn._ensure_session() + allowable = get_oem_diagnostic_allowable_values( + conn, parsed_args.log_service_path + ) + if allowable is None: + logger.warning( + "Could not read OEMDiagnosticDataType@Redfish.AllowableValues from LogService" ) - logger.info("Reference config written to: %s", path) - except Exception as exp: - logger.error(exp) + sys.exit(1) + logger.info("%s", json.dumps(allowable, indent=2)) + finally: + conn.close() sys.exit(0) - parse_gen_plugin_config( - parsed_args, plugin_reg, config_reg, logger, artifact_dir=log_path - ) + if parsed_args.subcmd == "gen-plugin-config": - parsed_plugin_args = {} - for plugin, plugin_args in plugin_arg_map.items(): - try: - parsed_plugin_args[plugin] = plugin_subparser_map[plugin][0].parse_args(plugin_args) - except Exception as e: - logger.error("%s exception parsing args for plugin: %s", str(e), plugin) + if parsed_args.reference_config_from_logs: + ref_config = generate_reference_config_from_logs( + parsed_args.reference_config_from_logs, plugin_reg, logger + ) + out_dir = log_path if log_path else parsed_args.output_path + path = os.path.join(out_dir, "reference_config.json") + try: + with open(path, "w") as f: + json.dump( + ref_config.model_dump(mode="json", exclude_none=True), + f, + indent=2, + ) + logger.info("Reference config written to: %s", path) + except Exception as exp: + logger.error(exp) + sys.exit(0) + + parse_gen_plugin_config( + parsed_args, plugin_reg, config_reg, logger, artifact_dir=log_path + ) - if not parsed_plugin_args and not parsed_args.plugin_configs: - logger.info( - "No plugins config args specified, running default config: %s", DEFAULT_CONFIG + parsed_plugin_args = {} + for plugin, plugin_args in plugin_arg_map.items(): + try: + parsed_plugin_args[plugin] = plugin_subparser_map[plugin][0].parse_args( + plugin_args + ) + except Exception as e: + logger.error("%s exception parsing args for plugin: %s", str(e), plugin) + + if not parsed_plugin_args and not parsed_args.plugin_configs: + logger.info( + "No plugins config args specified, running default config: %s", + DEFAULT_CONFIG, + ) + plugin_configs = [DEFAULT_CONFIG] + else: + plugin_configs = parsed_args.plugin_configs or [] + + plugin_config_inst_list = get_plugin_configs( + plugin_config_input=plugin_configs, + system_interaction_level=parsed_args.sys_interaction_level, + built_in_configs=config_reg.configs, + parsed_plugin_args=parsed_plugin_args, + plugin_subparser_map=plugin_subparser_map, ) - plugin_configs = [DEFAULT_CONFIG] - else: - plugin_configs = parsed_args.plugin_configs or [] - - plugin_config_inst_list = get_plugin_configs( - plugin_config_input=plugin_configs, - system_interaction_level=parsed_args.sys_interaction_level, - built_in_configs=config_reg.configs, - parsed_plugin_args=parsed_plugin_args, - plugin_subparser_map=plugin_subparser_map, - ) - if parsed_args.skip_sudo: - plugin_config_inst_list[-1].global_args.setdefault("collection_args", {})[ - "skip_sudo" - ] = True + if parsed_args.skip_sudo: + plugin_config_inst_list[-1].global_args.setdefault("collection_args", {})[ + "skip_sudo" + ] = True - except Exception as e: - parser.error(str(e)) + except Exception as e: + parser.error(str(e)) - try: results = run_plugin_queue_with_invocation( plugin_reg=plugin_reg, parsed_args=parsed_args, @@ -683,8 +701,7 @@ def main( else: sys.exit(0) except KeyboardInterrupt: - logger.info("Received Ctrl+C. Shutting down...") - sys.exit(130) + _handle_keyboard_interrupt(logger) if __name__ == "__main__": diff --git a/nodescraper/cli/constants.py b/nodescraper/cli/constants.py index 5b8b8922..2637362e 100644 --- a/nodescraper/cli/constants.py +++ b/nodescraper/cli/constants.py @@ -25,3 +25,4 @@ ############################################################################### META_VAR_MAP = {int: "INT", bool: "BOOL", dict: "JSON_STRING", float: "FLOAT", str: "STRING"} DEFAULT_CONFIG = "NodeStatus" +KEYBOARD_INTERRUPT_EXIT_CODE = 130 diff --git a/nodescraper/cli/embed.py b/nodescraper/cli/embed.py index b1e91c37..37f6498f 100644 --- a/nodescraper/cli/embed.py +++ b/nodescraper/cli/embed.py @@ -30,14 +30,9 @@ from collections.abc import Callable, Sequence from typing import Optional -from nodescraper.cli.cli import get_cli_top_level_subcommands from nodescraper.models.pluginresult import PluginResult -CLI_TOP_LEVEL_SUBCOMMANDS = get_cli_top_level_subcommands() - __all__ = [ - "CLI_TOP_LEVEL_SUBCOMMANDS", - "get_cli_top_level_subcommands", "run_cli_return_code", "run_main_return_code", ] diff --git a/nodescraper/connection/redfish/redfish_connection.py b/nodescraper/connection/redfish/redfish_connection.py index 882acbd5..eabee716 100644 --- a/nodescraper/connection/redfish/redfish_connection.py +++ b/nodescraper/connection/redfish/redfish_connection.py @@ -26,8 +26,9 @@ from __future__ import annotations import json -from typing import Any, ClassVar, Optional, Union -from urllib.parse import urljoin +import socket +from typing import Any, Callable, ClassVar, Optional, TypeVar, Union +from urllib.parse import urljoin, urlparse import requests import urllib3 # type: ignore[import-untyped] @@ -40,6 +41,8 @@ DEFAULT_REDFISH_API_ROOT = "redfish/v1" +_T = TypeVar("_T") + class RedfishGetResult(BaseModel): """Artifact for the result of a Redfish GET request. @@ -100,12 +103,49 @@ def __init__( self._session_token: Optional[str] = None self._session_uri: Optional[str] = None # For logout DELETE + def _host_label(self) -> str: + return urlparse(self.base_url).hostname or self.base_url + + @staticmethod + def _is_name_resolution_error(exc: BaseException) -> bool: + current: Optional[BaseException] = exc + seen: set[int] = set() + while current is not None and id(current) not in seen: + if isinstance(current, socket.gaierror): + return True + if type(current).__name__ == "NameResolutionError": + return True + seen.add(id(current)) + current = current.__cause__ or current.__context__ + message = str(exc) + return "Failed to resolve" in message or "No address associated with hostname" in message + + def _transport_error_message(self, exc: Exception) -> str: + host = self._host_label() + if isinstance(exc, socket.gaierror) or self._is_name_resolution_error(exc): + return f"Redfish hostname could not be resolved: {host}" + if isinstance(exc, requests.exceptions.Timeout): + return f"Redfish connection timed out: {host}" + if isinstance(exc, requests.exceptions.ConnectionError): + return f"Redfish connection failed: {host}" + return f"Redfish connection failed: {exc}" + + def _execute_request(self, request_fn: Callable[[], _T]) -> _T: + try: + return request_fn() + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exc: + raise RedfishConnectionError(self._transport_error_message(exc)) from exc + except socket.gaierror as exc: + raise RedfishConnectionError(self._transport_error_message(exc)) from exc + def _ensure_session(self) -> requests.Session: if self._session is None: if not self.verify_ssl: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) self._session = requests.Session() self._session.verify = self.verify_ssl + if not self.verify_ssl: + self._session.trust_env = False self._session.headers["Content-Type"] = "application/json" self._session.headers["Accept"] = "application/json" if self.use_session_auth and self.password: @@ -116,13 +156,16 @@ def _ensure_session(self) -> requests.Session: def _login_session(self) -> None: """Create a Redfish session and set X-Auth-Token.""" - assert self._session is not None + session = self._session + assert session is not None sess_url = urljoin(self.base_url + "/", f"{self.api_root}/SessionService/Sessions") payload = {"UserName": self.username, "Password": self.password} - resp = self._session.post( - sess_url, - json=payload, - timeout=self.timeout, + resp = self._execute_request( + lambda: session.post( + sess_url, + json=payload, + timeout=self.timeout, + ) ) if not resp.ok: raise RedfishConnectionError( @@ -137,9 +180,9 @@ def _login_session(self) -> None: else urljoin(self.base_url + "/", location.lstrip("/")) ) if self._session_token: - self._session.headers["X-Auth-Token"] = self._session_token + session.headers["X-Auth-Token"] = self._session_token else: - self._session.auth = HTTPBasicAuth(self.username, self.password) + session.auth = HTTPBasicAuth(self.username, self.password) def get(self, path: RedfishPath) -> dict[str, Any]: """GET a Redfish path and return the JSON body. path must be a RedfishPath.""" @@ -157,7 +200,7 @@ def get_response(self, path: Union[str, "RedfishPath"]) -> Response: path = str(path) session = self._ensure_session() url = path if path.startswith("http") else urljoin(self.base_url + "/", path.lstrip("/")) - return session.get(url, timeout=self.timeout) + return self._execute_request(lambda: session.get(url, timeout=self.timeout)) def post( self, path: Union[str, "RedfishPath"], json: Optional[dict[str, Any]] = None @@ -166,7 +209,9 @@ def post( path = str(path) session = self._ensure_session() url = path if path.startswith("http") else urljoin(self.base_url + "/", path.lstrip("/")) - return session.post(url, json=json or {}, timeout=self.timeout) + return self._execute_request( + lambda: session.post(url, json=json or {}, timeout=self.timeout) + ) def run_get(self, path: Union[str, RedfishPath]) -> RedfishGetResult: """Run a Redfish GET request and return a result object. path may be a string or RedfishPath.""" diff --git a/nodescraper/connection/redfish/redfish_manager.py b/nodescraper/connection/redfish/redfish_manager.py index 4413ee86..bc5b37d2 100644 --- a/nodescraper/connection/redfish/redfish_manager.py +++ b/nodescraper/connection/redfish/redfish_manager.py @@ -32,7 +32,6 @@ from nodescraper.interfaces.connectionmanager import ConnectionManager from nodescraper.interfaces.taskresulthook import TaskResultHook from nodescraper.models import SystemInfo, TaskResult -from nodescraper.utils import get_exception_traceback from .redfish_connection import RedfishConnection, RedfishConnectionError from .redfish_params import RedfishConnectionParams @@ -116,8 +115,7 @@ def connect(self) -> TaskResult: except RedfishConnectionError as exc: self._log_event( category=EventCategory.RUNTIME, - description=f"Redfish connection error: {exc}", - data=get_exception_traceback(exc) if exc.response is None else None, + description=str(exc), priority=EventPriority.CRITICAL, console_log=True, ) @@ -127,7 +125,6 @@ def connect(self) -> TaskResult: self._log_event( category=EventCategory.RUNTIME, description=f"Redfish connection failed: {exc}", - data=get_exception_traceback(exc), priority=EventPriority.CRITICAL, console_log=True, ) diff --git a/nodescraper/interfaces/dataanalyzertask.py b/nodescraper/interfaces/dataanalyzertask.py index fd6cc284..c91fb251 100644 --- a/nodescraper/interfaces/dataanalyzertask.py +++ b/nodescraper/interfaces/dataanalyzertask.py @@ -55,7 +55,10 @@ def wrapper( analyzer._log_event( category=EventCategory.RUNTIME, description="Analyzer passed invalid data", - data={"data_type": type(data), "expected": analyzer.DATA_MODEL.__name__}, + data={ + "data_type": type(data), + "expected": analyzer.DATA_MODEL.__name__, + }, priority=EventPriority.CRITICAL, console_log=True, ) @@ -118,7 +121,7 @@ def __init_subclass__(cls, **kwargs: dict[str, Any]) -> None: if not inspect.isabstract(cls) and cls.DATA_MODEL is None: raise TypeError(f"No data model set for {cls.__name__}") - if hasattr(cls, "analyze_data"): + if "analyze_data" in vars(cls): setattr(cls, "analyze_data", analyze_decorator(cls.analyze_data)) # noqa @abc.abstractmethod diff --git a/nodescraper/plugins/serviceability/__init__.py b/nodescraper/plugins/serviceability/__init__.py index c5e9f857..c34007e3 100644 --- a/nodescraper/plugins/serviceability/__init__.py +++ b/nodescraper/plugins/serviceability/__init__.py @@ -7,7 +7,7 @@ # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# to use, copy, modify, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # @@ -24,7 +24,19 @@ # ############################################################################### from .afid_events import build_afid_events_from_data +from .afid_sag_lookup import ( + format_collected_afid_fru_summary_lines, + group_afid_events_by_fru, + load_afid_sag_data, +) +from .afid_sag_paths import ( + default_afid_sag_path, + resolve_configured_afid_sag_path, + validate_afid_sag_path, +) +from .analysis_window import ServiceabilityWindowResult, analyze_serviceability_window from .analyzer_args import ServiceabilityAnalyzerArgs +from .event_log_utils import event_timestamp, filter_event_log_members from .mi3xx import ( MI3XXAnalyzer, MI3XXCollector, @@ -35,18 +47,40 @@ ServiceabilityPluginMI3XX, build_mi3xx_reporting_version_fields, ) +from .mi4xx import ( + MI4XXAnalyzer, + MI4XXCollector, + MI4XXCollectorArgs, + Mi4xxServiceabilityAnalyzerArgs, + Mi4xxServiceabilityPlugin, +) from .se_adapter import ( format_serviceability_solution_lines, + serviceability_block_from_entry_point_hub, serviceability_block_from_service_result, ) -from .se_models import AfidEvent, ServiceabilityBlock, ServiceabilitySolution -from .se_runner import SeRunError, run_service_hub +from .se_models import ( + AfidEvent, + HubTriageResult, + ServiceabilityBlock, + ServiceabilitySolution, +) +from .se_runner import ( + HUB_ENTRY_POINT_GROUP, + HubRunError, + SeRunError, + list_hub_entry_point_names, + load_hub_from_entry_point, + run_entry_point_hub, + run_service_hub, +) from .serviceability_collector import ServiceabilityCollectorBase from .serviceability_data import ( DeviceInfo, ServiceabilityDataModel, ServiceabilityResult, ) +from .serviceability_hub_analyzer import ServiceabilityHubAnalyzer from .serviceability_plugin_base import ServiceabilityPluginBase from .time_utils import ( TimeOperator, @@ -60,30 +94,53 @@ __all__ = [ "AfidEvent", "DeviceInfo", + "HUB_ENTRY_POINT_GROUP", "MI3XXAnalyzer", "MI3XXCollector", "MI3XXCollectorArgs", "MI3XXDataModel", "MI3XXDeviceInfo", "MI3XXResult", + "MI4XXAnalyzer", + "MI4XXCollector", + "MI4XXCollectorArgs", + "Mi4xxServiceabilityAnalyzerArgs", + "Mi4xxServiceabilityPlugin", + "HubRunError", "SeRunError", + "HubTriageResult", "ServiceabilityAnalyzerArgs", "ServiceabilityBlock", "ServiceabilityCollectorBase", "ServiceabilityDataModel", + "ServiceabilityHubAnalyzer", "ServiceabilityPluginBase", "ServiceabilityPluginMI3XX", "ServiceabilityResult", "ServiceabilitySolution", + "ServiceabilityWindowResult", "TimeOperator", + "analyze_serviceability_window", "build_afid_events_from_data", "build_mi3xx_reporting_version_fields", "compare_iso_datetime", + "default_afid_sag_path", + "event_timestamp", + "filter_event_log_members", + "format_collected_afid_fru_summary_lines", "format_serviceability_solution_lines", + "group_afid_events_by_fru", "is_valid_iso_datetime", + "list_hub_entry_point_names", + "load_afid_sag_data", + "load_hub_from_entry_point", "normalize_se_timestamp", "parse_iso_datetime", + "resolve_configured_afid_sag_path", + "run_entry_point_hub", "run_service_hub", + "serviceability_block_from_entry_point_hub", "serviceability_block_from_service_result", "satisfies_time_check", + "validate_afid_sag_path", ] diff --git a/nodescraper/plugins/serviceability/afid_events.py b/nodescraper/plugins/serviceability/afid_events.py index a84af503..645d9177 100644 --- a/nodescraper/plugins/serviceability/afid_events.py +++ b/nodescraper/plugins/serviceability/afid_events.py @@ -36,7 +36,7 @@ def build_afid_events_from_data(data: ServiceabilityDataModel) -> list[AfidEvent]: - """Build SE input events from collected Redfish and CPER fields.""" + """Build hub input events from collected Redfish and CPER fields.""" events: list[AfidEvent] = [] seen: set[tuple[int, str, str]] = set() @@ -63,6 +63,11 @@ def build_afid_events_from_data(data: ServiceabilityDataModel) -> list[AfidEvent return events +def rf_member_is_afid_parseable(member: Any) -> bool: + """Return True when a Redfish log member has AFID, serviceable unit, and timestamp.""" + return _afid_event_from_rf_member(member) is not None + + def _afid_event_from_rf_member(member: Any) -> Optional[AfidEvent]: if not isinstance(member, dict): return None @@ -106,18 +111,31 @@ def _extract_afid(payload: dict[str, Any]) -> Optional[int]: return None +def _afid_from_field_identifiers(identifiers: Any) -> Optional[int]: + if not isinstance(identifiers, list): + return None + for item in identifiers: + if not isinstance(item, dict): + continue + for key in _AFID_KEYS: + if key in item and item[key] is not None: + return int(item[key]) + return None + + def _extract_afid_from_oem_fragment(vendor_payload: Any) -> Optional[int]: """Resolve AFID from one ``Oem`` property value (dict or list of dicts, e.g. ``AMDFieldIdentifiers``).""" if isinstance(vendor_payload, dict): for key in _AFID_KEYS: if key in vendor_payload and vendor_payload[key] is not None: return int(vendor_payload[key]) + found = _afid_from_field_identifiers(vendor_payload.get("AMDFieldIdentifiers")) + if found is not None: + return found elif isinstance(vendor_payload, list): - for item in vendor_payload: - if isinstance(item, dict): - for key in _AFID_KEYS: - if key in item and item[key] is not None: - return int(item[key]) + found = _afid_from_field_identifiers(vendor_payload) + if found is not None: + return found return None @@ -163,6 +181,16 @@ def _extract_serviceable_unit(payload: dict[str, Any]) -> Optional[str]: ) if unit is not None and str(unit).strip(): return str(unit).strip() + identifiers = vendor_payload.get("AMDFieldIdentifiers") + if isinstance(identifiers, list): + for item in identifiers: + if not isinstance(item, dict): + continue + su = item.get("ServiceableUnits") or item.get("serviceable_units") + if isinstance(su, list) and su: + u = _origin_dict_to_unit(su[0]) + if u: + return u elif isinstance(vendor_payload, list): for item in vendor_payload: if not isinstance(item, dict): diff --git a/nodescraper/plugins/serviceability/afid_fru_csv.py b/nodescraper/plugins/serviceability/afid_fru_csv.py new file mode 100644 index 00000000..22111698 --- /dev/null +++ b/nodescraper/plugins/serviceability/afid_fru_csv.py @@ -0,0 +1,362 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +import csv +import logging +import os +from typing import Any, Optional + +from .afid_events import build_afid_events_from_data +from .afid_sag_lookup import ( + afid_entry_from_sag, + afid_fru_from_sag, + afid_summary_from_sag, + group_afid_events_by_fru, + load_afid_sag_data, + normalize_fru_name, + sag_fru_display_names, + service_action_label_from_sag, +) +from .se_models import HubTriageResult, ServiceabilityBlock +from .serviceability_data import DeviceInfo, ServiceabilityDataModel + +AFID_FRU_SUMMARY_CSV = "afid_fru_summary.csv" + +_SERIAL_KEYS: tuple[str, ...] = ( + "SerialNumber", + "serial_number", + "UbbSerial", + "ubb_serial", + "ProductSerialNumber", + "product_serial_number", +) +_PART_KEYS: tuple[str, ...] = ( + "PartNumber", + "part_number", + "ProductPartNumber", + "product_part_number", + "BoardPartNumber", + "board_part_number", +) +_NAME_KEYS: tuple[str, ...] = ("Name", "name", "Model", "model") +_VERSION_KEYS: tuple[str, ...] = ("Version", "version", "FirmwareVersion", "firmware_version") + +AFID_FRU_CSV_COLUMNS: tuple[str, ...] = ( + "fru", + "fru_text", + "serial_number", + "part_number", + "unit_name", + "unit_version", + "afid", + "fault", + "fault_severity", + "unit", + "event_time", + "priority", + "service_action_num", + "service_action_title", + "sa_severity", + "tier", + "event_count", +) + + +def _empty_row() -> dict[str, str]: + return {column: "" for column in AFID_FRU_CSV_COLUMNS} + + +def _text_value(value: Any) -> str: + if value is None: + return "" + return str(value).strip() + + +def _first_nonempty(mapping: dict[str, Any], keys: tuple[str, ...]) -> str: + for key in keys: + text = _text_value(mapping.get(key)) + if text: + return text + return "" + + +def _identity_from_mapping(mapping: dict[str, Any]) -> dict[str, str]: + return { + "unit_name": _first_nonempty(mapping, _NAME_KEYS), + "serial_number": _first_nonempty(mapping, _SERIAL_KEYS), + "part_number": _first_nonempty(mapping, _PART_KEYS), + "unit_version": _first_nonempty(mapping, _VERSION_KEYS), + } + + +def _normalize_unit_key(unit: str) -> str: + return str(unit).strip().upper().replace("-", "_") + + +def _assembly_by_unit(data: ServiceabilityDataModel) -> dict[str, DeviceInfo]: + out: dict[str, DeviceInfo] = {} + for key, info in (data.assembly_info or {}).items(): + norm = _normalize_unit_key(key) + out[norm] = info + parts = norm.split("_") + if len(parts) >= 2: + out["_".join(parts[-2:])] = info + return out + + +def _match_assembly(unit: str, assembly_by_unit: dict[str, DeviceInfo]) -> Optional[DeviceInfo]: + norm = _normalize_unit_key(unit) + if norm in assembly_by_unit: + return assembly_by_unit[norm] + for key, info in assembly_by_unit.items(): + if norm.endswith(key) or key.endswith(norm): + return info + return None + + +def _identity_from_rf_member(member: dict[str, Any]) -> dict[str, str]: + identity = _identity_from_mapping(member) + oem = member.get("Oem") + if not isinstance(oem, dict): + return identity + for fragment in oem.values(): + if not isinstance(fragment, dict): + continue + for field, value in _identity_from_mapping(fragment).items(): + if value and not identity[field]: + identity[field] = value + err_arr = fragment.get("ErrDataArr") + if not isinstance(err_arr, list): + continue + for entry in err_arr: + if not isinstance(entry, dict): + continue + for nested in (entry.get("MetaData"), entry.get("DecodedData")): + if not isinstance(nested, dict): + continue + for field, value in _identity_from_mapping(nested).items(): + if value and not identity[field]: + identity[field] = value + return identity + + +def _format_fru_text(identity: dict[str, str]) -> str: + bits: list[str] = [] + if identity["unit_name"]: + bits.append(identity["unit_name"]) + if identity["serial_number"]: + bits.append(f"SN: {identity['serial_number']}") + if identity["part_number"]: + bits.append(f"PN: {identity['part_number']}") + if identity["unit_version"]: + bits.append(f"Ver: {identity['unit_version']}") + return "; ".join(bits) + + +def _unit_identity( + unit: str, + *, + assembly_by_unit: dict[str, DeviceInfo], + rf_member: Optional[dict[str, Any]], +) -> dict[str, str]: + identity = { + "unit_name": "", + "serial_number": "", + "part_number": "", + "unit_version": "", + } + device = _match_assembly(unit, assembly_by_unit) + if device is not None: + identity["unit_name"] = _text_value(device.name) + identity["serial_number"] = _text_value(device.serial_number) + identity["part_number"] = _text_value(device.part_number) + identity["unit_version"] = _text_value(device.version) + if rf_member is not None: + from_rf = _identity_from_rf_member(rf_member) + for field in identity: + if not identity[field] and from_rf[field]: + identity[field] = from_rf[field] + return identity + + +def _rf_members_by_afid_unit( + data: ServiceabilityDataModel, +) -> dict[tuple[int, str], dict[str, Any]]: + from .afid_events import _afid_event_from_rf_member + + out: dict[tuple[int, str], dict[str, Any]] = {} + for member in data.rf_events: + if not isinstance(member, dict): + continue + parsed = _afid_event_from_rf_member(member) + if parsed is None: + continue + key = (parsed.afid, parsed.serviceable_unit) + if key not in out: + out[key] = member + return out + + +def _apply_unit_identity(row: dict[str, str], identity: dict[str, str]) -> None: + row["fru_text"] = _format_fru_text(identity) + row["serial_number"] = identity["serial_number"] + row["part_number"] = identity["part_number"] + row["unit_name"] = identity["unit_name"] + row["unit_version"] = identity["unit_version"] + + +def _triage_lookup( + block: Optional[ServiceabilityBlock], +) -> dict[tuple[int, str], HubTriageResult]: + if block is None: + return {} + out: dict[tuple[int, str], HubTriageResult] = {} + for row in block.hub_triage_results: + out[(row.afid, row.location)] = row + return out + + +def _row_from_event( + *, + fru: str, + event, + sag: Optional[dict[str, Any]], + triage: Optional[HubTriageResult], + unit_identity: dict[str, str], +) -> dict[str, str]: + row = _empty_row() + entry = afid_entry_from_sag(event.afid, sag) + row["fru"] = fru + _apply_unit_identity(row, unit_identity) + row["afid"] = str(event.afid) + row["fault"] = afid_summary_from_sag(event.afid, sag) or "" + row["fault_severity"] = str(entry.get("error_severity") or "") if entry else "" + row["unit"] = event.serviceable_unit + row["event_time"] = event.time + row["priority"] = str(entry.get("priority") or "") if entry else "" + if triage is not None: + row["service_action_num"] = str(triage.service_action_num) + row["service_action_title"] = triage.service_action_title or "" + row["sa_severity"] = str(triage.sa_severity or "") + row["tier"] = triage.tier_label or "" + row["event_count"] = str(triage.count) + if triage.afid_summary: + row["fault"] = triage.afid_summary + if triage.priority is not None: + row["priority"] = str(triage.priority) + elif entry: + san = entry.get("service_action_num") + if san is not None: + row["service_action_num"] = str(san) + row["service_action_title"] = service_action_label_from_sag(int(san), sag) or str( + entry.get("service_action") or "" + ) + row["event_count"] = "1" + else: + row["event_count"] = "1" + return row + + +def build_afid_fru_csv_rows( + data: ServiceabilityDataModel, + sag: Optional[dict[str, Any]], +) -> list[dict[str, str]]: + """Build CSV rows for AFID events grouped by FRU, including empty rows for SAG FRUs with no events.""" + events = data.afid_events or build_afid_events_from_data(data) + grouped = group_afid_events_by_fru(events, sag) + triage_by_key = _triage_lookup(data.serviceability) + assembly_by_unit = _assembly_by_unit(data) + rf_by_key = _rf_members_by_afid_unit(data) + rows: list[dict[str, str]] = [] + + seen_fru: set[str] = set() + for fru_key in sorted(grouped.keys()): + seen_fru.add(fru_key) + display_fru = afid_fru_from_sag(grouped[fru_key][0].afid, sag) or fru_key + for event in sorted( + grouped[fru_key], + key=lambda item: (item.afid, item.serviceable_unit, item.time), + ): + triage = triage_by_key.get((event.afid, event.serviceable_unit)) + unit_identity = _unit_identity( + event.serviceable_unit, + assembly_by_unit=assembly_by_unit, + rf_member=rf_by_key.get((event.afid, event.serviceable_unit)), + ) + rows.append( + _row_from_event( + fru=display_fru, + event=event, + sag=sag, + triage=triage, + unit_identity=unit_identity, + ) + ) + + for fru_name in sag_fru_display_names(sag): + if normalize_fru_name(fru_name) in seen_fru: + continue + row = _empty_row() + row["fru"] = fru_name + rows.append(row) + + return rows + + +def write_afid_fru_summary_csv( + data: ServiceabilityDataModel, + log_path: str, + *, + logger: Optional[logging.Logger] = None, + parent: Optional[str] = None, +) -> Optional[str]: + """Write afid_fru_summary.csv under log_path when AFID_SAG path is configured.""" + sag_path = data.afid_sag_path + if not sag_path or not str(sag_path).strip(): + return None + sag = load_afid_sag_data(sag_path) + if sag is None: + return None + + rows = build_afid_fru_csv_rows(data, sag) + os.makedirs(log_path, exist_ok=True) + csv_path = os.path.join(log_path, AFID_FRU_SUMMARY_CSV) + with open(csv_path, "w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=AFID_FRU_CSV_COLUMNS) + writer.writeheader() + writer.writerows(rows) + + if logger is not None: + label = parent or "serviceability" + logger.info( + "(%s) Wrote %s (%d row(s)) to %s", + label, + AFID_FRU_SUMMARY_CSV, + len(rows), + csv_path, + ) + return csv_path diff --git a/nodescraper/plugins/serviceability/afid_sag_lookup.py b/nodescraper/plugins/serviceability/afid_sag_lookup.py new file mode 100644 index 00000000..52957ce5 --- /dev/null +++ b/nodescraper/plugins/serviceability/afid_sag_lookup.py @@ -0,0 +1,244 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +import json +import logging +from collections import defaultdict +from typing import Any, Optional + +from .afid_events import build_afid_events_from_data +from .se_models import AfidEvent +from .serviceability_data import ServiceabilityDataModel + + +def load_afid_sag_data(path: Optional[str]) -> Optional[dict[str, Any]]: + """Load AFID_SAG.json when path is set and readable.""" + if not path or not str(path).strip(): + return None + try: + with open(path, encoding="utf-8") as handle: + data = json.load(handle) + except (OSError, json.JSONDecodeError): + return None + return data if isinstance(data, dict) else None + + +def normalize_fru_name(name: str) -> str: + """Normalize FRU labels for comparison (SAG serviceable_fru vs afid.fru).""" + return str(name).strip().upper().replace("-", "_") + + +def known_fru_names_from_sag(sag: Optional[dict[str, Any]]) -> list[str]: + """Return normalized FRU names declared under serviceable_fru in the SAG.""" + if not sag: + return [] + raw = sag.get("serviceable_fru") + if not isinstance(raw, list): + return [] + names: list[str] = [] + for item in raw: + if not isinstance(item, dict): + continue + for key in item: + text = str(key).strip() + if text: + names.append(normalize_fru_name(text)) + return sorted(set(names)) + + +def sag_fru_display_names(sag: Optional[dict[str, Any]]) -> list[str]: + """Return sorted FRU labels from SAG serviceable_fru using the original key text.""" + if not sag: + return [] + raw = sag.get("serviceable_fru") + if not isinstance(raw, list): + return [] + names: list[str] = [] + for item in raw: + if not isinstance(item, dict): + continue + for key in item: + text = str(key).strip() + if text: + names.append(text) + return sorted(set(names)) + + +def afid_entry_from_sag(afid: int, sag: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]: + """Return the afid map entry for one AFID when present.""" + if not sag: + return None + afid_map = sag.get("afid") + if not isinstance(afid_map, dict): + return None + entry = afid_map.get(str(afid)) + return entry if isinstance(entry, dict) else None + + +def afid_fru_from_sag(afid: int, sag: Optional[dict[str, Any]]) -> Optional[str]: + """Return the FRU string for an AFID from the SAG afid table.""" + entry = afid_entry_from_sag(afid, sag) + if not entry: + return None + fru = entry.get("fru") + if fru is None or not str(fru).strip(): + return None + return str(fru).strip() + + +def afid_summary_from_sag(afid: int, sag: Optional[dict[str, Any]]) -> Optional[str]: + """Build a short human-readable AFID fault summary from SAG metadata.""" + entry = afid_entry_from_sag(afid, sag) + if not entry: + return None + parts = [ + str(entry[key]).strip() + for key in ("error_category", "error_type") + if entry.get(key) is not None and str(entry[key]).strip() + ] + if parts: + return " / ".join(parts) + fallback = entry.get("service_action") + if fallback is not None and str(fallback).strip(): + return str(fallback).strip() + return None + + +def service_action_entry_from_sag( + service_action_num: int, + sag: Optional[dict[str, Any]], +) -> Optional[dict[str, Any]]: + """Return the service_actions map entry for one service action number.""" + if not sag: + return None + actions = sag.get("service_actions") + if not isinstance(actions, dict): + return None + entry = actions.get(str(service_action_num)) + return entry if isinstance(entry, dict) else None + + +def service_action_label_from_sag( + service_action_num: int, + sag: Optional[dict[str, Any]], +) -> Optional[str]: + """Return the display title for a service action from the SAG.""" + entry = service_action_entry_from_sag(service_action_num, sag) + if not entry: + return None + for key in ("title", "service_action"): + value = entry.get(key) + if value is not None and str(value).strip(): + return str(value).strip() + return None + + +def service_action_step_descriptions_from_sag( + service_action_num: int, + sag: Optional[dict[str, Any]], +) -> list[str]: + """Return ordered step descriptions for a service action from the SAG.""" + entry = service_action_entry_from_sag(service_action_num, sag) + if not entry: + return [] + steps = entry.get("steps") + if not isinstance(steps, list): + return [] + out: list[str] = [] + for step in steps: + if not isinstance(step, dict): + continue + desc = step.get("description") + if desc is not None and str(desc).strip(): + out.append(str(desc).strip()) + return out + + +def group_afid_events_by_fru( + events: list[AfidEvent], + sag: Optional[dict[str, Any]], +) -> dict[str, list[AfidEvent]]: + """Group parsed AFID events by SAG FRU (unknown FRU bucket when lookup fails).""" + grouped: dict[str, list[AfidEvent]] = defaultdict(list) + for event in events: + fru = afid_fru_from_sag(event.afid, sag) or "UNKNOWN_FRU" + grouped[normalize_fru_name(fru)].append(event) + return dict(grouped) + + +def format_collected_afid_fru_summary_lines( + events: list[AfidEvent], + sag: Optional[dict[str, Any]], + *, + rf_event_count: int = 0, +) -> list[str]: + """Build log lines summarizing collected AFID events grouped by FRU.""" + if not events: + return [f"No parseable AFID events from {rf_event_count} Redfish log member(s)."] + lines = [ + f"Collected AFID events by FRU ({len(events)} parseable from " + f"{rf_event_count} Redfish log member(s)):" + ] + grouped = group_afid_events_by_fru(events, sag) + for fru in sorted(grouped.keys()): + fru_events = grouped[fru] + afid_counts: dict[int, int] = defaultdict(int) + unit_counts: dict[str, int] = defaultdict(int) + for event in fru_events: + afid_counts[event.afid] += 1 + unit_counts[event.serviceable_unit] += 1 + afid_bits = ", ".join(f"{afid} x{count}" for afid, count in sorted(afid_counts.items())) + unit_bits = ", ".join(f"{unit} x{count}" for unit, count in sorted(unit_counts.items())) + lines.append(f" {fru}: AFIDs [{afid_bits}]") + lines.append(f" units: {unit_bits}") + known = known_fru_names_from_sag(sag) + if known: + missing = sorted(set(known) - set(grouped.keys())) + if missing: + lines.append(f" FRUs in SAG with no collected AFID events: {', '.join(missing)}") + else: + lines.append(" All SAG serviceable FRUs have at least one collected AFID event.") + return lines + + +def log_afid_fru_summary( + logger: logging.Logger, + parent: str, + data: ServiceabilityDataModel, + sag_path: Optional[str], +) -> None: + """Log AFID counts grouped by FRU when AFID_SAG path is configured.""" + if not sag_path or not str(sag_path).strip(): + return + sag = load_afid_sag_data(sag_path) + events = build_afid_events_from_data(data) + for line in format_collected_afid_fru_summary_lines( + events, + sag, + rf_event_count=len(data.rf_events), + ): + logger.info("(%s) %s", parent, line) diff --git a/nodescraper/plugins/serviceability/afid_sag_paths.py b/nodescraper/plugins/serviceability/afid_sag_paths.py new file mode 100644 index 00000000..6f0b86fd --- /dev/null +++ b/nodescraper/plugins/serviceability/afid_sag_paths.py @@ -0,0 +1,53 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from pathlib import Path +from typing import Optional + +DEFAULT_AFID_SAG_PATH = "/opt/amd/afid/AFID_SAG.json" + + +def default_afid_sag_path() -> str: + """Return the default AFID_SAG.json path when analysis_args does not override it.""" + return DEFAULT_AFID_SAG_PATH + + +def resolve_configured_afid_sag_path(configured_path: Optional[str]) -> str: + """Resolve AFID SAG path from analysis_args or the built-in default.""" + if configured_path is not None and str(configured_path).strip(): + return str(configured_path).strip() + return default_afid_sag_path() + + +def validate_afid_sag_path(path: str) -> str: + """Return path when the AFID SAG file exists, otherwise raise HubRunError.""" + from .se_runner import HubRunError + + sag_path = Path(path) + if not sag_path.is_file(): + raise HubRunError(f"AFID SAG file not found: {path}") + return path diff --git a/nodescraper/plugins/serviceability/analysis_window.py b/nodescraper/plugins/serviceability/analysis_window.py new file mode 100644 index 00000000..911246e1 --- /dev/null +++ b/nodescraper/plugins/serviceability/analysis_window.py @@ -0,0 +1,231 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +"""Shared serviceability analysis path.""" +from __future__ import annotations + +import logging +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + +from .afid_events import build_afid_events_from_data +from .analyzer_args import ServiceabilityAnalyzerArgs +from .cper_decode import CperDecodeError, decode_cper_raw_attachments +from .se_models import AfidEvent, ServiceabilityBlock +from .se_runner import HubRunError, run_entry_point_hub, run_service_hub +from .serviceability_data import ServiceabilityDataModel + + +@dataclass +class ServiceabilityWindowResult: + """Outcome of analyze_serviceability_window.""" + + ok: bool + message: str + afid_events: list[AfidEvent] + serviceability: Optional[ServiceabilityBlock] = None + error: Optional[str] = None + + +def _cper_raw_needing_decode(data: ServiceabilityDataModel) -> dict[str, str]: + """Return CPER attachments that still need configured decode.""" + from .mi3xx.mi3xx_cper_utils import should_skip_cper_fetch_or_decode + + raw = data.cper_raw or {} + if not raw: + return {} + by_id: dict[str, dict[str, Any]] = {} + for member in data.rf_events: + if not isinstance(member, dict): + continue + eid = member.get("Id") + if eid is not None: + by_id[str(eid)] = member + out: dict[str, str] = {} + for event_id, blob in raw.items(): + ev = by_id.get(str(event_id)) + if ev is not None and should_skip_cper_fetch_or_decode(ev): + continue + out[str(event_id)] = blob + return out + + +def analyze_serviceability_window( + data: ServiceabilityDataModel, + args: ServiceabilityAnalyzerArgs, + *, + logger: Optional[logging.Logger] = None, + parent: str = "analyze_serviceability_window", +) -> ServiceabilityWindowResult: + """Build AFID events and optionally run the configured service hub.""" + log = logger or logging.getLogger(__name__) + events = data.afid_events or build_afid_events_from_data(data) + data.afid_events = events + + if args.skip_hub: + if args.afid_sag_path and str(args.afid_sag_path).strip(): + data.afid_sag_path = str(args.afid_sag_path).strip() + block = ServiceabilityBlock(afid_events=events) + data.serviceability = block + return ServiceabilityWindowResult( + ok=True, + message=f"Built {len(events)} AFID event(s); hub skipped", + afid_events=events, + serviceability=block, + ) + + cper_data = data.cper_data or {} + cper_raw_to_decode = _cper_raw_needing_decode(data) + skipped_cper = len(data.cper_raw or {}) - len(cper_raw_to_decode) + if skipped_cper: + from .mi3xx.mi3xx_cper_utils import CPER_METHOD_AFID_MAX + + log.info( + "(%s) Skipping CPER decode for %d CPER attachment(s); Redfish log " + "already has usable ACA fields (CPER-method AFID<=%s or no serial on decode)", + parent, + skipped_cper, + CPER_METHOD_AFID_MAX, + ) + if cper_raw_to_decode and not cper_data: + if not args.cper_decode_module: + log.warning( + "(%s) %d CPER attachment(s) collected but cper_decode_module is " + "not set in analysis_args; skipping CPER decode", + parent, + len(cper_raw_to_decode), + ) + else: + log.info( + "(%s) Decoding %d CPER attachment(s) via %s.%s", + parent, + len(cper_raw_to_decode), + args.cper_decode_module, + args.cper_decode_method, + ) + try: + cper_data = decode_cper_raw_attachments( + cper_raw_to_decode, + cper_decode_module=args.cper_decode_module, + cper_decode_method=args.cper_decode_method, + logger=log, + ) + data.cper_data = cper_data + log.info( + "(%s) CPER decode finished: %d of %d attachment(s) decoded", + parent, + len(cper_data), + len(cper_raw_to_decode), + ) + except CperDecodeError as exc: + log.warning("(%s) %s; continuing without decoded CPER", parent, exc) + elif cper_data: + log.info( + "(%s) Using %d pre-decoded CPER record(s) from collection", + parent, + len(cper_data), + ) + + if args.uses_entry_point_hub() and cper_data: + events = build_afid_events_from_data(data) + data.afid_events = events + + if args.uses_entry_point_hub() and not events: + return ServiceabilityWindowResult( + ok=False, + message="No AFID events could be built from collected Redfish data", + afid_events=events, + error="empty afid_events", + ) + + sag_path = args.resolved_afid_sag_path() + data.afid_sag_path = sag_path + log.info( + "(%s) Using AFID_SAG file: %s", + parent, + Path(sag_path).expanduser().resolve(), + ) + try: + if args.uses_module_hub(): + block = run_service_hub( + hub_python_module=args.hub_python_module, # type: ignore[arg-type] + hub_display_name=args.hub_display_name, + afid_events=events, + afid_sag_path=sag_path, + rf_events=data.rf_events, + cper_data=cper_data or None, + hub_options=args.resolved_hub_options(), + hub_analyze_method=args.hub_analyze_method, + hub_init_path_kwarg=args.hub_init_path_kwarg, + ) + else: + block = run_entry_point_hub( + hub_entry_point=args.resolved_hub_entry_point(), + hub_display_name=args.hub_display_name, + afid_events=events, + afid_sag_path=sag_path, + rf_events=data.rf_events, + rf_event_count=len(data.rf_events), + raise_on_error=args.hub_raise_on_error, + prefer_rf_events=args.hub_prefer_rf_events and not bool(cper_data), + ) + except (HubRunError, ValueError) as exc: + return ServiceabilityWindowResult( + ok=False, + message=str(exc), + afid_events=events, + error=str(exc), + ) + + data.serviceability = block + hub_label = ( + args.hub_display_name + or args.hub_python_module + or (args.resolved_hub_entry_point() if args.uses_entry_point_hub() else None) + ) + cper_summary = "" + if cper_data: + cper_summary = f", {len(cper_data)} decoded CPER(s)" + elif cper_raw_to_decode: + cper_summary = f", {len(cper_raw_to_decode)} CPER attachment(s) not decoded" + elif data.cper_raw: + cper_summary = f", {len(data.cper_raw)} CPER attachment(s) omitted (ACA on log entry)" + ver_bits: list[str] = [] + if block.hub_version: + ver_bits.append(f"hub {block.hub_version}") + if block.afid_sag_file_version: + ver_bits.append(f"AFID_SAG {block.afid_sag_file_version}") + ver_suffix = f" [{'; '.join(ver_bits)}]" if ver_bits else "" + message = ( + f"{hub_label}: {len(block.solution)} solution(s) " + f"from {len(data.rf_events)} Redfish event(s){cper_summary}{ver_suffix}" + ) + return ServiceabilityWindowResult( + ok=True, + message=message, + afid_events=events, + serviceability=block, + ) diff --git a/nodescraper/plugins/serviceability/analyzer_args.py b/nodescraper/plugins/serviceability/analyzer_args.py index 639822cc..3099e6a5 100644 --- a/nodescraper/plugins/serviceability/analyzer_args.py +++ b/nodescraper/plugins/serviceability/analyzer_args.py @@ -27,25 +27,42 @@ from typing import Any, Optional -from pydantic import Field, field_validator, model_validator +from pydantic import Field, field_validator from nodescraper.models import AnalyzerArgs +from .afid_sag_paths import resolve_configured_afid_sag_path + class ServiceabilityAnalyzerArgs(AnalyzerArgs): - """Analyzer args for serviceability plugins that run a configurable Python hub.""" + """Analyzer args for serviceability plugins that run a Python or entry-point service hub.""" hub_python_module: Optional[str] = Field( default=None, description="Import path for the hub module (class implements hub_analyze_method); hub_options forwards kwargs.", ) + hub_entry_point: Optional[str] = Field( + default=None, + description="Registered hub entry point name when hub_python_module is omitted (required in analysis_args).", + ) + hub_raise_on_error: bool = Field( + default=False, + description="When True, entry-point hub analyze() exceptions raise instead of status:error.", + ) + hub_prefer_rf_events: bool = Field( + default=True, + description=( + "When True and Redfish rf_events are available without decoded CPER data, " + "pass raw rf_events to the entry-point hub for redfish transformer parsing." + ), + ) hub_display_name: Optional[str] = Field( default=None, description="Optional label for analyzer status messages.", ) afid_sag_path: Optional[str] = Field( default=None, - description="Path to hub config (e.g. AFID_SAG.json); passed as hub_init_path_kwarg.", + description="Path to AFID_SAG.json. When omitted, uses /opt/amd/afid/AFID_SAG.json.", ) hub_init_path_kwarg: str = Field( default="afid_sag", @@ -130,6 +147,7 @@ def _strip_from_date(cls, value: object) -> Optional[str]: "afid_sag_path", "hub_python_module", "hub_display_name", + "hub_entry_point", "cper_decode_module", ) @classmethod @@ -139,12 +157,26 @@ def _strip_optional_strings(cls, value: Optional[str]) -> Optional[str]: text = str(value).strip() return text or None - @model_validator(mode="after") - def _require_hub_config_when_running(self) -> ServiceabilityAnalyzerArgs: - if self.skip_hub: - return self - if not self.afid_sag_path: - raise ValueError("afid_sag_path is required when running the service hub.") - if not self.hub_python_module: - raise ValueError("hub_python_module is required when running the service hub.") - return self + def resolved_afid_sag_path(self) -> str: + """Return the AFID SAG path from analysis_args or the built-in default.""" + return resolve_configured_afid_sag_path(self.afid_sag_path) + + def resolved_hub_entry_point(self) -> str: + """Return the configured entry-point hub name from analysis_args.""" + from .se_runner import list_hub_entry_point_names + + if not self.hub_entry_point: + available = ", ".join(list_hub_entry_point_names()) or "(none installed)" + raise ValueError( + "hub_entry_point is required in analysis_args when hub_python_module is omitted; " + f"available: {available}" + ) + return self.hub_entry_point.strip() + + def uses_module_hub(self) -> bool: + """Return True when analysis_args selects a Python import-path hub.""" + return bool(self.hub_python_module) + + def uses_entry_point_hub(self) -> bool: + """Return True when analysis uses a registered hub entry point.""" + return not self.uses_module_hub() diff --git a/nodescraper/plugins/serviceability/event_log_utils.py b/nodescraper/plugins/serviceability/event_log_utils.py new file mode 100644 index 00000000..95200cf1 --- /dev/null +++ b/nodescraper/plugins/serviceability/event_log_utils.py @@ -0,0 +1,122 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from typing import Any, Optional + +from nodescraper.connection.redfish import RF_MEMBERS + +from .time_utils import TimeOperator, satisfies_time_check + +_EVENT_TIMESTAMP_KEYS = ("Created", "EventTimestamp", "Timestamp") + + +def event_timestamp(event: dict[str, Any]) -> Optional[str]: + """Return the first populated Redfish log-entry timestamp field.""" + for key in _EVENT_TIMESTAMP_KEYS: + value = event.get(key) + if value is not None and str(value).strip(): + return str(value).strip() + return None + + +def filter_event_log_members( + members: list[Any], + *, + reference_time: Optional[str] = None, + time_operator: Optional[TimeOperator] = None, +) -> list[Any]: + """Filter log members by optional reference-time bounds.""" + if reference_time is None or time_operator is None: + return list(members) + filtered: list[Any] = [] + for member in members: + if not isinstance(member, dict): + filtered.append(member) + continue + timestamp = event_timestamp(member) + if timestamp is None or satisfies_time_check(timestamp, reference_time, time_operator): + filtered.append(member) + return filtered + + +def _collect_members_from_responses(responses: dict[str, Any]) -> list[Any]: + """Merge Members arrays from a URI-keyed redfish_responses map.""" + merged_members: list[Any] = [] + for body in responses.values(): + if not isinstance(body, dict): + continue + members = body.get(RF_MEMBERS) + if isinstance(members, list): + merged_members.extend(members) + return merged_members + + +def _is_redfish_responses_map(payload: dict[str, Any]) -> bool: + """Return True when payload looks like a collector redfish_responses.json dump.""" + if not payload: + return False + has_members = False + for key, value in payload.items(): + if not isinstance(key, str) or not key.startswith("/"): + return False + if not isinstance(value, dict): + return False + if isinstance(value.get(RF_MEMBERS), list): + has_members = True + return has_members + + +def rf_events_from_json_payload(payload: Any) -> tuple[list[Any], dict[str, Any]]: + """Normalize Redfish Entries JSON or a bare member list into rf_events + responses.""" + if isinstance(payload, list): + return list(payload), {} + + if not isinstance(payload, dict): + raise ValueError( + "Serviceability data input must be a JSON object, Redfish Entries collection, " + "redfish_responses.json dump, or a list of LogEntry members" + ) + + members = payload.get(RF_MEMBERS) + if isinstance(members, list): + responses: dict[str, Any] = {} + odata_id = payload.get("@odata.id") + if odata_id: + responses[str(odata_id)] = payload + return list(members), responses + + nested_responses = payload.get("responses") + if isinstance(nested_responses, dict) and _is_redfish_responses_map(nested_responses): + return _collect_members_from_responses(nested_responses), dict(nested_responses) + + if _is_redfish_responses_map(payload): + return _collect_members_from_responses(payload), dict(payload) + + raise ValueError( + "Serviceability data JSON must include rf_events, a Redfish Members array, " + "or a redfish_responses.json URI map" + ) diff --git a/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py b/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py index 87bbaa3f..f210894e 100644 --- a/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py +++ b/nodescraper/plugins/serviceability/mi3xx/mi3xx_analyzer.py @@ -7,7 +7,7 @@ # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights -# to use, copy, modify, distribute, sublicense, and/or sell +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # @@ -25,43 +25,13 @@ ############################################################################### from __future__ import annotations -from typing import Any, ClassVar, Optional - -from pydantic import BaseModel, Field - -from nodescraper.enums import ExecutionStatus -from nodescraper.interfaces import DataAnalyzer -from nodescraper.models import TaskResult -from nodescraper.plugins.serviceability.afid_events import build_afid_events_from_data -from nodescraper.plugins.serviceability.analyzer_args import ServiceabilityAnalyzerArgs -from nodescraper.plugins.serviceability.cper_decode import ( - CperDecodeError, - decode_cper_raw_attachments, -) -from nodescraper.plugins.serviceability.se_adapter import ( - format_serviceability_solution_lines, +from nodescraper.plugins.serviceability.serviceability_hub_analyzer import ( + ServiceabilityHubAnalyzer, ) -from nodescraper.plugins.serviceability.se_models import ServiceabilityBlock -from nodescraper.plugins.serviceability.se_runner import SeRunError, run_service_hub -from nodescraper.plugins.serviceability.serviceability_data import ( - ServiceabilityDataModel, -) - -from .mi3xx_cper_utils import CPER_METHOD_AFID_MAX, should_skip_cper_fetch_or_decode - - -class AfidSagMetadataArtifact(BaseModel): - """Hub AFID_SAG metadata snapshot; written to ``afid_sag_metadata.json``.""" - - ARTIFACT_LOG_BASENAME: ClassVar[str] = "afid_sag_metadata" - - metadata: dict[str, Any] = Field(default_factory=dict) - -class MI3XXAnalyzer(DataAnalyzer[ServiceabilityDataModel, ServiceabilityAnalyzerArgs]): - """Build AFID events from collected data and run the configured service hub.""" - DATA_MODEL = ServiceabilityDataModel +class MI3XXAnalyzer(ServiceabilityHubAnalyzer): + """Build AFID events from collected data and run the configured Python service hub.""" DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( "Builds AFID events from collected Redfish event log members (and optional assembly metadata).", @@ -69,152 +39,3 @@ class MI3XXAnalyzer(DataAnalyzer[ServiceabilityDataModel, ServiceabilityAnalyzer "Runs the configured Python service hub (hub_python_module) to produce service recommendations.", "When analysis_args.skip_hub is true, only builds AFID events without running the hub.", ) - - def analyze_data( - self, - data: ServiceabilityDataModel, - args: Optional[ServiceabilityAnalyzerArgs] = None, - ) -> TaskResult: - if args is None: - self.result.status = ExecutionStatus.NOT_RAN - self.result.message = "ServiceabilityAnalyzerArgs are required" - return self.result - - events = data.afid_events or build_afid_events_from_data(data) - data.afid_events = events - - if args.skip_hub: - data.serviceability = ServiceabilityBlock(afid_events=events) - self.result.status = ExecutionStatus.OK - self.result.message = f"Built {len(events)} AFID event(s); hub skipped" - self._log_serviceability_solutions(data.serviceability) - return self.result - - parent = self.parent or self.__class__.__name__ - cper_data = data.cper_data or {} - cper_raw_to_decode = self._cper_raw_needing_decode(data) - skipped_cper = len(data.cper_raw or {}) - len(cper_raw_to_decode) - if skipped_cper: - self.logger.info( - "(%s) Skipping CPER decode for %d CPER attachment(s); Redfish log " - "already has usable ACA fields (CPER-method AFID<=%s or no serial on decode)", - parent, - skipped_cper, - CPER_METHOD_AFID_MAX, - ) - if cper_raw_to_decode and not cper_data: - if not args.cper_decode_module: - self.logger.warning( - "(%s) %d CPER attachment(s) collected but cper_decode_module is " - "not set in analysis_args; skipping CPER decode", - parent, - len(cper_raw_to_decode), - ) - else: - self.logger.info( - "(%s) Decoding %d CPER attachment(s) via %s.%s", - parent, - len(cper_raw_to_decode), - args.cper_decode_module, - args.cper_decode_method, - ) - try: - cper_data = decode_cper_raw_attachments( - cper_raw_to_decode, - cper_decode_module=args.cper_decode_module, - cper_decode_method=args.cper_decode_method, - logger=self.logger, - ) - data.cper_data = cper_data - self.logger.info( - "(%s) CPER decode finished: %d of %d attachment(s) decoded", - parent, - len(cper_data), - len(cper_raw_to_decode), - ) - except CperDecodeError as exc: - self.logger.warning( - "(%s) %s; continuing without decoded CPER", - parent, - exc, - ) - elif cper_data: - self.logger.info( - "(%s) Using %d pre-decoded CPER record(s) from collection", - parent, - len(cper_data), - ) - - try: - block = run_service_hub( - hub_python_module=args.hub_python_module, # type: ignore[arg-type] - hub_display_name=args.hub_display_name, - afid_events=events, - afid_sag_path=args.afid_sag_path, # type: ignore[arg-type] - rf_events=data.rf_events, - cper_data=cper_data or None, - hub_options=args.resolved_hub_options(), - hub_analyze_method=args.hub_analyze_method, - hub_init_path_kwarg=args.hub_init_path_kwarg, - ) - except (SeRunError, ValueError) as exc: - self.result.status = ExecutionStatus.ERROR - self.result.message = str(exc) - return self.result - - data.serviceability = block - self._append_afid_sag_metadata_artifact(block) - self._log_serviceability_solutions(block) - hub_label = args.hub_display_name or args.hub_python_module - self.result.status = ExecutionStatus.OK - cper_summary = "" - if cper_data: - cper_summary = f", {len(cper_data)} decoded CPER(s)" - elif cper_raw_to_decode: - cper_summary = f", {len(cper_raw_to_decode)} CPER attachment(s) not decoded" - elif data.cper_raw: - cper_summary = f", {len(data.cper_raw)} CPER attachment(s) omitted (ACA on log entry)" - ver_bits: list[str] = [] - if block.hub_version: - ver_bits.append(f"hub {block.hub_version}") - if block.afid_sag_file_version: - ver_bits.append(f"AFID_SAG {block.afid_sag_file_version}") - ver_suffix = f" [{'; '.join(ver_bits)}]" if ver_bits else "" - self.result.message = ( - f"{hub_label}: {len(block.solution)} solution(s) " - f"from {len(data.rf_events)} Redfish event(s){cper_summary}{ver_suffix}" - ) - return self.result - - @staticmethod - def _cper_raw_needing_decode(data: ServiceabilityDataModel) -> dict[str, str]: - """Subset of ``cper_raw`` that still needs configured CPER decode (not already on the log).""" - raw = data.cper_raw or {} - if not raw: - return {} - by_id: dict[str, dict[str, Any]] = {} - for member in data.rf_events: - if not isinstance(member, dict): - continue - eid = member.get("Id") - if eid is not None: - by_id[str(eid)] = member - out: dict[str, str] = {} - for event_id, blob in raw.items(): - ev = by_id.get(str(event_id)) - if ev is not None and should_skip_cper_fetch_or_decode(ev): - continue - out[str(event_id)] = blob - return out - - def _append_afid_sag_metadata_artifact(self, block: ServiceabilityBlock) -> None: - if block.afid_sag_metadata is None: - return - self.result.artifacts.append( - AfidSagMetadataArtifact(metadata=dict(block.afid_sag_metadata)) - ) - - def _log_serviceability_solutions(self, block: ServiceabilityBlock) -> None: - parent = self.parent or self.__class__.__name__ - for line in format_serviceability_solution_lines(block): - self.logger.info("(%s) %s", parent, line) diff --git a/nodescraper/plugins/serviceability/mi4xx/__init__.py b/nodescraper/plugins/serviceability/mi4xx/__init__.py new file mode 100644 index 00000000..a862c0bc --- /dev/null +++ b/nodescraper/plugins/serviceability/mi4xx/__init__.py @@ -0,0 +1,38 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from .mi4xx_analyzer import MI4XXAnalyzer +from .mi4xx_analyzer_args import Mi4xxServiceabilityAnalyzerArgs +from .mi4xx_collector import MI4XXCollector +from .mi4xx_collector_args import MI4XXCollectorArgs +from .serviceability_plugin_mi4xx import Mi4xxServiceabilityPlugin + +__all__ = [ + "MI4XXAnalyzer", + "MI4XXCollector", + "MI4XXCollectorArgs", + "Mi4xxServiceabilityAnalyzerArgs", + "Mi4xxServiceabilityPlugin", +] diff --git a/nodescraper/plugins/serviceability/mi4xx/mi4xx_analyzer.py b/nodescraper/plugins/serviceability/mi4xx/mi4xx_analyzer.py new file mode 100644 index 00000000..6676a370 --- /dev/null +++ b/nodescraper/plugins/serviceability/mi4xx/mi4xx_analyzer.py @@ -0,0 +1,41 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from nodescraper.plugins.serviceability.serviceability_hub_analyzer import ( + ServiceabilityHubAnalyzer, +) + + +class MI4XXAnalyzer(ServiceabilityHubAnalyzer): + """Build AFID events from collected data and run the configured entry-point hub.""" + + DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( + "Builds AFID events from collected Redfish event log members (and optional assembly metadata).", + "Runs the configured entry-point service hub (hub_entry_point in analysis_args) to produce service recommendations.", + "When analysis_args.skip_hub is true, only builds AFID events without running the hub.", + "Supports offline analysis from a prior collection via --data with --collection False.", + ) diff --git a/nodescraper/plugins/serviceability/mi4xx/mi4xx_analyzer_args.py b/nodescraper/plugins/serviceability/mi4xx/mi4xx_analyzer_args.py new file mode 100644 index 00000000..a87cdf7c --- /dev/null +++ b/nodescraper/plugins/serviceability/mi4xx/mi4xx_analyzer_args.py @@ -0,0 +1,76 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from typing import Optional + +from pydantic import Field, field_validator, model_validator + +from nodescraper.plugins.serviceability.analyzer_args import ServiceabilityAnalyzerArgs + + +class Mi4xxServiceabilityAnalyzerArgs(ServiceabilityAnalyzerArgs): + """Analysis args for Mi4xxServiceabilityPlugin (AFSE entry point).""" + + hub_entry_point: str = Field( + default="afse", + description="Registered AFSE entry point name (MI4XX service hub).", + ) + hub_display_name: Optional[str] = Field( + default="AFSE", + description="Label for analyzer status messages.", + ) + hub_python_module: Optional[str] = Field( + default=None, + description="Not used for MI4XX; AFSE is selected via hub_entry_point afse.", + ) + rf_event_log_uri: str = Field( + default="/redfish/v1/Systems/Instinct_Accelerators/LogServices/EventLog/Entries", + description="Redfish URI for the Instinct accelerator event log Entries collection.", + ) + + @field_validator("rf_event_log_uri") + @classmethod + def _strip_rf_event_log_uri(cls, value: object) -> str: + text = str(value).strip() + if not text: + raise ValueError("rf_event_log_uri must be a non-empty Redfish URI") + return text + + def resolved_rf_event_log_uri(self) -> str: + """Return the configured event log Entries URI.""" + return str(self.rf_event_log_uri).strip() + + @model_validator(mode="after") + def _mi4xx_uses_afse(self) -> "Mi4xxServiceabilityAnalyzerArgs": + if self.hub_python_module: + raise ValueError( + "Mi4xxServiceabilityPlugin uses AFSE via hub_entry_point; " + "hub_python_module is not supported" + ) + if str(self.hub_entry_point).strip().lower() != "afse": + raise ValueError("Mi4xxServiceabilityPlugin supports hub_entry_point 'afse' only") + return self diff --git a/nodescraper/plugins/serviceability/mi4xx/mi4xx_collector.py b/nodescraper/plugins/serviceability/mi4xx/mi4xx_collector.py new file mode 100644 index 00000000..6b8a682c --- /dev/null +++ b/nodescraper/plugins/serviceability/mi4xx/mi4xx_collector.py @@ -0,0 +1,130 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from typing import Any, Optional + +from nodescraper.connection.redfish import RF_MEMBERS_COUNT +from nodescraper.plugins.serviceability.afid_sag_lookup import log_afid_fru_summary +from nodescraper.plugins.serviceability.event_log_utils import filter_event_log_members +from nodescraper.plugins.serviceability.serviceability_collector import ( + ServiceabilityCollectorBase, +) +from nodescraper.plugins.serviceability.serviceability_data import ( + DeviceInfo, + ServiceabilityDataModel, +) + +from .mi4xx_collector_args import MI4XXCollectorArgs +from .mi4xx_event_log_paging import fetch_mi4xx_event_log + + +class MI4XXCollector(ServiceabilityCollectorBase[MI4XXCollectorArgs]): + """Collect MI4xx BMC Redfish event logs for service hub analysis.""" + + DOCUMENTATION_COLLECTION_ITEMS: tuple[str, ...] = ( + "Redfish GET: Instinct accelerator event log Entries (collection_args.rf_event_log_uri).", + "MI4xx-only pagination: follows Members@odata.nextLink and falls back to $skip when the BMC reports more entries than one page returns.", + "Paginated Members collection and optional top, reference_time/time_operator filters.", + "Optional chassis Assembly GETs (rf_assembly_uri_template + rf_chassis_devices).", + "Optional firmware bundle inventory GET (rf_firmware_bundle_uri) for component details.", + "Optional AFID_SAG-backed FRU grouping summary when collection_args.afid_sag_path is set.", + ) + + def _fetch_event_log(self, args: MI4XXCollectorArgs, uri: str): + if args.follow_next_link: + return fetch_mi4xx_event_log(self, uri, max_pages=args.max_pages) + return self._run_redfish_get(uri, log_artifact=True) + + def _fetch_top(self, args: MI4XXCollectorArgs, top: int, max_pages: int): + event_uri = args.resolved_event_log_uri() + probe = self._run_redfish_get(f"{event_uri}?$top=1", log_artifact=True) + if not probe.success or probe.data is None: + return probe + + count = probe.data.get(RF_MEMBERS_COUNT, 0) + if count <= top: + return self._fetch_event_log(args, event_uri) + + skip_uri = f"{event_uri}?$skip={count - top}" + if args.follow_next_link: + return fetch_mi4xx_event_log(self, skip_uri, max_pages=max_pages) + return self._run_redfish_get(skip_uri, log_artifact=True) + + def _after_collect_data( + self, + data: ServiceabilityDataModel, + args: MI4XXCollectorArgs, + ) -> None: + data.afid_sag_path = args.resolved_afid_sag_path_for_collection() + parent = self.parent or self.__class__.__name__ + log_afid_fru_summary( + self.logger, + parent, + data, + data.afid_sag_path, + ) + + def filter_event_members( + self, + members: list[Any], + args: MI4XXCollectorArgs, + ) -> list[Any]: + return filter_event_log_members( + members, + reference_time=args.reference_time, + time_operator=args.time_operator, + ) + + def is_cper_event(self, event: dict) -> bool: + return False + + def collect_cper_attachments(self, rf_events: list[Any]) -> dict[str, str]: + return {} + + def parse_assembly_entry( + self, + designation: str, + assembly_member_entry: dict[str, Any], + args: MI4XXCollectorArgs, + ) -> DeviceInfo: + return DeviceInfo( + name=assembly_member_entry.get("Name") or designation, + part_number=assembly_member_entry.get("PartNumber"), + production_date=assembly_member_entry.get("ProductionDate"), + serial_number=assembly_member_entry.get("SerialNumber"), + version=assembly_member_entry.get("Version"), + ) + + def extract_component_details( + self, + firmware_inventory_payload: dict[str, Any], + args: MI4XXCollectorArgs, + ) -> Optional[str]: + details = firmware_inventory_payload.get("Details") + if details is not None: + return str(details) + return None diff --git a/nodescraper/plugins/serviceability/mi4xx/mi4xx_collector_args.py b/nodescraper/plugins/serviceability/mi4xx/mi4xx_collector_args.py new file mode 100644 index 00000000..2334817a --- /dev/null +++ b/nodescraper/plugins/serviceability/mi4xx/mi4xx_collector_args.py @@ -0,0 +1,154 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from typing import List, Optional + +from pydantic import Field, field_validator, model_validator + +from nodescraper.models import CollectorArgs +from nodescraper.plugins.serviceability.time_utils import ( + TimeOperator, + is_valid_iso_datetime, +) + +from .mi4xx_analyzer_args import Mi4xxServiceabilityAnalyzerArgs + + +class MI4XXCollectorArgs(CollectorArgs): + """MI4xx OOB Redfish serviceability collector arguments.""" + + rf_event_log_uri: str = Field( + default_factory=lambda: Mi4xxServiceabilityAnalyzerArgs().resolved_rf_event_log_uri(), + description="Redfish URI for the Instinct accelerator event log Entries collection.", + ) + follow_next_link: bool = Field( + default=True, + description="If True, follow Members@odata.nextLink up to max_pages; else single GET.", + ) + max_pages: int = Field( + default=200, + ge=1, + le=10_000, + description="Safety cap on the number of pages when following event log pagination.", + ) + top: Optional[int] = Field( + default=None, + ge=1, + description="Most recent N entries via $skip after count probe; None collects full window.", + ) + reference_time: Optional[str] = Field( + default=None, + description=( + "Optional ISO-8601 date or date-time used with time_operator " + "(e.g. 2026-05-17 or 2026-05-17T13:01:00)." + ), + ) + time_operator: Optional[TimeOperator] = Field( + default=None, + description="Comparison operator applied when reference_time is set.", + ) + rf_assembly_uri_template: Optional[str] = Field( + default=None, + description="Optional Redfish URI template containing {device} for chassis Assembly GETs.", + ) + rf_chassis_devices: Optional[List[str]] = Field( + default=None, + description="Optional chassis designations paired with rf_assembly_uri_template.", + ) + rf_firmware_bundle_uri: Optional[str] = Field( + default=None, + description="Optional Redfish URI for firmware bundle inventory.", + ) + afid_sag_path: Optional[str] = Field( + default=None, + description=( + "Optional AFID_SAG.json path for collector FRU grouping logs. " + "When omitted, FRU summary logging is skipped." + ), + ) + + @field_validator("afid_sag_path") + @classmethod + def _strip_afid_sag_path(cls, value: Optional[str]) -> Optional[str]: + if value is None: + return None + text = str(value).strip() + return text or None + + @field_validator("rf_event_log_uri") + @classmethod + def _strip_rf_event_log_uri(cls, value: object) -> str: + text = str(value).strip() + if not text: + raise ValueError("rf_event_log_uri must be a non-empty Redfish URI") + return text + + @field_validator("reference_time") + @classmethod + def _validate_reference_time_iso(cls, value: Optional[str]) -> Optional[str]: + if value is None: + return None + text = str(value).strip() + if not text: + raise ValueError("reference_time must be a non-empty ISO-8601 string") + if not is_valid_iso_datetime(text): + raise ValueError(f"reference_time is not ISO-8601 compliant: {value!r}") + return text + + @model_validator(mode="after") + def _reference_time_requires_operator(self) -> MI4XXCollectorArgs: + has_ref = self.reference_time is not None + has_op = self.time_operator is not None + if has_ref != has_op: + raise ValueError("Provide both reference_time and time_operator, or omit both.") + return self + + @model_validator(mode="after") + def _assembly_consistency(self) -> MI4XXCollectorArgs: + has_tpl = bool( + self.rf_assembly_uri_template and "{device}" in self.rf_assembly_uri_template + ) + has_dev = bool(self.rf_chassis_devices) + if has_tpl != has_dev: + raise ValueError( + "Provide both rf_assembly_uri_template (with '{device}') and rf_chassis_devices, " + "or omit both to skip assembly collection." + ) + return self + + def resolved_event_log_uri(self) -> str: + """Return the configured event log Entries URI.""" + return str(self.rf_event_log_uri).strip() + + @classmethod + def default_event_log_uri(cls) -> str: + """Return the built-in default for rf_event_log_uri from Mi4xxServiceabilityAnalyzerArgs.""" + return Mi4xxServiceabilityAnalyzerArgs().resolved_rf_event_log_uri() + + def resolved_afid_sag_path_for_collection(self) -> Optional[str]: + """Return AFID_SAG path when set for collector FRU summary logging.""" + return self.afid_sag_path diff --git a/nodescraper/plugins/serviceability/mi4xx/mi4xx_event_log_paging.py b/nodescraper/plugins/serviceability/mi4xx/mi4xx_event_log_paging.py new file mode 100644 index 00000000..1331ee71 --- /dev/null +++ b/nodescraper/plugins/serviceability/mi4xx/mi4xx_event_log_paging.py @@ -0,0 +1,142 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional + +from nodescraper.connection.redfish import ( + RF_MEMBERS, + RF_MEMBERS_COUNT, + RF_MEMBERS_NEXT_LINK, + RedfishGetResult, +) + +if TYPE_CHECKING: + from .mi4xx_collector import MI4XXCollector + + +def _base_event_log_uri(uri: str) -> str: + """Return the event log collection URI without query parameters.""" + return uri.split("?", 1)[0] + + +def _reported_member_total(payload: dict[str, Any]) -> Optional[int]: + """Return Members@odata.count when the BMC reports an integer total.""" + raw = payload.get(RF_MEMBERS_COUNT) + if isinstance(raw, int): + return raw + return None + + +def fetch_mi4xx_event_log( + collector: MI4XXCollector, + uri: str, + *, + max_pages: int, +) -> RedfishGetResult: + """Fetch and merge paginated MI4xx event log pages for collection only. + + Args: + collector: MI4xx collector instance used for Redfish GET requests. + uri: Event log collection URI for the first page. + max_pages: Maximum number of pages to fetch including the first page. + + Returns: + RedfishGetResult with merged Members across all fetched pages. + """ + parent = collector.parent or collector.__class__.__name__ + first = collector._run_redfish_get(uri, log_artifact=True) + if not first.success or first.data is None: + return first + + merged_members: list[Any] = list(first.data.get(RF_MEMBERS) or []) + merged_data = dict(first.data) + reported_total = _reported_member_total(first.data) + base_uri = _base_event_log_uri(uri) + + pages_fetched = 1 + next_link: Optional[str] = first.data.get(RF_MEMBERS_NEXT_LINK) + last_status = first.status_code + + while pages_fetched < max_pages: + if next_link: + page_path = next_link + elif reported_total is not None and len(merged_members) < reported_total: + page_path = f"{base_uri}?$skip={len(merged_members)}" + else: + break + + page = collector._run_redfish_get(page_path, log_artifact=True) + last_status = page.status_code + if not page.success or page.data is None: + collector.logger.warning( + "(%s) MI4xx event log page fetch failed at %s: %s", + parent, + page_path, + page.error, + ) + break + + page_members = page.data.get(RF_MEMBERS) or [] + if not page_members: + break + + merged_members.extend(page_members) + next_link = page.data.get(RF_MEMBERS_NEXT_LINK) + pages_fetched += 1 + + if reported_total is not None and len(merged_members) >= reported_total: + break + + merged_data[RF_MEMBERS] = merged_members + if reported_total is not None: + merged_data[RF_MEMBERS_COUNT] = reported_total + else: + merged_data[RF_MEMBERS_COUNT] = len(merged_members) + merged_data.pop(RF_MEMBERS_NEXT_LINK, None) + + if reported_total is not None and len(merged_members) < reported_total: + collector.logger.warning( + "(%s) MI4xx event log pagination incomplete: collected %d of %d reported member(s) across %d page(s)", + parent, + len(merged_members), + reported_total, + pages_fetched, + ) + elif pages_fetched > 1: + collector.logger.info( + "(%s) MI4xx event log pagination merged %d member(s) across %d page(s)", + parent, + len(merged_members), + pages_fetched, + ) + + return RedfishGetResult( + path=first.path, + success=True, + data=merged_data, + status_code=last_status, + ) diff --git a/nodescraper/plugins/serviceability/mi4xx/serviceability_plugin_mi4xx.py b/nodescraper/plugins/serviceability/mi4xx/serviceability_plugin_mi4xx.py new file mode 100644 index 00000000..4341f5d4 --- /dev/null +++ b/nodescraper/plugins/serviceability/mi4xx/serviceability_plugin_mi4xx.py @@ -0,0 +1,235 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from typing import Annotated, Any, Optional, Union + +from pydantic import Field + +from nodescraper.enums import EventPriority, ExecutionStatus, SystemInteractionLevel +from nodescraper.models import DataPluginResult, PluginResult, TaskResult +from nodescraper.plugins.serviceability.afid_sag_lookup import log_afid_fru_summary +from nodescraper.plugins.serviceability.serviceability_data import ( + ServiceabilityDataLoadError, + ServiceabilityDataModel, +) +from nodescraper.plugins.serviceability.serviceability_plugin_base import ( + ServiceabilityPluginBase, +) +from nodescraper.utils import register_log_dir_name + +from .mi4xx_analyzer import MI4XXAnalyzer +from .mi4xx_analyzer_args import Mi4xxServiceabilityAnalyzerArgs +from .mi4xx_collector import MI4XXCollector +from .mi4xx_collector_args import MI4XXCollectorArgs + +register_log_dir_name("Mi4xxServiceabilityPlugin", "serviceability_plugin_mi4xx") +register_log_dir_name("MI4XXCollector", "mi4xx_collector") +register_log_dir_name("MI4XXAnalyzer", "mi4xx_analyzer") + + +class Mi4xxServiceabilityPlugin(ServiceabilityPluginBase): + """MI4xx OOB Redfish serviceability via a registered hub entry point.""" + + DATA_MODEL = ServiceabilityDataModel + COLLECTOR = MI4XXCollector # type: ignore[assignment] + ANALYZER = MI4XXAnalyzer + COLLECTOR_ARGS = MI4XXCollectorArgs + ANALYZER_ARGS = Mi4xxServiceabilityAnalyzerArgs # type: ignore[assignment] + + @staticmethod + def _resolved_afid_sag_path( + collection_args: Optional[Union[MI4XXCollectorArgs, dict[str, Any]]], + analysis_args: Optional[Union[Mi4xxServiceabilityAnalyzerArgs, dict[str, Any]]], + ) -> Optional[str]: + if analysis_args is not None: + if isinstance(analysis_args, dict): + path = analysis_args.get("afid_sag_path") + if path and str(path).strip(): + return str(path).strip() + else: + return analysis_args.resolved_afid_sag_path() + if collection_args is not None: + if isinstance(collection_args, dict): + path = collection_args.get("afid_sag_path") + if path and str(path).strip(): + return str(path).strip() + else: + return collection_args.resolved_afid_sag_path_for_collection() + return None + + @staticmethod + def _merge_afid_sag_path( + afid_sag_path: Optional[str], + collection_args: Optional[Union[MI4XXCollectorArgs, dict[str, Any]]], + analysis_args: Optional[Union[Mi4xxServiceabilityAnalyzerArgs, dict[str, Any]]], + ) -> tuple[ + Optional[Union[MI4XXCollectorArgs, dict[str, Any]]], + Optional[Union[Mi4xxServiceabilityAnalyzerArgs, dict[str, Any]]], + ]: + if afid_sag_path is None or not str(afid_sag_path).strip(): + return collection_args, analysis_args + + path = str(afid_sag_path).strip() + if analysis_args is None: + analysis_args = {"afid_sag_path": path} + elif isinstance(analysis_args, dict): + merged_analysis = dict(analysis_args) + merged_analysis["afid_sag_path"] = path + analysis_args = merged_analysis + else: + analysis_args = analysis_args.model_copy(update={"afid_sag_path": path}) + + if collection_args is None: + collection_args = {"afid_sag_path": path} + elif isinstance(collection_args, dict): + merged_collection = dict(collection_args) + merged_collection["afid_sag_path"] = path + collection_args = merged_collection + else: + collection_args = collection_args.model_copy(update={"afid_sag_path": path}) + + return collection_args, analysis_args + + @staticmethod + def _merge_mi4xx_collection_args( + collection_args: Optional[Union[MI4XXCollectorArgs, dict[str, Any]]], + analysis_args: Optional[Union[Mi4xxServiceabilityAnalyzerArgs, dict[str, Any]]], + ) -> Optional[Union[MI4XXCollectorArgs, dict[str, Any]]]: + if analysis_args is None: + return collection_args + if isinstance(analysis_args, dict): + uri = analysis_args.get("rf_event_log_uri") + else: + uri = analysis_args.rf_event_log_uri + if not uri or not str(uri).strip(): + return collection_args + uri_text = str(uri).strip() + if collection_args is None: + return {"rf_event_log_uri": uri_text} + if isinstance(collection_args, dict): + if not str(collection_args.get("rf_event_log_uri") or "").strip(): + merged = dict(collection_args) + merged["rf_event_log_uri"] = uri_text + return merged + return collection_args + if "rf_event_log_uri" not in collection_args.model_fields_set: + return collection_args.model_copy(update={"rf_event_log_uri": uri_text}) + return collection_args + + def _plugin_error_result(self, message: str) -> PluginResult: + self.logger.error("(%s) %s", self.__class__.__name__, message) + return PluginResult( + status=ExecutionStatus.ERROR, + source=self.__class__.__name__, + message=message, + result_data=DataPluginResult( + collection_result=TaskResult( + status=ExecutionStatus.NOT_RAN, + parent=self.__class__.__name__, + message="Data collection skipped", + ), + analysis_result=TaskResult( + status=ExecutionStatus.ERROR, + parent=self.__class__.__name__, + message=message, + ), + ), + ) + + def run( # type: ignore[override] + self, + collection: Annotated[ + bool, + "Run the collector (True) or skip it (False).", + ] = True, + analysis: Annotated[ + bool, + "Run the analyzer (True) or skip it (False).", + ] = True, + max_event_priority_level: Union[EventPriority, str] = EventPriority.CRITICAL, + system_interaction_level: Annotated[ + Union[SystemInteractionLevel, str], + "System interaction level (e.g. PASSIVE, INTERACTIVE, DISRUPTIVE).", + ] = SystemInteractionLevel.INTERACTIVE, + preserve_connection: bool = False, + data: Annotated[ + Optional[Union[str, dict, ServiceabilityDataModel]], + Field( + description=( + "Path to pre-collected redfish_responses.json, Redfish Entries JSON, " + "or ServiceabilityDataModel JSON; use with --collection False to analyze offline." + ), + ), + ] = None, + afid_sag_path: Annotated[ + Optional[str], + Field(description="Path to AFID_SAG.json for hub analysis and FRU summary."), + ] = None, + collection_args: Optional[Union[MI4XXCollectorArgs, dict[str, Any]]] = None, + analysis_args: Optional[Union[Mi4xxServiceabilityAnalyzerArgs, dict[str, Any]]] = None, + ) -> PluginResult: + collection_args, analysis_args = self._merge_afid_sag_path( + afid_sag_path, + collection_args, + analysis_args, + ) + + if analysis and not collection and data is not None: + try: + loaded = ( + data + if isinstance(data, ServiceabilityDataModel) + else self.DATA_MODEL.import_model(data) + ) + except ServiceabilityDataLoadError as exc: + return self._plugin_error_result(str(exc)) + member_count = len(loaded.rf_events) + self.logger.info( + "(%s) Loaded %d event log member(s) from --data (collection skipped)", + self.__class__.__name__, + member_count, + ) + log_afid_fru_summary( + self.logger, + self.__class__.__name__, + loaded, + self._resolved_afid_sag_path(collection_args, analysis_args), + ) + sag_path = self._resolved_afid_sag_path(collection_args, analysis_args) + if sag_path: + loaded.afid_sag_path = sag_path + + return super().run( + collection=collection, + analysis=analysis, + max_event_priority_level=max_event_priority_level, + system_interaction_level=system_interaction_level, + preserve_connection=preserve_connection, + data=data, + collection_args=self._merge_mi4xx_collection_args(collection_args, analysis_args), + analysis_args=analysis_args, + ) diff --git a/nodescraper/plugins/serviceability/se_adapter.py b/nodescraper/plugins/serviceability/se_adapter.py index 3db9394d..56b0f3b1 100644 --- a/nodescraper/plugins/serviceability/se_adapter.py +++ b/nodescraper/plugins/serviceability/se_adapter.py @@ -30,7 +30,18 @@ from collections import defaultdict from typing import Any, Dict, List, Optional, Tuple -from .se_models import AfidEvent, ServiceabilityBlock, ServiceabilitySolution +from .afid_sag_lookup import ( + afid_summary_from_sag, + load_afid_sag_data, + service_action_label_from_sag, + service_action_step_descriptions_from_sag, +) +from .se_models import ( + AfidEvent, + HubTriageResult, + ServiceabilityBlock, + ServiceabilitySolution, +) # Hub payload keys commonly holding a one-line human summary (not raw OEM metadata). _SUMMARY_VALUE_KEYS: Tuple[str, ...] = ( @@ -157,7 +168,9 @@ def _maybe_unwrap_outer_unit_map(d: dict[str, Any]) -> dict[str, Any]: return d -def _merged_short_service_lines_from_unit_messages(entries: List[Tuple[str, str]]) -> List[str]: +def _merged_short_service_lines_from_unit_messages( + entries: List[Tuple[str, str]], +) -> List[str]: """Group (unit, message) rows by message; merge units when the message is identical.""" by_message: dict[str, list[str]] = defaultdict(list) for unit, msg in entries: @@ -232,30 +245,202 @@ def _format_short_service_info_for_block(raw: Any) -> Optional[str]: return text or None +def _load_afid_sag_data(path: Optional[str]) -> Optional[dict[str, Any]]: + return load_afid_sag_data(path) + + +def _afid_summary_from_sag(afid: int, sag: Optional[dict[str, Any]]) -> Optional[str]: + return afid_summary_from_sag(afid, sag) + + +def _service_action_label_from_sag( + service_action_num: int, + sag: Optional[dict[str, Any]], +) -> Optional[str]: + return service_action_label_from_sag(service_action_num, sag) + + +def _service_action_entry_from_sag( + service_action_num: int, + sag: Optional[dict[str, Any]], +) -> Optional[dict[str, Any]]: + from .afid_sag_lookup import service_action_entry_from_sag + + return service_action_entry_from_sag(service_action_num, sag) + + +def _optional_int(value: Any) -> Optional[int]: + if value is None: + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _entry_point_result_rows(hub_result: dict[str, Any]) -> list[Any]: + """Extract triage rows from the entry-point hub analyze response.""" + results = hub_result.get("results") + return results if isinstance(results, list) else [] + + +def _service_action_title_from_row(row: dict[str, Any]) -> Optional[str]: + title = row.get("service_action_title") + if title is not None: + text = str(title).strip() + if text: + return text + sa = row.get("service_action") + if isinstance(sa, dict): + inner = sa.get("title") or sa.get("text") or sa.get("name") + if inner is not None: + text = str(inner).strip() + if text: + return text + elif sa is not None: + text = str(sa).strip() + if text: + return text + return None + + +def _hub_triage_result_from_row( + row: dict[str, Any], + sag: Optional[dict[str, Any]], +) -> Optional[HubTriageResult]: + afid_raw = row.get("afid_num", row.get("afid")) + san_raw = row.get("service_action_num") + location = row.get("location") or row.get("serviceable_unit") + if afid_raw is None or san_raw is None or location is None: + return None + try: + afid = int(afid_raw) + san = int(san_raw) + except (TypeError, ValueError): + return None + location_text = str(location).strip() + if not location_text: + return None + sa_entry = _service_action_entry_from_sag(san, sag) + title = _service_action_title_from_row(row) + if not title: + title = _service_action_label_from_sag(san, sag) + category = None + sa_severity = None + if sa_entry: + raw_cat = sa_entry.get("category") + if raw_cat is not None and str(raw_cat).strip(): + category = str(raw_cat).strip() + sa_severity = _optional_int(sa_entry.get("severity")) + tier_label = row.get("tier_label") + if tier_label is not None: + tier_label = str(tier_label).strip() or None + if tier_label is None and row.get("tier") is not None: + tier_label = str(row.get("tier")) + fru = row.get("fru") + if fru is not None: + fru = str(fru).strip() or None + return HubTriageResult( + afid=afid, + location=location_text, + count=max(1, _optional_int(row.get("count")) or 1), + service_action_num=san, + tier=_optional_int(row.get("tier")), + tier_label=tier_label, + fru=fru, + fru_rank=_optional_int(row.get("fru_rank")), + priority=_optional_int(row.get("priority")), + sa_severity=_optional_int(row.get("sa_severity")), + hub_sort_priority=_optional_int( + row.get("se_sort_priority") or row.get("hub_sort_priority") + ), + multi_mask=_optional_int(row.get("multi_mask")), + service_action_title=title, + service_action_category=category, + service_action_severity=sa_severity, + service_action_steps=service_action_step_descriptions_from_sag(san, sag), + afid_summary=_afid_summary_from_sag(afid, sag), + ) + + +def _format_hub_triage_result_lines(index: int, row: HubTriageResult) -> list[str]: + lines = [f"[{index}] AFID {row.afid} @ {row.location} (count={row.count})"] + if row.afid_summary: + lines.append(f" fault: {row.afid_summary}") + fru_bits = [] + if row.fru: + fru_bits.append(row.fru) + if row.fru_rank is not None: + fru_bits.append(f"rank {row.fru_rank}") + if fru_bits: + lines.append(f" FRU: {' '.join(fru_bits)}") + meta_bits = [] + if row.priority is not None: + meta_bits.append(f"priority={row.priority}") + if row.sa_severity is not None: + meta_bits.append(f"SA severity={row.sa_severity}") + if row.tier_label: + tier_bit = f"tier={row.tier_label}" + if row.tier is not None: + tier_bit = f"{tier_bit} ({row.tier})" + meta_bits.append(tier_bit) + if row.hub_sort_priority is not None: + meta_bits.append(f"sort=0x{row.hub_sort_priority:08x}") + if row.multi_mask is not None: + meta_bits.append(f"multi_mask={row.multi_mask}") + if meta_bits: + lines.append(f" {'; '.join(meta_bits)}") + action_bits = [f"service action {row.service_action_num}"] + if row.service_action_title: + action_bits.append(f'"{row.service_action_title}"') + if row.service_action_category: + action_bits.append(f"[{row.service_action_category}]") + if row.service_action_severity is not None: + action_bits.append(f"SA severity={row.service_action_severity}") + lines.append(f" {' '.join(action_bits)}") + for step_index, step in enumerate(row.service_action_steps): + lines.append(f" step {step_index}: {step}") + return lines + + +def _format_service_action_phrase(solution: ServiceabilitySolution) -> str: + title = (solution.service_action_title or "").strip() + tier = (solution.service_action_tier or "").strip() + base = f"service action {solution.service_action_num}" + if title: + base = f'{base}: "{title}"' + if tier: + base = f"{base} ({tier} tier)" + return base + + def format_serviceability_solution_lines(block: ServiceabilityBlock) -> list[str]: """Human-readable lines for logging or console output.""" lines: list[str] = [] - if block.short_service_info: - lines.append("short_service_info:") - for part in block.short_service_info.splitlines(): - lines.append(f" {part}" if part else " ") - lines.append("") if block.solution_reasoning: lines.append(block.solution_reasoning) if block.hub_version: lines.append(f"Hub version: {block.hub_version}") if block.afid_sag_file_version: lines.append(f"AFID_SAG file: {block.afid_sag_file_version}") + if block.hub_triage_results: + lines.append("Hub triage results:") + for index, row in enumerate(block.hub_triage_results, start=1): + lines.extend(_format_hub_triage_result_lines(index, row)) + return lines + if block.short_service_info: + lines.append("short_service_info:") + for part in block.short_service_info.splitlines(): + lines.append(f" {part}" if part else " ") + lines.append("") if not block.solution: lines.append("No service actions recommended.") return lines for index, solution in enumerate(block.solution, start=1): units = ", ".join(solution.serviceable_unit) - title = (solution.service_action_title or "").strip() - action = f"service action {solution.service_action_num}" - if title: - action = f"{action} ({title})" - lines.append(f"[{index}] AFID {solution.afid}, {action}, units: [{units}]") + summary = f" ({solution.afid_summary})" if solution.afid_summary else "" + action = _format_service_action_phrase(solution) + lines.append(f"[{index}] AFID {solution.afid}{summary}: {action}, units: [{units}]") return lines @@ -336,3 +521,113 @@ def _action_title(info: dict[str, Any]) -> str: afid_sag_metadata=meta_out, short_service_info=short_service_info, ) + + +def serviceability_block_from_entry_point_hub( + afid_events: list[AfidEvent], + hub_result: dict[str, Any], + *, + hub_label: str = "Service hub", + rf_event_count: int = 0, + afid_sag_path: Optional[str] = None, +) -> ServiceabilityBlock: + """Build a ServiceabilityBlock from a registered entry-point hub analyze() response.""" + hub_name = str(hub_result.get("engine") or hub_label) + hub_version_raw = hub_result.get("engine_version") + results = _entry_point_result_rows(hub_result) + sag = _load_afid_sag_data(afid_sag_path) + + grouped: dict[tuple[int, int], list[str]] = defaultdict(list) + titles: dict[tuple[int, int], str] = {} + tiers: dict[tuple[int, int], str] = {} + summaries: dict[int, str] = {} + + for row in results: + if not isinstance(row, dict): + continue + afid_raw = row.get("afid_num", row.get("afid")) + san_raw = row.get("service_action_num") + location = row.get("location") or row.get("serviceable_unit") + if afid_raw is None or san_raw is None: + continue + try: + afid = int(afid_raw) + san = int(san_raw) + except (TypeError, ValueError): + continue + unit = str(location).strip() if location is not None else "" + key = (afid, san) + if unit and unit not in grouped[key]: + grouped[key].append(unit) + row_title = _service_action_title_from_row(row) + if row_title is not None and key not in titles: + titles[key] = row_title + tier_label = row.get("tier_label") or row.get("tier") + if tier_label is not None and key not in tiers: + text = str(tier_label).strip() + if text: + tiers[key] = text + if afid not in summaries: + summary = _afid_summary_from_sag(afid, sag) + if summary: + summaries[afid] = summary + + solutions = [] + for (afid, san), units in sorted(grouped.items()): + title = titles.get((afid, san)) or _service_action_label_from_sag(san, sag) + if not title: + afid_map = (sag or {}).get("afid") if isinstance(sag, dict) else None + if isinstance(afid_map, dict): + afid_entry = afid_map.get(str(afid)) + if isinstance(afid_entry, dict): + fallback = afid_entry.get("service_action") + if fallback is not None and str(fallback).strip(): + title = str(fallback).strip() + solutions.append( + ServiceabilitySolution( + afid=afid, + serviceable_unit=units, + service_action_num=san, + service_action_title=title, + service_action_tier=tiers.get((afid, san)), + afid_summary=summaries.get(afid), + ) + ) + + hub_version = str(hub_version_raw).strip() if hub_version_raw else None + reasoning = ( + f"{hub_name}: {len(solutions)} recommendation(s) from " + f"{rf_event_count} Redfish event(s), {len(results)} triage row(s)." + ) + triage_results: list[HubTriageResult] = [] + for row in results: + if not isinstance(row, dict): + continue + parsed = _hub_triage_result_from_row(row, sag) + if parsed is not None: + triage_results.append(parsed) + sag_metadata = None + afid_sag_file_version = None + if sag: + sag_metadata = { + "pid": sag.get("pid"), + "revision": sag.get("revision"), + "variant": sag.get("variant"), + } + afid_sag_file_version = _afid_sag_file_version_display(sag_metadata) + elif hub_result.get("pid") or hub_result.get("revision"): + sag_metadata = { + "pid": hub_result.get("pid"), + "revision": hub_result.get("revision"), + } + afid_sag_file_version = _afid_sag_file_version_display(sag_metadata) + + return ServiceabilityBlock( + afid_events=list(afid_events), + solution=solutions, + solution_reasoning=reasoning, + hub_version=hub_version, + afid_sag_file_version=afid_sag_file_version, + afid_sag_metadata=sag_metadata, + hub_triage_results=triage_results, + ) diff --git a/nodescraper/plugins/serviceability/se_models.py b/nodescraper/plugins/serviceability/se_models.py index addef3ae..94baa835 100644 --- a/nodescraper/plugins/serviceability/se_models.py +++ b/nodescraper/plugins/serviceability/se_models.py @@ -38,7 +38,7 @@ class AfidEvent(BaseModel): description="Unit label (e.g. gpu02); standardized per platform.", ) time: str = Field( - description="First-occurrence timestamp (SE format, e.g. 2026-05-07 12:50:42.096-07:00).", + description="First-occurrence timestamp (hub wire format, e.g. 2026-05-07 12:50:42.096-07:00).", ) @field_validator("serviceable_unit") @@ -62,12 +62,42 @@ class ServiceabilitySolution(BaseModel): ) service_action_title: Optional[str] = Field( default=None, - description=("Short service action label from the hub."), + description=("Short service action label from the hub or AFID_SAG.json."), ) + service_action_tier: Optional[str] = Field( + default=None, + description="Service action tier label from the hub (e.g. Secondary).", + ) + afid_summary: Optional[str] = Field( + default=None, + description="Human-readable AFID fault summary from AFID_SAG.json when available.", + ) + + +class HubTriageResult(BaseModel): + """One service hub triage row with SAG-enriched action details.""" + + afid: int + location: str + count: int = 1 + service_action_num: int + tier: Optional[int] = None + tier_label: Optional[str] = None + fru: Optional[str] = None + fru_rank: Optional[int] = None + priority: Optional[int] = None + sa_severity: Optional[int] = None + hub_sort_priority: Optional[int] = None + multi_mask: Optional[int] = None + service_action_title: Optional[str] = None + service_action_category: Optional[str] = None + service_action_severity: Optional[int] = None + service_action_steps: List[str] = Field(default_factory=list) + afid_summary: Optional[str] = None class ServiceabilityBlock(BaseModel): - """ANC-style serviceability section: SE input, output, and optional reasoning.""" + """Serviceability section with hub input, output, and optional reasoning.""" afid_events: List[AfidEvent] = Field( default_factory=list, @@ -100,3 +130,7 @@ class ServiceabilityBlock(BaseModel): "per-unit dict payloads are collapsed, identical messages merged with unit lists)." ), ) + hub_triage_results: List[HubTriageResult] = Field( + default_factory=list, + description="Full service hub triage rows with SAG-enriched action details.", + ) diff --git a/nodescraper/plugins/serviceability/se_runner.py b/nodescraper/plugins/serviceability/se_runner.py index 6ff8b60e..c30858b5 100644 --- a/nodescraper/plugins/serviceability/se_runner.py +++ b/nodescraper/plugins/serviceability/se_runner.py @@ -23,17 +23,31 @@ # SOFTWARE. # ############################################################################### -"""Invoke a configured Python service hub against collected Redfish events.""" +"""Invoke a configured Python or entry-point service hub against collected Redfish events.""" + from __future__ import annotations +import dataclasses import importlib +import importlib.metadata import inspect from pathlib import Path -from typing import Any, Callable, Optional, Type +from typing import Any, Callable, Optional, Protocol, Type, Union, cast -from .se_adapter import serviceability_block_from_service_result +from .afid_sag_paths import validate_afid_sag_path +from .se_adapter import ( + serviceability_block_from_entry_point_hub, + serviceability_block_from_service_result, +) from .se_models import AfidEvent, ServiceabilityBlock +HUB_ENTRY_POINT_GROUP = "amd.serviceability_engines" + + +def normalize_hub_entry_point(hub_name: str) -> str: + """Strip whitespace from a configured hub entry point name.""" + return str(hub_name).strip() + def _signature_accepts_var_keyword(sig: inspect.Signature) -> bool: return any(p.kind == inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()) @@ -83,7 +97,7 @@ def _call_hub_analyze( return analyze(list(rf_events), **kw) -class SeRunError(RuntimeError): +class HubRunError(RuntimeError): """Raised when the service hub fails or returns invalid output.""" @@ -109,10 +123,10 @@ def run_service_hub( """ sag_path = Path(afid_sag_path) if not sag_path.is_file(): - raise SeRunError(f"Hub config file not found: {afid_sag_path}") + raise HubRunError(f"Hub config file not found: {afid_sag_path}") if not rf_events: - raise SeRunError( + raise HubRunError( "Collected Redfish events are required; re-run collection or use skip_hub." ) @@ -120,7 +134,7 @@ def run_service_hub( try: mod = importlib.import_module(hub_python_module) except ImportError as exc: - raise SeRunError(f"Cannot import {hub_python_module}: {exc}") from exc + raise HubRunError(f"Cannot import {hub_python_module}: {exc}") from exc hub_cls = _resolve_hub_class(mod, hub_analyze_method) @@ -139,7 +153,7 @@ def run_service_hub( hub_options, ) except Exception as exc: - raise SeRunError(f"{label} {hub_analyze_method}() failed: {exc}") from exc + raise HubRunError(f"{label} {hub_analyze_method}() failed: {exc}") from exc if result is None: return ServiceabilityBlock( @@ -186,9 +200,291 @@ def add_candidate(obj: Any) -> None: if len(candidates) == 1: return candidates[0] if not candidates: - raise SeRunError( + raise HubRunError( f"No class with {analyze_method}() found in {package}; " "check hub_python_module and hub_analyze_method in analysis_args." ) names = ", ".join(cls.__name__ for cls in candidates) - raise SeRunError(f"Multiple classes with {analyze_method}() in {package}: {names}.") + raise HubRunError(f"Multiple classes with {analyze_method}() in {package}: {names}.") + + +class EntryPointHubHook(Protocol): + name: str + + def analyze( + self, + events: Union[dict[str, Any], list[Any]], + afid_sag_path: str, + ) -> Any: ... + + +def _entry_points_for_group(group: str): + try: + return importlib.metadata.entry_points(group=group) # type: ignore[call-arg] + except TypeError: + all_eps = importlib.metadata.entry_points() # type: ignore[assignment] + return all_eps.get(group, []) # type: ignore[attr-defined] + + +def list_hub_entry_point_names() -> list[str]: + """Return registered hub entry point names.""" + return sorted({ep.name for ep in _entry_points_for_group(HUB_ENTRY_POINT_GROUP)}) + + +def load_hub_from_entry_point(hub_name: str) -> EntryPointHubHook: + """Load and instantiate a service hub from a registered entry point.""" + wanted = normalize_hub_entry_point(hub_name) + if not wanted: + raise HubRunError("hub_entry_point must be non-empty") + + matches = [ep for ep in _entry_points_for_group(HUB_ENTRY_POINT_GROUP) if ep.name == wanted] + if not matches: + available = ", ".join(list_hub_entry_point_names()) or "(none installed)" + raise HubRunError( + f"Service hub {wanted!r} not found among registered hub entry points; " + f"available: {available}. Install the package that registers this hub entry point." + ) + + try: + loaded = matches[0].load() + except Exception as exc: # noqa: BLE001 + raise HubRunError(f"Failed to load service hub {wanted!r}: {exc}") from exc + + if inspect.isclass(loaded): + try: + return cast(EntryPointHubHook, loaded()) + except Exception as exc: # noqa: BLE001 + raise HubRunError(f"Failed to instantiate service hub {wanted!r}: {exc}") from exc + return cast(EntryPointHubHook, loaded) + + +def afid_events_to_entry_point_payload(events: list[AfidEvent]) -> list[dict[str, Any]]: + """Convert AfidEvent models to CLI-shaped rows for the hub afid_events fallback path.""" + from collections import defaultdict + + counts: dict[tuple[int, str], int] = defaultdict(int) + for event in events: + counts[(event.afid, event.serviceable_unit)] += 1 + return [ + { + "afid": afid, + "location": unit, + "count": count, + } + for (afid, unit), count in sorted(counts.items()) + ] + + +def events_payload_for_hub_analyze( + *, + rf_events: Optional[list[Any]] = None, + afid_events_payload: Optional[list[Any]] = None, +) -> list[Any]: + """Return the event list passed to entry-point hub analyze().""" + if rf_events is not None: + if not isinstance(rf_events, list) or not rf_events: + raise HubRunError("rf_events must be a non-empty list") + return list(rf_events) + if afid_events_payload is not None: + if not isinstance(afid_events_payload, list) or not afid_events_payload: + raise HubRunError("afid_events must be a non-empty list") + return list(afid_events_payload) + raise HubRunError("rf_events or afid_events is required") + + +def resolve_hub_events_payload( + *, + afid_events: list[AfidEvent], + rf_events: Optional[list[Any]] = None, + prefer_rf_events: bool = True, +) -> list[Any]: + """Choose rf_events or aggregated afid_events for hub analyze().""" + if prefer_rf_events and rf_events: + return events_payload_for_hub_analyze(rf_events=rf_events) + return events_payload_for_hub_analyze( + afid_events_payload=afid_events_to_entry_point_payload(afid_events) + ) + + +def _hub_engine_version(hub: Any) -> str: + version = getattr(hub, "version", None) + if callable(version): + version = version() + if isinstance(version, str) and version.strip(): + return version.strip() + return "unknown" + + +def _tier_grouped_from_rows(rows: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: + grouped: dict[str, list[dict[str, Any]]] = {} + for row in rows: + tier_label = row.get("tier_label") + if tier_label is None: + continue + key = str(tier_label) + grouped.setdefault(key, []).append(row) + return grouped + + +def _hub_triage_error_message(triage: Any) -> Optional[str]: + error_obj = getattr(triage, "error", None) + if error_obj is None: + return None + message = getattr(error_obj, "message", None) + if message is None: + return None + text = str(message).strip() + return text or None + + +def _resolved_entry_to_hub_row(entry: Any) -> dict[str, Any]: + row = dataclasses.asdict(entry) + if "afid" in row and "afid_num" not in row: + row["afid_num"] = row["afid"] + return row + + +def entry_point_hub_result_from_triage( + triage: Any, + *, + engine_name: str, + engine_version: str, +) -> dict[str, Any]: + """Map a hub analyze result object into the dict consumed by se_adapter.""" + if isinstance(triage, dict) or not hasattr(triage, "status"): + raise HubRunError( + f"Expected entry-point hub analyze result object, got {type(triage).__name__}" + ) + + base: dict[str, Any] = { + "schema_version": getattr(triage, "schema_version", "1.0"), + "engine": engine_name, + "engine_version": engine_version, + "pid": getattr(triage, "pid", None), + "revision": getattr(triage, "revision", None), + } + status = str(getattr(triage, "status", "") or "") + error_message = _hub_triage_error_message(triage) + if status == "error" or error_message is not None: + return { + **base, + "status": "error", + "error": {"message": error_message or "Service hub analyze failed"}, + "results": [], + "tier_grouped": {}, + } + + triage_section = getattr(triage, "triage", None) + result_entries = ( + getattr(triage_section, "results", None) if triage_section is not None else None + ) + results = ( + [_resolved_entry_to_hub_row(entry) for entry in result_entries] + if isinstance(result_entries, list) + else [] + ) + return { + **base, + "status": "ok", + "error": None, + "results": results, + "tier_grouped": _tier_grouped_from_rows(results), + } + + +def _synthetic_error_hub_result( + message: str, + *, + engine_name: str, + engine_version: str, +) -> dict[str, Any]: + return { + "schema_version": "1.0", + "status": "error", + "error": {"message": message}, + "engine": engine_name, + "engine_version": engine_version, + "results": [], + "tier_grouped": {}, + } + + +def _entry_point_analyze_error(hub_result: dict[str, Any]) -> Optional[str]: + """Return an error message when analyze failed, or None on success.""" + status = hub_result.get("status") + if status == "ok": + return None + if status == "error": + error_obj = hub_result.get("error") or {} + if isinstance(error_obj, dict): + message = error_obj.get("message") + if message: + return str(message) + return str(error_obj or "Service hub analyze failed") + if status is None and isinstance(hub_result.get("results"), list): + return None + if status not in (None, "ok"): + return f"Service hub analyze failed (status={status!r})" + return "Service hub analyze failed" + + +def run_entry_point_hub( + *, + hub_entry_point: str, + hub_display_name: Optional[str] = None, + afid_events: list[AfidEvent], + afid_sag_path: str, + rf_events: Optional[list[Any]] = None, + rf_event_count: int = 0, + raise_on_error: bool = False, + prefer_rf_events: bool = True, +) -> ServiceabilityBlock: + """Run a registered entry-point service hub and return a :class:`ServiceabilityBlock`.""" + if not afid_events and not (prefer_rf_events and rf_events): + raise HubRunError("No AFID events to analyze") + + validate_afid_sag_path(afid_sag_path) + label = hub_display_name or normalize_hub_entry_point(hub_entry_point) + hub = load_hub_from_entry_point(hub_entry_point) + hub_label = getattr(hub, "name", label) + engine_version = _hub_engine_version(hub) + events = resolve_hub_events_payload( + afid_events=afid_events, + rf_events=rf_events, + prefer_rf_events=prefer_rf_events, + ) + try: + raw_result = hub.analyze(events, afid_sag_path) + except Exception as exc: # noqa: BLE001 + if raise_on_error: + raise HubRunError(f"Service hub {hub_label!r} analyze failed: {exc}") from exc + raw_result = _synthetic_error_hub_result( + str(exc), + engine_name=hub_label, + engine_version=engine_version, + ) + + if isinstance(raw_result, dict): + hub_result = raw_result + else: + hub_result = entry_point_hub_result_from_triage( + raw_result, + engine_name=hub_label, + engine_version=engine_version, + ) + + analyze_error = _entry_point_analyze_error(hub_result) + if analyze_error is not None: + raise HubRunError(analyze_error) + + event_count = rf_event_count + return serviceability_block_from_entry_point_hub( + afid_events, + hub_result, + hub_label=label, + rf_event_count=event_count, + afid_sag_path=afid_sag_path, + ) + + +SeRunError = HubRunError # backward-compatible alias diff --git a/nodescraper/plugins/serviceability/serviceability_collector.py b/nodescraper/plugins/serviceability/serviceability_collector.py index 0ad28643..b0ed8da5 100644 --- a/nodescraper/plugins/serviceability/serviceability_collector.py +++ b/nodescraper/plugins/serviceability/serviceability_collector.py @@ -218,8 +218,16 @@ def collect_data( ) self.result.status = ExecutionStatus.OK self.result.message = f"Collected {len(members)} event log member(s)" + self._after_collect_data(data, svc_args) return self.result, data + def _after_collect_data( + self, + data: ServiceabilityDataModel, + args: TServiceabilityCollectArg, + ) -> None: + """Optional hook for subclasses after successful event log collection.""" + def _fetch_component_details( self, responses: dict[str, Any], args: TServiceabilityCollectArg ) -> tuple[Optional[str], Optional[str]]: diff --git a/nodescraper/plugins/serviceability/serviceability_data.py b/nodescraper/plugins/serviceability/serviceability_data.py index b275c579..69f81fab 100644 --- a/nodescraper/plugins/serviceability/serviceability_data.py +++ b/nodescraper/plugins/serviceability/serviceability_data.py @@ -26,16 +26,23 @@ from __future__ import annotations import json +import logging import os -from typing import Any, Dict, List, Optional +from pathlib import Path +from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel, Field from nodescraper.models import DataModel +from .event_log_utils import rf_events_from_json_payload from .se_models import AfidEvent, ServiceabilityBlock +class ServiceabilityDataLoadError(ValueError): + """Raised when a serviceability data path cannot be loaded.""" + + class DeviceInfo(BaseModel): """Chassis fields from Assembly parsing; extra vendor keys belong in oem_extensions.""" @@ -77,16 +84,62 @@ class ServiceabilityDataModel(DataModel): component_details: Optional[str] = None log_path: Optional[str] = None bmc_host: Optional[str] = None + afid_sag_path: Optional[str] = Field( + default=None, + description="Optional AFID_SAG.json path used for FRU summary CSV export.", + ) afid_events: List[AfidEvent] = Field( default_factory=list, description="Service Hub input; built during analysis when not pre-filled.", ) serviceability: Optional[ServiceabilityBlock] = Field( default=None, - description="ANC-style serviceability block (SE input + output).", + description="Serviceability block populated by hub analysis.", ) result: Optional[ServiceabilityResult] = None + @classmethod + def import_model( + cls, + model_input: Union[dict, str], + ) -> ServiceabilityDataModel: + """Load from a file path, ServiceabilityDataModel dict, Redfish Entries JSON, or redfish_responses.json dump.""" + if isinstance(model_input, str): + path = Path(model_input).expanduser() + if not path.is_file(): + raise ServiceabilityDataLoadError( + f"Serviceability data file not found: {path.resolve()}" + ) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ServiceabilityDataLoadError( + f"Invalid JSON in serviceability data file {path.resolve()}: {exc}" + ) from exc + return cls._import_from_payload(payload) + + if isinstance(model_input, dict): + return cls._import_from_payload(model_input) + + return super().import_model(model_input) + + @classmethod + def _import_from_payload(cls, payload: Any) -> ServiceabilityDataModel: + if isinstance(payload, dict) and "rf_events" in payload: + return cls.model_validate(payload) + try: + rf_events, responses = rf_events_from_json_payload(payload) + except ValueError as exc: + raise ServiceabilityDataLoadError(str(exc)) from exc + if isinstance(payload, dict): + merged = dict(payload) + merged["rf_events"] = rf_events + if responses and not merged.get("responses"): + merged["responses"] = responses + known = set(cls.model_fields) + return cls.model_validate({k: v for k, v in merged.items() if k in known}) + return cls(rf_events=rf_events, responses=responses) + def log_model(self, log_path: str) -> None: """Write collector artifacts and optional serviceability.json under log_path.""" os.makedirs(log_path, exist_ok=True) @@ -105,3 +158,10 @@ def log_model(self, log_path: str) -> None: f, indent=2, ) + from .afid_fru_csv import write_afid_fru_summary_csv + + write_afid_fru_summary_csv( + self, + log_path, + logger=logging.getLogger("nodescraper"), + ) diff --git a/nodescraper/plugins/serviceability/serviceability_hub_analyzer.py b/nodescraper/plugins/serviceability/serviceability_hub_analyzer.py new file mode 100644 index 00000000..ca2e5234 --- /dev/null +++ b/nodescraper/plugins/serviceability/serviceability_hub_analyzer.py @@ -0,0 +1,105 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from __future__ import annotations + +from typing import Any, ClassVar, Optional + +from pydantic import BaseModel, Field + +from nodescraper.enums import ExecutionStatus +from nodescraper.interfaces import DataAnalyzer +from nodescraper.models import TaskResult + +from .analysis_window import analyze_serviceability_window +from .analyzer_args import ServiceabilityAnalyzerArgs +from .se_adapter import format_serviceability_solution_lines +from .se_models import ServiceabilityBlock +from .serviceability_data import ServiceabilityDataModel + + +class AfidSagMetadataArtifact(BaseModel): + """Hub AFID_SAG metadata snapshot; written to afid_sag_metadata.json.""" + + ARTIFACT_LOG_BASENAME: ClassVar[str] = "afid_sag_metadata" + + metadata: dict[str, Any] = Field(default_factory=dict) + + +class ServiceabilityHubAnalyzer( + DataAnalyzer[ServiceabilityDataModel, ServiceabilityAnalyzerArgs], +): + """Build AFID events from collected data and run the configured service hub.""" + + DATA_MODEL = ServiceabilityDataModel + + DOCUMENTATION_ANALYSIS_ITEMS: tuple[str, ...] = ( + "Builds AFID events from collected Redfish event log members (and optional assembly metadata).", + "Optionally decodes CPER attachments via analysis_args.cper_decode_module before hub analysis.", + "Runs the configured service hub (hub_python_module or hub_entry_point) to produce service recommendations.", + "When analysis_args.skip_hub is true, only builds AFID events without running the hub.", + ) + + def analyze_data( + self, + data: ServiceabilityDataModel, + args: Optional[ServiceabilityAnalyzerArgs] = None, + ) -> TaskResult: + if args is None: + self.result.status = ExecutionStatus.NOT_RAN + self.result.message = "ServiceabilityAnalyzerArgs are required" + return self.result + + parent = self.parent or self.__class__.__name__ + result = analyze_serviceability_window( + data, + args, + logger=self.logger, + parent=parent, + ) + if not result.ok: + self.result.status = ExecutionStatus.ERROR + self.result.message = result.message + return self.result + + if result.serviceability is not None: + self._append_hub_artifacts(result.serviceability) + self._log_serviceability_solutions(result.serviceability) + + self.result.status = ExecutionStatus.OK + self.result.message = result.message + return self.result + + def _append_hub_artifacts(self, block: ServiceabilityBlock) -> None: + if block.afid_sag_metadata is None: + return + self.result.artifacts.append( + AfidSagMetadataArtifact(metadata=dict(block.afid_sag_metadata)) + ) + + def _log_serviceability_solutions(self, block: ServiceabilityBlock) -> None: + parent = self.parent or self.__class__.__name__ + for line in format_serviceability_solution_lines(block): + self.logger.info("(%s) %s", parent, line) diff --git a/plugin_config_mi4xx_example.json b/plugin_config_mi4xx_example.json new file mode 100644 index 00000000..fa150b45 --- /dev/null +++ b/plugin_config_mi4xx_example.json @@ -0,0 +1,22 @@ +{ + "name": "Mi4xxServiceability", + "desc": "Helios MI4xx — Instinct accelerator event log + configured service hub entry point", + "global_args": {}, + "plugins": { + "Mi4xxServiceabilityPlugin": { + "collection": true, + "analysis": true, + "collection_args": { + "rf_event_log_uri": "/redfish/v1/Systems/Instinct_Accelerators/LogServices/EventLog/Entries", + "follow_next_link": true, + "max_pages": 200 + }, + "analysis_args": { + "afid_sag_path": "/opt/amd/afid/AFID_SAG.json", + "hub_entry_point": "hub", + "rf_event_log_uri": "/redfish/v1/Systems/Instinct_Accelerators/LogServices/EventLog/Entries" + } + } + }, + "result_collators": {} +} diff --git a/test/unit/cli/test_cli_embed_api.py b/test/unit/cli/test_cli_embed_api.py index 54b95043..14b04799 100644 --- a/test/unit/cli/test_cli_embed_api.py +++ b/test/unit/cli/test_cli_embed_api.py @@ -29,11 +29,8 @@ import pytest from nodescraper.cli.cli import get_cli_top_level_subcommands -from nodescraper.cli.embed import ( - CLI_TOP_LEVEL_SUBCOMMANDS, - run_cli_return_code, - run_main_return_code, -) +from nodescraper.cli.constants import KEYBOARD_INTERRUPT_EXIT_CODE +from nodescraper.cli.embed import run_cli_return_code, run_main_return_code def test_get_cli_top_level_subcommands_matches_argparse_subparsers() -> None: @@ -44,10 +41,6 @@ def test_get_cli_top_level_subcommands_matches_argparse_subparsers() -> None: assert all(isinstance(s, str) for s in subs) -def test_cli_top_level_subcommands_lazy_alias_matches_getter() -> None: - assert CLI_TOP_LEVEL_SUBCOMMANDS == get_cli_top_level_subcommands() - - def test_run_cli_return_code_and_run_main_return_code_delegate( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -66,3 +59,19 @@ def fake_main( assert run_cli_return_code(["describe", "plugin", "X"]) == 7 assert run_main_return_code(["a", "b"]) == 7 assert calls == [["describe", "plugin", "X"], ["a", "b"]] + + +def test_main_exits_cleanly_on_keyboard_interrupt( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + import nodescraper.cli.cli as cli_mod + + def _raise_keyboard_interrupt(*_args, **_kwargs): + raise KeyboardInterrupt + + monkeypatch.setattr(cli_mod, "PluginRegistry", _raise_keyboard_interrupt) + with pytest.raises(SystemExit) as exc_info: + cli_mod.main([]) + assert exc_info.value.code == KEYBOARD_INTERRUPT_EXIT_CODE + assert capsys.readouterr().err == "Interrupted.\n" diff --git a/test/unit/plugin/fixtures/afid_sag_se_compat.json b/test/unit/plugin/fixtures/afid_sag_se_compat.json new file mode 100644 index 00000000..cf280d8f --- /dev/null +++ b/test/unit/plugin/fixtures/afid_sag_se_compat.json @@ -0,0 +1,57 @@ +{ + "pid": "dummy-sag-pid", + "revision": "dummy-rev-0", + "variant": "dummy-variant-0", + "schema_version": "1.0.0", + "serviceable_fru": [ + {"dummy_fru_primary": 1}, + {"dummy_fru_secondary": 5} + ], + "afid": { + "9001": { + "error_category": "DummyErrorCategory", + "error_type": "DummyErrorType", + "error_severity": "Critical", + "fru": "dummy_fru_primary", + "status": "active", + "method": "CPER", + "threshold": 1, + "priority": 20, + "service_action_num": 99, + "service_action": "Dummy service action", + "supported_systems": ["DUMMY_SYSTEM"] + }, + "9002": { + "error_category": "DummyErrorCategory", + "error_type": "DummyErrorTypeAlt", + "error_severity": "Warning", + "fru": "dummy_fru_secondary", + "status": "active", + "method": "CPER", + "threshold": 1, + "priority": 30, + "service_action_num": 88, + "service_action": "Dummy service action alt", + "supported_systems": ["DUMMY_SYSTEM"] + } + }, + "service_actions": { + "99": { + "title": "Dummy service action", + "category": "DummyCategory", + "severity": 20, + "steps": [ + { + "step_num": 0, + "description": "Dummy step." + } + ] + }, + "88": { + "title": "Dummy service action alt", + "category": "DummyCategory", + "severity": 10, + "steps": [] + } + } +} diff --git a/test/unit/plugin/test_mi4xx_plugin.py b/test/unit/plugin/test_mi4xx_plugin.py new file mode 100644 index 00000000..5b654e3f --- /dev/null +++ b/test/unit/plugin/test_mi4xx_plugin.py @@ -0,0 +1,489 @@ +############################################################################### +# +# MIT License +# +# Copyright (c) 2026 Advanced Micro Devices, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +############################################################################### +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +from serviceability_dummy_data import ( + DUMMY_AFID_A, + DUMMY_BMC_HOST, + DUMMY_EVENT_URI, + DUMMY_HUB_VERSION_ENTRY, + DUMMY_SERVICE_ACTION_NUM, + DUMMY_TIER_CRITICAL, + DUMMY_TIMESTAMP, + DUMMY_UNIT_A, +) + +from nodescraper.connection.redfish import ( + RF_MEMBERS, + RF_MEMBERS_COUNT, + RF_MEMBERS_NEXT_LINK, + RedfishGetResult, +) +from nodescraper.enums import ExecutionStatus +from nodescraper.plugins.serviceability import ( + AfidEvent, + MI4XXAnalyzer, + MI4XXCollector, + MI4XXCollectorArgs, + Mi4xxServiceabilityAnalyzerArgs, + Mi4xxServiceabilityPlugin, + ServiceabilityDataModel, + ServiceabilityPluginBase, + analyze_serviceability_window, + default_afid_sag_path, + load_hub_from_entry_point, + resolve_configured_afid_sag_path, + run_entry_point_hub, + serviceability_block_from_entry_point_hub, + validate_afid_sag_path, +) +from nodescraper.plugins.serviceability.mi4xx.mi4xx_event_log_paging import ( + fetch_mi4xx_event_log, +) +from nodescraper.plugins.serviceability.se_runner import HubRunError +from nodescraper.plugins.serviceability.serviceability_hub_analyzer import ( + AfidSagMetadataArtifact, + ServiceabilityHubAnalyzer, +) + + +def _analyze_decorator_depth(func): + depth = 0 + while hasattr(func, "__wrapped__"): + func = func.__wrapped__ + depth += 1 + return depth + + +def test_mi4xx_analyzer_analyze_data_wrapped_once(): + assert _analyze_decorator_depth(MI4XXAnalyzer.analyze_data) == 1 + assert MI4XXAnalyzer.analyze_data is ServiceabilityHubAnalyzer.analyze_data + + +class _FakeHub: + name = "ExampleHub" + + def analyze(self, events, afid_sag_path): + assert afid_sag_path + assert events + first = events[0] if isinstance(events, list) else events + if isinstance(first, dict) and ("Oem" in first or "Id" in first): + row = { + "afid": DUMMY_AFID_A, + "serviceable_unit": DUMMY_UNIT_A, + "count": 1, + "artifact": "redfish", + } + else: + row = { + "afid": first.get("afid", DUMMY_AFID_A), + "serviceable_unit": first.get("location") + or first.get("serviceable_unit", DUMMY_UNIT_A), + "count": first.get("count", 1), + "artifact": first.get("artifact", "cli"), + } + return { + "schema_version": "1.0", + "status": "ok", + "error": None, + "engine": self.name, + "engine_version": DUMMY_HUB_VERSION_ENTRY, + "results": [ + { + "afid_num": row["afid"], + "location": row.get("serviceable_unit") or row.get("location"), + "count": row.get("count", 1), + "artifact": row.get("artifact", "redfish"), + "tier": 1, + "tier_label": DUMMY_TIER_CRITICAL, + "service_action_num": DUMMY_SERVICE_ACTION_NUM, + } + ], + "tier_grouped": {}, + } + + +def test_mi4xx_plugin_merge_afid_sag_path_into_args(tmp_path): + sag = tmp_path / "custom_afid_sag.json" + sag.write_text("{}", encoding="utf-8") + + collection_args, analysis_args = Mi4xxServiceabilityPlugin._merge_afid_sag_path( + str(sag), + None, + None, + ) + + assert collection_args == {"afid_sag_path": str(sag)} + assert analysis_args == {"afid_sag_path": str(sag)} + + +def test_mi4xx_plugin_merge_afid_sag_path_overrides_analysis_args(tmp_path): + sag = tmp_path / "override_afid_sag.json" + sag.write_text("{}", encoding="utf-8") + + _, analysis_args = Mi4xxServiceabilityPlugin._merge_afid_sag_path( + str(sag), + None, + {"afid_sag_path": "/tmp/old_sag.json", "hub_entry_point": "afse"}, + ) + + assert analysis_args == { + "afid_sag_path": str(sag), + "hub_entry_point": "afse", + } + + +def test_mi4xx_collector_args_default_event_log_uri(): + args = MI4XXCollectorArgs() + assert ( + args.resolved_event_log_uri() + == Mi4xxServiceabilityAnalyzerArgs().resolved_rf_event_log_uri() + ) + + +def test_mi4xx_analyzer_args_default_event_log_uri(): + args = Mi4xxServiceabilityAnalyzerArgs() + assert args.resolved_rf_event_log_uri() == ( + "/redfish/v1/Systems/Instinct_Accelerators/LogServices/EventLog/Entries" + ) + + +def test_mi4xx_serviceability_plugin_wiring(): + assert issubclass(Mi4xxServiceabilityPlugin, ServiceabilityPluginBase) + assert Mi4xxServiceabilityPlugin.COLLECTOR_ARGS is MI4XXCollectorArgs + assert Mi4xxServiceabilityPlugin.ANALYZER_ARGS is Mi4xxServiceabilityAnalyzerArgs + assert Mi4xxServiceabilityPlugin.ANALYZER is MI4XXAnalyzer + + +def test_mi4xx_analyzer_args_defaults_to_afse(): + args = Mi4xxServiceabilityAnalyzerArgs() + assert args.hub_entry_point == "afse" + assert args.hub_display_name == "AFSE" + assert args.resolved_hub_entry_point() == "afse" + assert args.skip_hub is False + + +def test_mi4xx_analyzer_args_rejects_hub_python_module(): + with pytest.raises(ValueError, match="hub_python_module is not supported"): + Mi4xxServiceabilityAnalyzerArgs(hub_python_module="instinct_service_assistant") + + +def test_mi4xx_analyzer_args_rejects_non_afse_entry_point(): + with pytest.raises(ValueError, match="hub_entry_point 'afse' only"): + Mi4xxServiceabilityAnalyzerArgs(hub_entry_point="hub") + + +def test_load_hub_from_entry_point(): + fake_ep = SimpleNamespace(name="afse", load=lambda: _FakeHub) + with patch( + "nodescraper.plugins.serviceability.se_runner._entry_points_for_group", + return_value=[fake_ep], + ): + hub = load_hub_from_entry_point("afse") + assert hub.name == "ExampleHub" + + +def test_mi4xx_analyzer_args_default_afid_sag_path(): + args = Mi4xxServiceabilityAnalyzerArgs() + assert args.resolved_afid_sag_path() == default_afid_sag_path() + + +def test_mi4xx_analyzer_args_override_afid_sag_path(tmp_path): + sag = tmp_path / "custom_sag.json" + sag.write_text("{}", encoding="utf-8") + args = Mi4xxServiceabilityAnalyzerArgs(afid_sag_path=str(sag)) + assert args.resolved_afid_sag_path() == str(sag) + + +def test_resolve_configured_afid_sag_path_prefers_explicit(tmp_path): + sag = tmp_path / "override.json" + sag.write_text("{}", encoding="utf-8") + assert resolve_configured_afid_sag_path(str(sag)) == str(sag) + + +def test_validate_afid_sag_path_validates_file(tmp_path): + sag = tmp_path / "afid_sag.json" + sag.write_text("{}", encoding="utf-8") + assert validate_afid_sag_path(str(sag)) == str(sag) + + +def test_run_entry_point_hub(tmp_path): + sag = tmp_path / "afid_sag.json" + sag.write_text("{}", encoding="utf-8") + fake_ep = SimpleNamespace(name="afse", load=lambda: _FakeHub) + events = [AfidEvent(afid=DUMMY_AFID_A, serviceable_unit=DUMMY_UNIT_A, time=DUMMY_TIMESTAMP)] + with patch( + "nodescraper.plugins.serviceability.se_runner._entry_points_for_group", + return_value=[fake_ep], + ): + block = run_entry_point_hub( + hub_entry_point="afse", + afid_events=events, + afid_sag_path=str(sag), + rf_event_count=1, + ) + assert block.hub_version == DUMMY_HUB_VERSION_ENTRY + assert len(block.solution) == 1 + + +def test_serviceability_block_from_entry_point_hub(): + events = [AfidEvent(afid=DUMMY_AFID_A, serviceable_unit=DUMMY_UNIT_A, time=DUMMY_TIMESTAMP)] + block = serviceability_block_from_entry_point_hub( + events, + { + "schema_version": "1.0", + "status": "ok", + "error": None, + "engine": "ExampleHub", + "engine_version": DUMMY_HUB_VERSION_ENTRY, + "results": [ + { + "afid_num": DUMMY_AFID_A, + "location": DUMMY_UNIT_A, + "service_action_num": DUMMY_SERVICE_ACTION_NUM, + "tier_label": DUMMY_TIER_CRITICAL, + } + ], + }, + rf_event_count=3, + ) + assert len(block.solution) == 1 + assert block.solution[0].afid == DUMMY_AFID_A + assert block.solution[0].service_action_num == DUMMY_SERVICE_ACTION_NUM + assert block.hub_version == DUMMY_HUB_VERSION_ENTRY + + +def test_analyze_serviceability_window_skip_hub(): + data = ServiceabilityDataModel( + rf_events=[ + { + "Afid": DUMMY_AFID_A, + "ServiceableUnit": DUMMY_UNIT_A, + "Created": DUMMY_TIMESTAMP, + } + ] + ) + args = Mi4xxServiceabilityAnalyzerArgs(skip_hub=True) + result = analyze_serviceability_window(data, args) + assert result.ok + assert result.serviceability is not None + assert len(result.afid_events) == 1 + + +def test_serviceability_hub_analyzer_runs_entry_point_hub(system_info, tmp_path): + sag = tmp_path / "afid_sag.json" + sag.write_text("{}", encoding="utf-8") + data = ServiceabilityDataModel( + rf_events=[ + { + "Afid": DUMMY_AFID_A, + "ServiceableUnit": DUMMY_UNIT_A, + "Created": DUMMY_TIMESTAMP, + } + ] + ) + fake_ep = SimpleNamespace(name="afse", load=lambda: _FakeHub) + analyzer = MI4XXAnalyzer(system_info=system_info) + with patch( + "nodescraper.plugins.serviceability.se_runner._entry_points_for_group", + return_value=[fake_ep], + ): + task = analyzer.analyze_data( + data, + Mi4xxServiceabilityAnalyzerArgs(afid_sag_path=str(sag)), + ) + assert task.status == ExecutionStatus.OK + assert "afse" in task.message.lower() + + +def test_mi4xx_analyzer_appends_afid_sag_metadata_artifact(system_info, tmp_path): + sag = tmp_path / "afid_sag.json" + sag.write_text('{"pid": "dummy-pid", "revision": "1"}', encoding="utf-8") + data = ServiceabilityDataModel( + rf_events=[ + { + "Afid": DUMMY_AFID_A, + "ServiceableUnit": DUMMY_UNIT_A, + "Created": DUMMY_TIMESTAMP, + } + ] + ) + fake_ep = SimpleNamespace(name="afse", load=lambda: _FakeHub) + analyzer = MI4XXAnalyzer(system_info=system_info) + with patch( + "nodescraper.plugins.serviceability.se_runner._entry_points_for_group", + return_value=[fake_ep], + ): + task = analyzer.analyze_data( + data, + Mi4xxServiceabilityAnalyzerArgs(afid_sag_path=str(sag)), + ) + assert task.status == ExecutionStatus.OK + assert any(isinstance(artifact, AfidSagMetadataArtifact) for artifact in task.artifacts) + + +def test_mi4xx_plugin_analyzes_offline_data_without_collection(system_info, tmp_path): + sag = tmp_path / "afid_sag.json" + sag.write_text("{}", encoding="utf-8") + data_path = tmp_path / "serviceability_data.json" + data = ServiceabilityDataModel( + rf_events=[ + { + "Afid": DUMMY_AFID_A, + "ServiceableUnit": DUMMY_UNIT_A, + "Created": DUMMY_TIMESTAMP, + } + ] + ) + data_path.write_text(data.model_dump_json(), encoding="utf-8") + fake_ep = SimpleNamespace(name="afse", load=lambda: _FakeHub) + plugin = Mi4xxServiceabilityPlugin(system_info=system_info) + with patch( + "nodescraper.plugins.serviceability.se_runner._entry_points_for_group", + return_value=[fake_ep], + ): + result = plugin.run( + collection=False, + analysis=True, + data=str(data_path), + analysis_args=Mi4xxServiceabilityAnalyzerArgs(afid_sag_path=str(sag)), + ) + assert result.status == ExecutionStatus.OK + assert result.result_data.analysis_result.status == ExecutionStatus.OK + + +def test_load_hub_from_entry_point_missing_raises(): + with patch( + "nodescraper.plugins.serviceability.se_runner._entry_points_for_group", + return_value=[], + ): + with pytest.raises(HubRunError, match="not found"): + load_hub_from_entry_point("missing") + + +@pytest.fixture +def mi4xx_collector(system_info, redfish_conn_mock): + redfish_conn_mock.base_url = f"https://{DUMMY_BMC_HOST}/redfish/v1" + return MI4XXCollector( + system_info=system_info, + connection=redfish_conn_mock, + log_path="/tmp/serviceability.log", + ) + + +def test_fetch_mi4xx_event_log_follows_next_link(mi4xx_collector, redfish_conn_mock): + page1_members = [{"Id": str(i)} for i in range(1000)] + page2_members = [{"Id": str(i)} for i in range(1000, 1687)] + next_link = "/redfish/v1/Systems/DummyAccelerators/LogServices/EventLog/Entries?$skip=1000" + + def run_get_side_effect(path: str, *_args, **_kwargs): + if path == DUMMY_EVENT_URI: + return RedfishGetResult( + path=DUMMY_EVENT_URI, + success=True, + data={ + RF_MEMBERS: page1_members, + RF_MEMBERS_COUNT: 1687, + RF_MEMBERS_NEXT_LINK: next_link, + }, + status_code=200, + ) + if path == next_link: + return RedfishGetResult( + path=next_link, + success=True, + data={RF_MEMBERS: page2_members}, + status_code=200, + ) + raise AssertionError(f"unexpected path: {path}") + + redfish_conn_mock.run_get.side_effect = run_get_side_effect + result = fetch_mi4xx_event_log( + mi4xx_collector, + DUMMY_EVENT_URI, + max_pages=200, + ) + assert result.success + assert len(result.data[RF_MEMBERS]) == 1687 + assert RF_MEMBERS_NEXT_LINK not in result.data + redfish_conn_mock.run_get_paged.assert_not_called() + + +def test_fetch_mi4xx_event_log_skip_fallback_when_no_next_link(mi4xx_collector, redfish_conn_mock): + page1_members = [{"Id": str(i)} for i in range(1000)] + page2_members = [{"Id": str(i)} for i in range(1000, 1200)] + skip_uri = f"{DUMMY_EVENT_URI}?$skip=1000" + + def run_get_side_effect(path: str, *_args, **_kwargs): + if path == DUMMY_EVENT_URI: + return RedfishGetResult( + path=DUMMY_EVENT_URI, + success=True, + data={ + RF_MEMBERS: page1_members, + RF_MEMBERS_COUNT: 1200, + }, + status_code=200, + ) + if path == skip_uri: + return RedfishGetResult( + path=skip_uri, + success=True, + data={RF_MEMBERS: page2_members}, + status_code=200, + ) + raise AssertionError(f"unexpected path: {path}") + + redfish_conn_mock.run_get.side_effect = run_get_side_effect + result = fetch_mi4xx_event_log( + mi4xx_collector, + DUMMY_EVENT_URI, + max_pages=200, + ) + assert result.success + assert len(result.data[RF_MEMBERS]) == 1200 + redfish_conn_mock.run_get_paged.assert_not_called() + + +def test_mi4xx_collector_collect_uses_mi4xx_paging_not_run_get_paged( + mi4xx_collector, redfish_conn_mock +): + members = [{"Id": "1"}, {"Id": "2"}] + redfish_conn_mock.run_get.return_value = RedfishGetResult( + path=DUMMY_EVENT_URI, + success=True, + data={RF_MEMBERS: members, RF_MEMBERS_COUNT: 2}, + status_code=200, + ) + args = MI4XXCollectorArgs(rf_event_log_uri=DUMMY_EVENT_URI, follow_next_link=True) + result, data = mi4xx_collector.collect_data(args=args) + assert result.status == ExecutionStatus.OK + assert data is not None + assert len(data.rf_events) == 2 + redfish_conn_mock.run_get.assert_called() + redfish_conn_mock.run_get_paged.assert_not_called() diff --git a/test/unit/plugin/test_se_runner.py b/test/unit/plugin/test_se_runner.py index 025aef25..c972a27a 100644 --- a/test/unit/plugin/test_se_runner.py +++ b/test/unit/plugin/test_se_runner.py @@ -61,6 +61,7 @@ format_serviceability_solution_lines, normalize_se_timestamp, run_service_hub, + serviceability_block_from_entry_point_hub, serviceability_block_from_service_result, ) from nodescraper.plugins.serviceability.se_models import ServiceabilitySolution @@ -84,16 +85,114 @@ def test_normalize_se_timestamp_preserves_format_value(): assert normalize_se_timestamp(sample) == sample -def test_analyzer_args_require_hub_config(): - with pytest.raises(ValidationError): - ServiceabilityAnalyzerArgs() - with pytest.raises(ValidationError, match="hub_python_module"): - ServiceabilityAnalyzerArgs(afid_sag_path=str(AFID_SAG)) - args = ServiceabilityAnalyzerArgs( +def test_analyzer_args_hub_config_fields(): + args = ServiceabilityAnalyzerArgs() + assert args.hub_python_module is None + assert args.hub_entry_point is None + assert args.uses_entry_point_hub() is True + args_mod = ServiceabilityAnalyzerArgs( hub_python_module="dummy.test.module", afid_sag_path=str(AFID_SAG), ) - assert args.hub_python_module == "dummy.test.module" + assert args_mod.hub_python_module == "dummy.test.module" + assert args_mod.uses_module_hub() is True + assert args_mod.uses_entry_point_hub() is False + args_ep = ServiceabilityAnalyzerArgs(hub_entry_point="hub") + assert args_ep.hub_entry_point == "hub" + assert args_ep.resolved_hub_entry_point() == "hub" + with pytest.raises(ValueError, match="hub_entry_point is required"): + ServiceabilityAnalyzerArgs().resolved_hub_entry_point() + assert args_ep.uses_entry_point_hub() is True + + +def test_afid_events_to_entry_point_payload_matches_hub_contract(): + from nodescraper.plugins.serviceability.se_runner import ( + afid_events_to_entry_point_payload, + ) + + payload = afid_events_to_entry_point_payload( + [ + AfidEvent(afid=DUMMY_AFID_A, serviceable_unit=DUMMY_UNIT_A, time=DUMMY_TIMESTAMP), + AfidEvent(afid=DUMMY_AFID_A, serviceable_unit=DUMMY_UNIT_A, time=DUMMY_TIMESTAMP), + AfidEvent(afid=DUMMY_AFID_B, serviceable_unit=DUMMY_UNIT_B, time=DUMMY_TIMESTAMP), + ] + ) + assert payload == [ + { + "afid": DUMMY_AFID_A, + "location": DUMMY_UNIT_A, + "count": 2, + }, + { + "afid": DUMMY_AFID_B, + "location": DUMMY_UNIT_B, + "count": 1, + }, + ] + + +def test_entry_point_analyze_error_accepts_slim_hub_response(): + from nodescraper.plugins.serviceability.se_runner import _entry_point_analyze_error + + assert _entry_point_analyze_error({"results": []}) is None + assert ( + _entry_point_analyze_error( + { + "engine": "ExampleHub", + "engine_version": "0.1.0", + "results": [{"afid_num": 100, "location": "GPU-1"}], + "tier_grouped": {}, + } + ) + is None + ) + assert _entry_point_analyze_error({"status": "ok", "results": []}) is None + assert ( + _entry_point_analyze_error({"status": "error", "error": {"message": "bad input"}}) + == "bad input" + ) + + +def test_resolve_hub_events_payload_prefers_rf_events(): + from nodescraper.plugins.serviceability.se_runner import ( + resolve_hub_events_payload, + ) + + rf_events = [{"Oem": {"AMD": {"AMDFieldIdentifiers": []}}}] + payload = resolve_hub_events_payload( + afid_events=EXAMPLE_EVENTS, + rf_events=rf_events, + prefer_rf_events=True, + ) + assert payload == rf_events + + +def test_resolve_hub_events_payload_uses_afid_events_without_rf(): + from nodescraper.plugins.serviceability.se_runner import ( + resolve_hub_events_payload, + ) + + payload = resolve_hub_events_payload( + afid_events=EXAMPLE_EVENTS[:1], + rf_events=None, + prefer_rf_events=True, + ) + assert payload[0]["afid"] == EXAMPLE_EVENTS[0].afid + assert payload[0]["location"] == EXAMPLE_EVENTS[0].serviceable_unit + + +def test_resolve_hub_events_payload_honors_prefer_rf_events_false(): + from nodescraper.plugins.serviceability.se_runner import ( + resolve_hub_events_payload, + ) + + payload = resolve_hub_events_payload( + afid_events=EXAMPLE_EVENTS[:1], + rf_events=[{"Id": "event-1"}], + prefer_rf_events=False, + ) + assert payload[0]["afid"] == EXAMPLE_EVENTS[0].afid + assert payload[0]["location"] == EXAMPLE_EVENTS[0].serviceable_unit def test_resolved_hub_options_explicit_fields_override_options_bag(): @@ -140,7 +239,77 @@ def test_format_serviceability_solution_lines(): ) assert f"AFID {DUMMY_AFID_A}" in lines[3] assert DUMMY_DESIGNATION_A in lines[3] - assert "service action 99 (RMA)" in lines[3] + assert 'service action 99: "RMA"' in lines[3] + + +def test_serviceability_block_from_entry_point_hub_uses_sag_labels(tmp_path): + sag = tmp_path / "sag.json" + sag.write_text( + json.dumps( + { + "afid": { + "11110": { + "error_category": "ReadingAboveUpperFatalThreshold", + "error_type": "DegreesC", + "service_action_num": 111, + "service_action": "Update FW", + } + }, + "service_actions": { + "111": { + "title": "Update FW", + "category": "Reflash", + "severity": 20, + "steps": [ + { + "step_num": 0, + "description": "Check higher priority AFIDs first.", + } + ], + }, + }, + } + ), + encoding="utf-8", + ) + hub_result = { + "schema_version": "1.0", + "status": "ok", + "error": None, + "engine": "ExampleHub", + "engine_version": "0.1.0", + "results": [ + { + "afid_num": 11110, + "location": "Instinct_EAM_0", + "service_action_num": 111, + "tier_label": "Secondary", + "tier": 2, + "fru": "EAM_AMC-COMPUTE", + "fru_rank": 1, + "priority": 20, + "sa_severity": 20, + "count": 1, + } + ], + } + block = serviceability_block_from_entry_point_hub( + EXAMPLE_EVENTS[:1], + hub_result, + afid_sag_path=str(sag), + rf_event_count=1, + ) + assert block.solution[0].service_action_title == "Update FW" + assert block.solution[0].service_action_tier == "Secondary" + assert block.solution[0].afid_summary == "ReadingAboveUpperFatalThreshold / DegreesC" + assert len(block.hub_triage_results) == 1 + triage = block.hub_triage_results[0] + assert triage.service_action_steps + assert triage.service_action_category == "Reflash" + lines = format_serviceability_solution_lines(block) + assert "Hub triage results:" in lines + assert "priority=" in "\n".join(lines) + assert "step 0:" in "\n".join(lines) def test_serviceability_block_from_service_result(): @@ -180,7 +349,10 @@ def test_serviceability_block_from_service_result(): assert block.solution[0].afid == DUMMY_AFID_A assert block.solution[0].service_action_num == DUMMY_SERVICE_ACTION_NUM assert block.solution[0].service_action_title == "Dummy service action" - assert set(block.solution[0].serviceable_unit) == {DUMMY_DESIGNATION_A, DUMMY_DESIGNATION_B} + assert set(block.solution[0].serviceable_unit) == { + DUMMY_DESIGNATION_A, + DUMMY_DESIGNATION_B, + } assert block.hub_version == DUMMY_HUB_VERSION assert block.afid_sag_file_version == ( f"PID {DUMMY_SAG_PID}, revision {DUMMY_SAG_REVISION}, variant {DUMMY_SAG_VARIANT}" @@ -231,8 +403,16 @@ def test_resolve_hub_class_finds_package_export(): def test_run_service_hub_with_mock_module(): rf_events = [ - {"Afid": DUMMY_AFID_A, "serviceable_unit": DUMMY_UNIT_A, "Created": DUMMY_TIMESTAMP}, - {"Afid": DUMMY_AFID_C, "serviceable_unit": DUMMY_UNIT_C, "Created": DUMMY_TIMESTAMP}, + { + "Afid": DUMMY_AFID_A, + "serviceable_unit": DUMMY_UNIT_A, + "Created": DUMMY_TIMESTAMP, + }, + { + "Afid": DUMMY_AFID_C, + "serviceable_unit": DUMMY_UNIT_C, + "Created": DUMMY_TIMESTAMP, + }, ] block = run_service_hub( hub_python_module="mock_python_engine", @@ -285,7 +465,11 @@ def analyze_events(self, rf_events, cper_data=None): def test_run_service_hub_accepts_hub_options(): rf_events = [ - {"Afid": DUMMY_AFID_A, "serviceable_unit": DUMMY_UNIT_A, "Created": DUMMY_TIMESTAMP}, + { + "Afid": DUMMY_AFID_A, + "serviceable_unit": DUMMY_UNIT_A, + "Created": DUMMY_TIMESTAMP, + }, ] block = run_service_hub( hub_python_module="mock_python_engine", @@ -302,7 +486,11 @@ def test_run_service_hub_forwards_full_hub_options_kwargs(): clear_last_call() rf_events = [ - {"Afid": DUMMY_AFID_A, "serviceable_unit": DUMMY_UNIT_A, "Created": DUMMY_TIMESTAMP}, + { + "Afid": DUMMY_AFID_A, + "serviceable_unit": DUMMY_UNIT_A, + "Created": DUMMY_TIMESTAMP, + }, ] run_service_hub( hub_python_module="instinct_shaped_engine", @@ -330,7 +518,11 @@ def test_run_service_hub_collected_cper_overrides_hub_options_cper_data(): clear_last_call() rf_events = [ - {"Afid": DUMMY_AFID_A, "serviceable_unit": DUMMY_UNIT_A, "Created": DUMMY_TIMESTAMP}, + { + "Afid": DUMMY_AFID_A, + "serviceable_unit": DUMMY_UNIT_A, + "Created": DUMMY_TIMESTAMP, + }, ] run_service_hub( hub_python_module="instinct_shaped_engine", diff --git a/test/unit/serviceability_dummy_data.py b/test/unit/serviceability_dummy_data.py index 06c78d2e..6652b3fc 100644 --- a/test/unit/serviceability_dummy_data.py +++ b/test/unit/serviceability_dummy_data.py @@ -66,6 +66,60 @@ DUMMY_CPER_EVENT_ID_RF = "dummy-cper-evt-rf" DUMMY_CPER_BYTES_BASIC = b"\x01\x02dummy-cper" DUMMY_CPER_BYTES_RF = b"\xaa\xbb" +DUMMY_NESTED_OEM_AFID = 9101 +DUMMY_UNIT_NESTED = "dummy_nested_unit_0" +DUMMY_FRU_PRIMARY = "DUMMY-FRU-PRIMARY" +DUMMY_FRU_SECONDARY = "DUMMY-FRU-SECONDARY" +DUMMY_FRU_TERTIARY = "DUMMY-FRU-TERTIARY" +DUMMY_FRU_PRIMARY_NORM = "DUMMY_FRU_PRIMARY" +DUMMY_FRU_SECONDARY_NORM = "DUMMY_FRU_SECONDARY" +DUMMY_SERIAL_PRIMARY = "DUMMY-SERIAL-001" +DUMMY_SERIAL_RF_META = "DUMMY-SERIAL-RF-001" +DUMMY_PART_PRIMARY = "DUMMY-PART-001" +DUMMY_PART_RF_META = "DUMMY-PART-RF-001" +DUMMY_UNIT_NAME_PRIMARY = "Dummy unit primary" +DUMMY_UNIT_VERSION_PRIMARY = "0.0.1-dummy" +DUMMY_ERROR_CATEGORY = "DummyErrorCategory" +DUMMY_ERROR_TYPE = "DummyErrorType" +DUMMY_ERROR_SEVERITY = "Critical" +DUMMY_TIER_LABEL = "Secondary" +DUMMY_TIER_CRITICAL = "Critical" +DUMMY_SA_SEVERITY = 20 +DUMMY_PRIORITY = 20 +DUMMY_AFID_SUMMARY = "DummyErrorCategory / DummyErrorType" +DUMMY_RF_EVENT_COUNT_SAMPLE = 99 +DUMMY_MESSAGE_ID = "DummyEvent.1.0.DummyThreshold" +DUMMY_SERVICE_ACTION_NUM_ALT = 88 +DUMMY_HUB_VERSION_ENTRY = "0.0.0-dummy-entry" +DUMMY_SERVICE_ACTION_CATEGORY = "DummyCategory" +DUMMY_SERVICE_ACTION_STEP = "Dummy step." +DUMMY_NESTED_OEM_AFID = 9101 +DUMMY_UNIT_NESTED = "dummy_nested_unit_0" +DUMMY_FRU_PRIMARY = "DUMMY-FRU-PRIMARY" +DUMMY_FRU_SECONDARY = "DUMMY-FRU-SECONDARY" +DUMMY_FRU_TERTIARY = "DUMMY-FRU-TERTIARY" +DUMMY_FRU_PRIMARY_NORM = "DUMMY_FRU_PRIMARY" +DUMMY_FRU_SECONDARY_NORM = "DUMMY_FRU_SECONDARY" +DUMMY_SERIAL_PRIMARY = "DUMMY-SERIAL-001" +DUMMY_SERIAL_RF_META = "DUMMY-SERIAL-RF-001" +DUMMY_PART_PRIMARY = "DUMMY-PART-001" +DUMMY_PART_RF_META = "DUMMY-PART-RF-001" +DUMMY_UNIT_NAME_PRIMARY = "Dummy unit primary" +DUMMY_UNIT_VERSION_PRIMARY = "0.0.1-dummy" +DUMMY_ERROR_CATEGORY = "DummyErrorCategory" +DUMMY_ERROR_TYPE = "DummyErrorType" +DUMMY_ERROR_SEVERITY = "Critical" +DUMMY_TIER_LABEL = "Secondary" +DUMMY_TIER_CRITICAL = "Critical" +DUMMY_SA_SEVERITY = 20 +DUMMY_PRIORITY = 20 +DUMMY_AFID_SUMMARY = "DummyErrorCategory / DummyErrorType" +DUMMY_RF_EVENT_COUNT_SAMPLE = 99 +DUMMY_MESSAGE_ID = "DummyEvent.1.0.DummyThreshold" +DUMMY_SERVICE_ACTION_NUM_ALT = 88 +DUMMY_HUB_VERSION_ENTRY = "0.0.0-dummy-entry" +DUMMY_SERVICE_ACTION_CATEGORY = "DummyCategory" +DUMMY_SERVICE_ACTION_STEP = "Dummy step." def dummy_chassis_uri(unit: str) -> str: diff --git a/tools/smoke_se_hub.py b/tools/smoke_se_hub.py new file mode 100644 index 00000000..dc0b0ef0 --- /dev/null +++ b/tools/smoke_se_hub.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Smoke test service hub entry-point integration with an installed hub package.""" +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +from nodescraper.plugins.serviceability import AfidEvent, run_entry_point_hub +from nodescraper.plugins.serviceability.se_runner import list_hub_entry_point_names + +ROOT = Path(__file__).resolve().parents[1] +COMPAT_SAG = ROOT / "test/unit/plugin/fixtures/afid_sag_se_compat.json" + + +def resolve_sag_path() -> Path: + if COMPAT_SAG.is_file(): + return COMPAT_SAG + home_sag = Path.home() / "AFID_SAG.json" + if home_sag.is_file(): + maker = ROOT / "tools/make_se_compat_sag.py" + subprocess.run( + [sys.executable, str(maker), "--source", str(home_sag), "--output", str(COMPAT_SAG)], + check=True, + ) + return COMPAT_SAG + fixture = ROOT / "test/unit/plugin/fixtures/afid_sag_sample.json" + if fixture.is_file(): + return fixture + raise FileNotFoundError("No AFID SAG file found") + + +def main() -> int: + print("hub entry points:", list_hub_entry_point_names()) # noqa: T201 + sag = resolve_sag_path() + print("using sag:", sag) # noqa: T201 + events = [ + AfidEvent(afid=9001, serviceable_unit="dummy_unit_a", time="2000-01-01T12:00:00+00:00"), + AfidEvent(afid=9002, serviceable_unit="dummy_unit_b", time="2000-01-01T12:00:00+00:00"), + ] + block = run_entry_point_hub( + hub_entry_point="hub", + afid_events=events, + afid_sag_path=str(sag), + rf_event_count=2, + ) + print("solutions:", len(block.solution)) # noqa: T201 + print("hub_version:", block.hub_version) # noqa: T201 + for solution in block.solution[:5]: + print( # noqa: T201 + f" AFID {solution.afid} SA {solution.service_action_num} " + f"units={solution.serviceable_unit}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())