diff --git a/docs/test-plan.yaml b/docs/test-plan.yaml index 79650540..6a0350bc 100644 --- a/docs/test-plan.yaml +++ b/docs/test-plan.yaml @@ -2060,6 +2060,8 @@ domains: tests: - summary: Query serial numbers of installed hardware (chassis, baseboard, NICs, CPU, GPU) — obfuscated but stable IDs are OK labels: + - bare_metal + - breakfix - min_req priority: P1 milestone: M5 @@ -2281,7 +2283,9 @@ domains: status: pending - summary: Verify topology observability — visibility into failure domains for physical diversity labels: + - bare_metal - min_req + - sds_controller priority: P1 milestone: M5 notes: "" diff --git a/isvctl/configs/providers/nico/config/bare_metal.yaml b/isvctl/configs/providers/nico/config/bare_metal.yaml index 1e0e0f20..228a11d2 100644 --- a/isvctl/configs/providers/nico/config/bare_metal.yaml +++ b/isvctl/configs/providers/nico/config/bare_metal.yaml @@ -80,6 +80,7 @@ commands: # Output includes per-machine status, health, GPU/DPU counts. - name: verify_ingestion phase: test + continue_on_failure: true command: "python ../scripts/hardware_ingestion/verify_ingestion.py" args: - "--org" @@ -95,6 +96,7 @@ commands: # probes, agent heartbeat status, and capability inventory. - name: check_dpu_health phase: test + continue_on_failure: true command: "python ../scripts/dpu/check_dpu_health.py" args: - "--org" @@ -111,6 +113,7 @@ commands: # produced for both nodes and GPUs. - name: query_governance_metrics phase: test + continue_on_failure: true command: "python ../scripts/governance/query_metrics.py" args: - "--org" @@ -126,6 +129,7 @@ commands: # and memory health, plus the freshness of the observation. - name: query_host_health phase: test + continue_on_failure: true command: "python ../scripts/health/query_host_health.py" args: - "--org" @@ -141,6 +145,7 @@ commands: # with per-group healthy/unhealthy counts and an aggregate status. - name: query_health_aggregation phase: test + continue_on_failure: true command: "python ../scripts/health/query_health_aggregation.py" args: - "--org" @@ -159,6 +164,7 @@ commands: # across tenants. - name: query_ib_tenant_isolation phase: test + continue_on_failure: true command: "python ../scripts/infiniband/query_ib_tenant_isolation.py" args: - "--org" @@ -177,6 +183,7 @@ commands: # unverified. IbKeysConfiguredCheck validates the required subset. - name: query_ib_keys phase: test + continue_on_failure: true command: "python ../scripts/infiniband/query_ib_keys.py" args: - "--org" @@ -197,6 +204,7 @@ commands: # secure erase. - name: query_sanitization phase: test + continue_on_failure: true command: "python ../scripts/sanitization/query_sanitization.py" args: - "--org" @@ -207,6 +215,44 @@ commands: - "{{nico_api_base}}" timeout: 120 + # ─── Query Hardware Serial Numbers (BFX03-01) ────────────────── + # Reads each machine's discovered hardware metadata and reduces it to a + # per-component stable-identifier record (chassis / baseboard DMI serials, + # GPU serials, NIC MAC/GUID, CPU model). HardwareSerialCheck asserts a + # stable identifier is queryable for every present component so break/fix + # can identify installed hardware. Obfuscated-but-stable IDs are accepted. + - name: query_serial_numbers + phase: test + continue_on_failure: true + command: "python ../scripts/hardware_inventory/query_serial_numbers.py" + args: + - "--org" + - "{{org}}" + - "--site-id" + - "{{site_id}}" + - "--api-base" + - "{{nico_api_base}}" + timeout: 120 + + # ─── Query Failure-Domain Topology (STG05-01) ────────────────── + # Reads each machine's rack label and reduces it to a per-host + # failure-domain identifier plus the distinct set of failure domains at + # the site. FailureDomainObservabilityCheck asserts the topology is + # observable (every host maps to a named failure domain) so physical + # diversity across racks/enclosures can be reasoned about. + - name: query_topology + phase: test + continue_on_failure: true + command: "python ../scripts/topology/query_topology.py" + args: + - "--org" + - "{{org}}" + - "--site-id" + - "{{site_id}}" + - "--api-base" + - "{{nico_api_base}}" + timeout: 120 + # ─── Query Attestation (SEC22-01 / CNP09-02) ──────────────────── # Enumerates site machines via tenant REST, then reads SPDM attestation # and measured-boot status from the Forge control-plane API via @@ -215,6 +261,7 @@ commands: # NICO_FORGE_ROOT_CA_PATH, NICO_CLIENT_KEY_PATH, NICO_CLIENT_CERT_PATH). - name: query_attestation phase: test + continue_on_failure: true command: "python ../scripts/attestation/query_attestation.py" args: - "--org" @@ -229,6 +276,7 @@ commands: # Lists instances visible to NICo at the configured site. - name: list_instances phase: test + continue_on_failure: true command: "python ../scripts/bare_metal/list_instances.py" args: - "--org" @@ -244,6 +292,7 @@ commands: # Describes NICO_INSTANCE_ID, or the first instance found at the site. - name: describe_instance phase: test + continue_on_failure: true command: "python ../scripts/bare_metal/describe_instance.py" args: - "--org" diff --git a/isvctl/configs/providers/nico/scripts/hardware_inventory/query_serial_numbers.py b/isvctl/configs/providers/nico/scripts/hardware_inventory/query_serial_numbers.py new file mode 100644 index 00000000..40903d84 --- /dev/null +++ b/isvctl/configs/providers/nico/scripts/hardware_inventory/query_serial_numbers.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Query stable hardware serial numbers for a NICo site (BFX03-01). + +Break/fix workflows need to identify the physical hardware installed in a host: +chassis, baseboard, network interfaces (NICs), CPU, and GPU. NICo discovers this +inventory during ingestion and exposes it on the Machine resource. This script +reads the per-machine hardware metadata and reduces it to a provider-neutral +per-component identifier record so ``HardwareSerialCheck`` can assert that a +stable identifier is queryable for every component class that is present. + +Per BFX03-01 the identifiers may be obfuscated as long as they are stable, so +this script never depends on globally-unique serials. Where NICo exposes a true +serial (chassis / baseboard / GPU) it is used; NICs are identified by their +stable MAC address (Ethernet) or GUID (InfiniBand); CPUs have no per-socket +serial in DMI, so the stable CPU model descriptor is reported as the identifier. +The identifier *values* are hardware asset IDs (not secrets), so they are +emitted as-is. + +NICo Machine hardware sources (``includeMetadata=true``): + - chassis: ``metadata.dmiData.chassisSerial`` + - baseboard: ``metadata.dmiData.boardSerial`` + - gpu: ``metadata.gpus[].serial`` + - nic: ``metadata.networkInterfaces[].macAddress`` + + ``metadata.infinibandInterfaces[].guid`` + - cpu: ``machineCapabilities[type=CPU]`` (model descriptor; no serial) + - machine: top-level ``serialNumber`` (provider-visible), used as a + chassis-serial fallback. + +NICo API endpoints used: + GET /v2/org/{org}/carbide/machine?siteId={site_id}&includeMetadata=true + +Auth: + - NICO_BEARER_TOKEN, or + - OIDC client_credentials via NICO_SSA_ISSUER, + NICO_CLIENT_ID, NICO_CLIENT_SECRET, and optional NICO_OIDC_SCOPE. + +Required JSON output fields: + { + "success": true, + "platform": "nico", + "site_id": "...", + "machines_checked": 1, + "machines": [ + { + "machine_id": "...", + "components": { + "chassis": {"present": true, "identifiers": ["J1050ACR"]}, + "baseboard": {"present": true, "identifiers": [".C1KS2CS002G."]}, + "cpu": {"present": true, "identifiers": ["Intel(R) Xeon(R) Gold 6354 CPU @ 3.00GHz"]}, + "gpu": {"present": true, "identifiers": ["1654422006434"]}, + "nic": {"present": true, "identifiers": ["c8:4b:d6:7b:ac:a8", "1070fd0300bd43ac"]} + } + } + ] + } + +A site with no ingested machines emits a structured skip (``skipped`` / +``skip_reason``) so the validation does not hard-fail a site that has no +hardware discovered yet. + +Usage: + NICO_BEARER_TOKEN= python query_serial_numbers.py \ + --org --site-id --api-base + + Wired via the bare_metal suite: + uv run isvctl test run -f isvctl/configs/providers/nico/config/bare_metal.yaml + +Reference: + OpenAPI spec: rest-api/openapi/spec.yaml + (MachineMetadata / MachineDMIData / MachineGPUInfo / + MachineNetworkInterface / MachineInfiniBandInterface / MachineCapability) +""" + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +# Allow importing from sibling common/ directory +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from common.nico_client import NicoAuthError, forge_get_all, resolve_auth + + +def _dedupe(identifiers: list[str]) -> list[str]: + """Drop non-string/blank values and duplicates while preserving first-seen order.""" + seen: dict[str, None] = {} + for identifier in identifiers: + cleaned = identifier.strip() if isinstance(identifier, str) else "" + if cleaned and cleaned not in seen: + seen[cleaned] = None + return list(seen) + + +def _component(present: bool, identifiers: list[str]) -> dict[str, Any]: + """Build a per-component record with presence and deduplicated identifiers.""" + return {"present": present, "identifiers": _dedupe(identifiers)} + + +def machine_serials(machine: dict[str, Any]) -> dict[str, Any]: + """Build the provider-neutral per-component serial record for one machine. + + A component is ``present`` when the machine reports that hardware class at + all (so a CPU-only storage node correctly reports ``gpu.present = false`` + rather than failing for a missing GPU serial). ``identifiers`` holds the + stable IDs that were queryable for a present component; a present component + with no queryable identifier yields an empty list, which the validation + treats as a failure. + """ + metadata = machine.get("metadata") or {} + dmi = metadata.get("dmiData") or {} + gpus = metadata.get("gpus") or [] + nics = metadata.get("networkInterfaces") or [] + ib_nics = metadata.get("infinibandInterfaces") or [] + capabilities = [c for c in (machine.get("machineCapabilities") or []) if isinstance(c, dict)] + + # Chassis: both the DMI chassis serial and the provider-visible machine + # serial number are stable chassis identifiers; reporting both keeps the + # component queryable when DMI leaves chassisSerial blank. + chassis_ids = [dmi.get("chassisSerial"), machine.get("serialNumber")] + + cpu_ids = [c.get("name") for c in capabilities if c.get("type") == "CPU"] + + nic_ids = [n.get("macAddress") for n in nics if isinstance(n, dict)] + nic_ids += [n.get("guid") for n in ib_nics if isinstance(n, dict)] + + gpu_ids = [g.get("serial") for g in gpus if isinstance(g, dict)] + # A GPU host is one that reports GPUs in metadata or an ingested GPU + # capability. A CPU/storage node legitimately has no GPU, so gpu.present + # stays false there and the check does not expect a GPU serial. + gpu_present = bool(gpus) or any(c.get("type") == "GPU" for c in capabilities) + + # Chassis, baseboard, CPU, and NIC exist on every physical host, so they are + # always present: the check then requires each to expose a stable identifier. + # GPU presence is conditional on the host actually having accelerators. + return { + "machine_id": machine.get("id", ""), + "components": { + "chassis": _component(present=True, identifiers=chassis_ids), + "baseboard": _component(present=True, identifiers=[dmi.get("boardSerial")]), + "cpu": _component(present=True, identifiers=cpu_ids), + "gpu": _component(present=gpu_present, identifiers=gpu_ids), + "nic": _component(present=True, identifiers=nic_ids), + }, + } + + +def main() -> int: + """Query NICo machines and print per-machine hardware serial records as JSON.""" + parser = argparse.ArgumentParser(description="Query NICo hardware serial numbers") + parser.add_argument("--org", required=True, help="NGC org name") + parser.add_argument("--site-id", required=True, help="NICo site UUID") + parser.add_argument("--api-base", required=True, help="NICo API base URL") + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "nico", + "site_id": args.site_id, + "machines_checked": 0, + "machines": [], + } + + try: + auth = resolve_auth() + + machines = forge_get_all( + args.org, + "machine", + auth.token, + base_url=args.api_base, + params={"siteId": args.site_id, "includeMetadata": "true"}, + result_key="machines", + ) + + if not machines: + result["success"] = True + result["skipped"] = True + result["skip_reason"] = "No machines found at site; no hardware discovered to query serial numbers for" + print(json.dumps(result, indent=2)) + return 0 + + result["machines"] = [machine_serials(machine) for machine in machines] + result["machines_checked"] = len(result["machines"]) + result["success"] = True + + except NicoAuthError as e: + result["error_type"] = "auth" + result["error"] = str(e) + except Exception as e: + result["error"] = f"{type(e).__name__}: {e}" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/nico/scripts/topology/query_topology.py b/isvctl/configs/providers/nico/scripts/topology/query_topology.py new file mode 100644 index 00000000..fdc39a54 --- /dev/null +++ b/isvctl/configs/providers/nico/scripts/topology/query_topology.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Query physical topology / failure-domain observability for a NICo site (STG05-01). + +Physical diversity decisions (spreading a tenant's storage or compute across +independent failure domains) require that the platform expose which failure +domain each host belongs to. In NICo a Rack is "a physical enclosure that +contains a number of Machines ... the physical building blocks of a Site", and +each ingested Machine carries its rack identity as a well-known label. That rack +identifier is the provider-neutral failure domain. + +This script reads each machine's labels, extracts the rack/failure-domain +identifier, and emits a provider-neutral per-host record. +``FailureDomainObservabilityCheck`` then asserts the topology is observable +(every host maps to a named failure domain) and that the site exposes enough +distinct domains for the configured diversity floor. + +NICo API endpoints used: + GET /v2/org/{org}/carbide/machine?siteId={site_id} + +Auth: + - NICO_BEARER_TOKEN, or + - OIDC client_credentials via NICO_SSA_ISSUER, + NICO_CLIENT_ID, NICO_CLIENT_SECRET, and optional NICO_OIDC_SCOPE. + +Required JSON output fields: + { + "success": true, + "platform": "nico", + "site_id": "...", + "hosts_checked": 2, + "hosts": [ + {"host_id": "...", "failure_domain": "GVX11F01C02"} + ] + } + +A site with no ingested machines emits a structured skip (``skipped`` / +``skip_reason``) so the validation does not hard-fail a site with no hardware +discovered yet. + +Usage: + NICO_BEARER_TOKEN= python query_topology.py \ + --org --site-id --api-base + + Wired via the bare_metal suite: + uv run isvctl test run -f isvctl/configs/providers/nico/config/bare_metal.yaml + +Reference: + OpenAPI spec: rest-api/openapi/spec.yaml (Machine.labels; Rack tag / well-known + location.* + rack labels described under Expected Rack) +""" + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +# Allow importing from sibling common/ directory +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from common.inventory import first_string +from common.nico_client import NicoAuthError, forge_get_all, resolve_auth + +# Machine label keys that carry the rack (failure-domain) identity, in priority +# order. NICo populates ``RackIdentifier`` on ingested machines; expected-machine +# manifests use the shorter ``rack`` label. +RACK_LABEL_KEYS = ("RackIdentifier", "rack") + + +def failure_domain(machine: dict[str, Any]) -> str: + """Return the machine's rack/failure-domain identifier, or an empty string.""" + labels = machine.get("labels") or {} + if not isinstance(labels, dict): + return "" + return first_string(labels, *RACK_LABEL_KEYS) + + +def host_record(machine: dict[str, Any]) -> dict[str, Any]: + """Build the provider-neutral topology record for one NICo machine.""" + return { + "host_id": machine.get("id", ""), + "failure_domain": failure_domain(machine), + } + + +def main() -> int: + """Query NICo machines and print per-host failure-domain topology JSON.""" + parser = argparse.ArgumentParser(description="Query NICo failure-domain topology") + parser.add_argument("--org", required=True, help="NGC org name") + parser.add_argument("--site-id", required=True, help="NICo site UUID") + parser.add_argument("--api-base", required=True, help="NICo API base URL") + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "nico", + "site_id": args.site_id, + "hosts_checked": 0, + "hosts": [], + } + + try: + auth = resolve_auth() + + machines = forge_get_all( + args.org, + "machine", + auth.token, + base_url=args.api_base, + params={"siteId": args.site_id}, + result_key="machines", + ) + + if not machines: + result["success"] = True + result["skipped"] = True + result["skip_reason"] = "No machines found at site; no hosts to report failure-domain topology for" + print(json.dumps(result, indent=2)) + return 0 + + result["hosts"] = [host_record(machine) for machine in machines] + result["hosts_checked"] = len(result["hosts"]) + result["success"] = True + + except NicoAuthError as e: + result["error_type"] = "auth" + result["error"] = str(e) + except Exception as e: + result["error"] = f"{type(e).__name__}: {e}" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index 41591485..2cf92c6f 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -125,6 +125,8 @@ For the domain / script-count / AWS-reference overview see the | `query_ib_keys` | test | `providers/nico/scripts/infiniband/query_ib_keys.py` | `partitions_with_pkey`, `keys..{configured,source,detail}` | | `query_sanitization` | test | `providers/nico/scripts/sanitization/query_sanitization.py` | `machines_checked`, `machines[].{available,in_use,has_gpu,served_tenant,sanitized,stale_tenant_binding,vendor,product_name,bios_version,transitions}` | | `query_attestation` | test | `providers/nico/scripts/attestation/query_attestation.py` | `machines_checked`, `machines[].{attestation_supported,nonce_verified,attestation_signature_valid,secure_boot_enabled,boot_measurements_attested,measured_boot_state}` | +| `query_serial_numbers` | test | `providers/nico/scripts/hardware_inventory/query_serial_numbers.py` | `machines_checked`, `machines[].components.{chassis,baseboard,cpu,gpu,nic}.{present,identifiers}` | +| `query_topology` | test | `providers/nico/scripts/topology/query_topology.py` | `hosts_checked`, `hosts[].{host_id,failure_domain}` | ### Storage (`storage.yaml`) diff --git a/isvctl/configs/suites/bare_metal.yaml b/isvctl/configs/suites/bare_metal.yaml index 1326cbcd..0901e9ee 100644 --- a/isvctl/configs/suites/bare_metal.yaml +++ b/isvctl/configs/suites/bare_metal.yaml @@ -572,6 +572,33 @@ tests: test_id: "SEC21-02" labels: ["bare_metal", "disk", "min_req", "sanitization", "security"] + # Hardware serial-number inventory (BFX03-01). Asserts a stable identifier + # is queryable for every present hardware component (chassis, baseboard, CPU, + # GPU, NIC) so break/fix can identify installed hardware. Obfuscated but + # stable IDs are accepted. Only runs for providers that implement the + # query_serial_numbers step. + hardware_serials: + step: query_serial_numbers + checks: + HardwareSerialCheck: + test_id: "BFX03-01" + labels: ["bare_metal", "breakfix", "min_req"] + min_machines: 1 + + # Failure-domain topology observability (STG05-01). Asserts every host + # reports the physical failure domain (rack/enclosure) it belongs to, so a + # tenant can spread storage/compute across independent domains for physical + # diversity. Raise min_failure_domains above 1 to enforce that a site spans + # multiple domains. Only runs for providers that implement the query_topology + # step. + failure_domain_observability: + step: query_topology + checks: + FailureDomainObservabilityCheck: + test_id: "STG05-01" + labels: ["bare_metal", "sds_controller", "min_req"] + min_failure_domains: 1 + # Hardware/firmware attestation (SEC22 / CNP09). Each check asserts a host # establishes a hardware root of trust. Only runs for providers that # implement the query_attestation step. diff --git a/isvctl/tests/providers/nico/test_nico_provider.py b/isvctl/tests/providers/nico/test_nico_provider.py index 55d3c710..4a1eedf2 100644 --- a/isvctl/tests/providers/nico/test_nico_provider.py +++ b/isvctl/tests/providers/nico/test_nico_provider.py @@ -32,6 +32,7 @@ import pytest from isvtest.validations.attestation import FirmwareAttestationCheck, NonceAttestationCheck from isvtest.validations.governance import GovernanceMetricsCheck +from isvtest.validations.hardware import HardwareSerialCheck from isvtest.validations.health import HealthAggregationCheck, HostHealthCheck from isvtest.validations.infiniband import IbKeysConfiguredCheck, IbTenantIsolationCheck from isvtest.validations.sanitization import ( @@ -39,6 +40,7 @@ GpuMemorySanitizationCheck, MemorySanitizationCheck, ) +from isvtest.validations.topology import FailureDomainObservabilityCheck from isvctl.config.merger import merge_yaml_files from isvctl.config.schema import RunConfig @@ -2591,3 +2593,215 @@ def test_sanitization_script_output_satisfies_disk_check( assert "1/1 machine(s)" in bad._error sub = next(r for r in bad._subtest_results if r["name"] == "disk_m-1") assert "without sanitization" in sub["message"] + + +# --------------------------------------------------------------------------- +# query_serial_numbers (BFX03-01) script +# --------------------------------------------------------------------------- + + +def _load_serial_numbers_script() -> ModuleType: + """Load the query_serial_numbers script as a module for direct unit testing.""" + return _load_nico_script("hardware_inventory/query_serial_numbers.py", "test_query_serial_numbers") + + +def _serial_api_machine( + *, + machine_id: str = "m-1", + chassis_serial: str | None = "J1050ACR", + board_serial: str | None = ".C1KS2CS002G.", + machine_serial: str | None = "J1060ACR.D3KS2CS001G", + gpus: list[dict[str, Any]] | None = None, + nics: list[dict[str, Any]] | None = None, + ib_nics: list[dict[str, Any]] | None = None, + capabilities: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Build a raw NICo machine payload with hardware metadata.""" + if gpus is None: + gpus = [{"name": "NVIDIA H100 PCIe", "serial": "1654422006434"}] + if nics is None: + nics = [{"macAddress": "c8:4b:d6:7b:ac:a8", "vendor": "Broadcom"}] + if ib_nics is None: + ib_nics = [{"guid": "1070fd0300bd43ac", "vendor": "Mellanox"}] + if capabilities is None: + capabilities = [{"type": "CPU", "name": "Intel(R) Xeon(R) Gold 6354", "count": 2}] + return { + "id": machine_id, + "serialNumber": machine_serial, + "machineCapabilities": capabilities, + "metadata": { + "dmiData": {"chassisSerial": chassis_serial, "boardSerial": board_serial}, + "gpus": gpus, + "networkInterfaces": nics, + "infinibandInterfaces": ib_nics, + }, + } + + +def _run_serial_numbers( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + machines: list[dict[str, Any]], +) -> dict[str, Any]: + """Drive the query_serial_numbers script with mocked NICo machines.""" + module = _load_serial_numbers_script() + return _run_script(module, monkeypatch, capsys, script_name="query_serial_numbers.py", machines=machines) + + +def test_serial_numbers_script_maps_all_components( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Every hardware component class is reduced to stable identifiers.""" + payload = _run_serial_numbers(monkeypatch, capsys, [_serial_api_machine()]) + + assert payload["success"] is True + assert payload["platform"] == "nico" + assert payload["machines_checked"] == 1 + components = payload["machines"][0]["components"] + assert components["chassis"] == {"present": True, "identifiers": ["J1050ACR", "J1060ACR.D3KS2CS001G"]} + assert components["baseboard"]["identifiers"] == [".C1KS2CS002G."] + assert components["cpu"]["identifiers"] == ["Intel(R) Xeon(R) Gold 6354"] + assert components["gpu"] == {"present": True, "identifiers": ["1654422006434"]} + assert components["nic"]["identifiers"] == ["c8:4b:d6:7b:ac:a8", "1070fd0300bd43ac"] + + +def test_serial_numbers_script_gpu_absent_on_cpu_node( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """A node with no GPUs reports gpu.present=false rather than an empty serial.""" + machine = _serial_api_machine(gpus=[], capabilities=[{"type": "CPU", "name": "AMD EPYC", "count": 1}]) + payload = _run_serial_numbers(monkeypatch, capsys, [machine]) + + gpu = payload["machines"][0]["components"]["gpu"] + assert gpu["present"] is False + assert gpu["identifiers"] == [] + + +def test_serial_numbers_script_chassis_falls_back_to_machine_serial( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """A blank DMI chassis serial falls back to the provider-visible machine serial.""" + machine = _serial_api_machine(chassis_serial=None, machine_serial="FALLBACK-123") + payload = _run_serial_numbers(monkeypatch, capsys, [machine]) + + assert payload["machines"][0]["components"]["chassis"]["identifiers"] == ["FALLBACK-123"] + + +def test_serial_numbers_script_empty_site_skips( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """A site with no machines emits a structured skip.""" + payload = _run_serial_numbers(monkeypatch, capsys, []) + + assert payload["success"] is True + assert payload["skipped"] is True + assert "No machines found" in payload["skip_reason"] + + +def test_serial_numbers_script_output_satisfies_check( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """End-to-end: fully-populated inventory passes; a present GPU with no serial fails.""" + good = _run_serial_numbers(monkeypatch, capsys, [_serial_api_machine()]) + check = HardwareSerialCheck(config={"step_output": good}) + check.run() + assert check._passed is True, check._error + + # A GPU host whose GPU exposes no serial fails. + gpu_no_serial = _serial_api_machine(gpus=[{"name": "NVIDIA H100 PCIe", "serial": None}]) + bad_payload = _run_serial_numbers(monkeypatch, capsys, [gpu_no_serial]) + bad = HardwareSerialCheck(config={"step_output": bad_payload}) + bad.run() + assert bad._passed is False + assert "gpu" in bad._error + + +# --------------------------------------------------------------------------- +# query_topology (STG05-01) script +# --------------------------------------------------------------------------- + + +def _load_topology_script() -> ModuleType: + """Load the query_topology script as a module for direct unit testing.""" + return _load_nico_script("topology/query_topology.py", "test_query_topology") + + +def _topology_api_machine(machine_id: str = "m-1", labels: dict[str, str] | None = None) -> dict[str, Any]: + """Build a raw NICo machine payload carrying rack labels.""" + return {"id": machine_id, "labels": labels if labels is not None else {"RackIdentifier": "GVX11F01C02"}} + + +def _run_topology( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + machines: list[dict[str, Any]], +) -> dict[str, Any]: + """Drive the query_topology script with mocked NICo machines.""" + module = _load_topology_script() + return _run_script(module, monkeypatch, capsys, script_name="query_topology.py", machines=machines) + + +def test_topology_script_extracts_rack_identifier( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """The RackIdentifier label becomes the per-host failure domain.""" + machines = [ + _topology_api_machine("m-1", {"RackIdentifier": "rack-A"}), + _topology_api_machine("m-2", {"rack": "rack-B"}), + ] + payload = _run_topology(monkeypatch, capsys, machines) + + assert payload["success"] is True + assert payload["hosts_checked"] == 2 + assert payload["hosts"][0] == {"host_id": "m-1", "failure_domain": "rack-A"} + assert payload["hosts"][1] == {"host_id": "m-2", "failure_domain": "rack-B"} + + +def test_topology_script_unlabeled_host_has_no_domain( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """A machine with no rack label reports an empty failure domain.""" + machines = [_topology_api_machine("m-1", {})] + payload = _run_topology(monkeypatch, capsys, machines) + + assert payload["hosts"][0]["failure_domain"] == "" + + +def test_topology_script_empty_site_skips( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """A site with no machines emits a structured skip.""" + payload = _run_topology(monkeypatch, capsys, []) + + assert payload["success"] is True + assert payload["skipped"] is True + assert "No machines found" in payload["skip_reason"] + + +def test_topology_script_output_satisfies_check( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """End-to-end: mapped hosts pass; an unlabeled host flows through to a failure.""" + good = _run_topology( + monkeypatch, + capsys, + [_topology_api_machine("m-1", {"RackIdentifier": "rack-A"})], + ) + check = FailureDomainObservabilityCheck(config={"step_output": good}) + check.run() + assert check._passed is True, check._error + + bad_payload = _run_topology(monkeypatch, capsys, [_topology_api_machine("m-1", {})]) + bad = FailureDomainObservabilityCheck(config={"step_output": bad_payload}) + bad.run() + assert bad._passed is False + assert "m-1" in bad._error diff --git a/isvtest/src/isvtest/validations/hardware.py b/isvtest/src/isvtest/validations/hardware.py index c091c365..9ee4ec67 100644 --- a/isvtest/src/isvtest/validations/hardware.py +++ b/isvtest/src/isvtest/validations/hardware.py @@ -13,10 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Hardware ingestion and DPU health validations. +"""Hardware ingestion, inventory, and DPU health validations. Validations for NICo bare metal hardware lifecycle: - Hardware ingestion verification (expected vs actual machines) +- Hardware serial-number inventory (chassis, baseboard, NIC, CPU, GPU) - DPU health checks (agent heartbeat, probes, capabilities) - DPU network readiness (interfaces, BGP, extension services) """ @@ -175,6 +176,103 @@ def run(self) -> None: self.set_passed(f"All {matched_count} expected machines ingested and healthy") +class HardwareSerialCheck(BaseValidation): + """Validate stable hardware serial numbers are queryable (BFX03-01). + + Break/fix workflows must be able to identify the physical hardware installed + in a host. This check asserts that, for every machine, a stable identifier + is queryable for each configured component class that is present on the host + (chassis, baseboard, CPU, GPU, NIC). Per BFX03-01 the identifiers may be + obfuscated as long as they are stable, so it only verifies that a non-empty + identifier exists -- not that it is globally unique. + + A component that is not present on a host (e.g. GPUs on a CPU-only node) is + reported as an informational skipped subtest, never a failure. A component + that is present but exposes no queryable identifier fails. + + Config: + step_output: The step output containing per-machine serial records. + required_components: Component classes to require identifiers for + (default: chassis, baseboard, cpu, gpu, nic). + min_machines: Minimum number of machines expected (default: 1). + + Step output (from query_serial_numbers.py): + success: bool + platform: "nico" + site_id: str + machines_checked: int + machines: list[dict]: + machine_id: str + components: dict[str, dict]: + : {present: bool, identifiers: list[str]} + """ + + description: ClassVar[str] = "Check stable hardware serial numbers are queryable for installed components" + timeout: ClassVar[int] = 120 + + default_components: ClassVar[tuple[str, ...]] = ("chassis", "baseboard", "cpu", "gpu", "nic") + + def run(self) -> None: + """Validate every present component on every machine exposes a stable identifier.""" + step_output = self.config.get("step_output", {}) + + if not step_output.get("success"): + self.set_failed(f"Serial-number query step failed: {step_output.get('error', 'Unknown error')}") + return + + machines = step_output.get("machines") + if not isinstance(machines, list): + self.set_failed("Serial-number step output is missing the 'machines' list") + return + + min_machines = self._parse_positive_int("min_machines", default=1) + if min_machines is None: + return + + if len(machines) < min_machines: + self.set_failed(f"Expected at least {min_machines} machine(s) with hardware inventory, got {len(machines)}") + return + + required = self.config.get("required_components") or list(self.default_components) + + failed: dict[str, str] = {} + for machine in machines: + label = _machine_label(machine) + components = machine.get("components") or {} + for name in required: + component = components.get(name) or {} + if not component.get("present"): + self.report_subtest( + f"{name}_{label}", + passed=True, + skipped=True, + message=f"{label}: no {name} present on this host", + ) + continue + + identifiers = [i for i in (component.get("identifiers") or []) if i] + if identifiers: + self.report_subtest( + f"{name}_{label}", + passed=True, + message=f"{label}: {name} identifier(s) {', '.join(identifiers)}", + ) + else: + reason = f"{label}: {name} present but no stable identifier is queryable" + self.report_subtest(f"{name}_{label}", passed=False, message=reason) + failed[f"{label}/{name}"] = reason + + total = len(machines) + if failed: + sample = ", ".join(list(failed)[:3]) + more = len(failed) - min(len(failed), 3) + summary = f"{sample} (+{more} more)" if more else sample + self.set_failed(f"Missing hardware identifiers for {len(failed)} component(s): {summary}") + return + + self.set_passed(f"Stable hardware identifiers queryable for all present components on {total} machine(s)") + + class DpuHealthCheck(BaseValidation): """Validate DPU health status for ingested machines. diff --git a/isvtest/src/isvtest/validations/topology.py b/isvtest/src/isvtest/validations/topology.py new file mode 100644 index 00000000..672ef1b2 --- /dev/null +++ b/isvtest/src/isvtest/validations/topology.py @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Physical topology / failure-domain observability validations (STG05). + +Provider-agnostic checks that assert the platform exposes the physical failure +domain each host belongs to, so a tenant can spread compute or storage across +independent failure domains for physical diversity. +""" + +from __future__ import annotations + +from typing import Any, ClassVar + +from isvtest.core.validation import BaseValidation + + +def _host_label(host: dict[str, Any]) -> str: + """Human-facing identifier for a host record.""" + return host.get("host_id") or "unknown" + + +class FailureDomainObservabilityCheck(BaseValidation): + """Validate failure-domain topology is observable per host (STG05-01). + + Asserts that every host reports the failure domain (rack / physical + enclosure) it belongs to, and that the site exposes at least + ``min_failure_domains`` distinct domains. Topology observability is the + prerequisite for physical-diversity placement; raise ``min_failure_domains`` + above 1 in a provider config to additionally enforce that a site actually + spans multiple failure domains. + + Config: + step_output: Step output containing per-host failure-domain records. + min_hosts: Minimum number of hosts expected (default: 1). + min_failure_domains: Minimum distinct failure domains required + (default: 1 -- observability only). + require_all_hosts_mapped: Fail if any host lacks a failure domain + (default: true). + + Step output (from query_topology.py): + success: bool + platform: str + site_id: str + hosts_checked: int + hosts: list[dict]: + host_id: str + failure_domain: str -- rack/enclosure identifier ("" if unknown) + """ + + description: ClassVar[str] = "Check failure-domain topology is observable for physical diversity" + timeout: ClassVar[int] = 120 + + def run(self) -> None: + """Validate each host reports a failure domain and the site meets the diversity floor.""" + step_output = self.config.get("step_output", {}) + + if not step_output.get("success"): + self.set_failed(f"Topology query step failed: {step_output.get('error', 'Unknown error')}") + return + + hosts = step_output.get("hosts") + if not isinstance(hosts, list): + self.set_failed("Topology step output is missing the 'hosts' list") + return + + min_hosts = self._parse_positive_int("min_hosts", default=1) + min_failure_domains = self._parse_positive_int("min_failure_domains", default=1) + if min_hosts is None or min_failure_domains is None: + return + + if len(hosts) < min_hosts: + self.set_failed(f"Expected at least {min_hosts} host(s) with topology data, got {len(hosts)}") + return + + require_all_mapped = self.config.get("require_all_hosts_mapped", True) + + unmapped: list[str] = [] + domains: set[str] = set() + for host in hosts: + label = _host_label(host) + domain = host.get("failure_domain") or "" + if domain: + domains.add(domain) + self.report_subtest(f"failure_domain_{label}", passed=True, message=f"{label}: failure domain {domain}") + else: + unmapped.append(label) + self.report_subtest( + f"failure_domain_{label}", + passed=not require_all_mapped, + message=f"{label}: no failure domain reported", + ) + + if require_all_mapped and unmapped: + sample = ", ".join(unmapped[:3]) + more = len(unmapped) - min(len(unmapped), 3) + summary = f"{sample} (+{more} more)" if more else sample + self.set_failed(f"{len(unmapped)}/{len(hosts)} host(s) report no failure domain: {summary}") + return + + if len(domains) < min_failure_domains: + self.set_failed( + f"Site exposes {len(domains)} failure domain(s), requires at least {min_failure_domains} " + f"for physical diversity" + ) + return + + self.set_passed( + f"Failure-domain topology observable for {len(hosts)} host(s) across {len(domains)} failure domain(s)" + ) diff --git a/isvtest/tests/test_hardware.py b/isvtest/tests/test_hardware.py index 27eb6928..cbdfa00b 100644 --- a/isvtest/tests/test_hardware.py +++ b/isvtest/tests/test_hardware.py @@ -23,6 +23,7 @@ DpuHealthCheck, DpuNetworkCheck, HardwareIngestionCheck, + HardwareSerialCheck, ) # --------------------------------------------------------------------------- @@ -640,3 +641,119 @@ def test_empty_step_output(self) -> None: check = DpuNetworkCheck(config={"step_output": {}}) check.run() assert check._passed is False + + +# --------------------------------------------------------------------------- +# HardwareSerialCheck (BFX03-01) +# --------------------------------------------------------------------------- + + +def _serial_component(present: bool = True, identifiers: list[str] | None = None) -> dict[str, Any]: + """Build one per-component serial record.""" + return {"present": present, "identifiers": identifiers or []} + + +def _serial_machine( + *, + machine_id: str = "m-001", + chassis: list[str] | None = None, + baseboard: list[str] | None = None, + cpu: list[str] | None = None, + gpu_present: bool = True, + gpu: list[str] | None = None, + nic: list[str] | None = None, +) -> dict[str, Any]: + """Build a per-machine serial record with sensible defaults.""" + return { + "machine_id": machine_id, + "components": { + "chassis": _serial_component(True, chassis if chassis is not None else ["J1050ACR"]), + "baseboard": _serial_component(True, baseboard if baseboard is not None else [".C1KS2CS002G."]), + "cpu": _serial_component(True, cpu if cpu is not None else ["Intel Xeon Gold 6354"]), + "gpu": _serial_component(gpu_present, gpu if gpu is not None else ["1654422006434"]), + "nic": _serial_component(True, nic if nic is not None else ["c8:4b:d6:7b:ac:a8"]), + }, + } + + +def _serial_output( + *, + success: bool = True, + machines: list[dict[str, Any]] | None = None, + error: str = "", +) -> dict[str, Any]: + """Build a minimal hardware serial-number step output.""" + if machines is None: + machines = [_serial_machine()] + return { + "success": success, + "platform": "nico", + "site_id": "test-site-001", + "machines_checked": len(machines), + "machines": machines, + "error": error, + } + + +class TestHardwareSerialCheck: + """Tests for HardwareSerialCheck (BFX03-01).""" + + def test_all_identifiers_present_passes(self) -> None: + """A fully-populated inventory passes.""" + check = HardwareSerialCheck(config={"step_output": _serial_output()}) + check.run() + assert check._passed is True + + def test_missing_gpu_is_skipped_not_failed(self) -> None: + """A CPU-only host (no GPU present) passes; GPU is skipped, not failed.""" + machine = _serial_machine(gpu_present=False, gpu=[]) + check = HardwareSerialCheck(config={"step_output": _serial_output(machines=[machine])}) + check.run() + assert check._passed is True + + def test_present_component_without_identifier_fails(self) -> None: + """A present GPU with no queryable serial fails.""" + machine = _serial_machine(gpu_present=True, gpu=[]) + check = HardwareSerialCheck(config={"step_output": _serial_output(machines=[machine])}) + check.run() + assert check._passed is False + assert "gpu" in check._error + + def test_missing_chassis_serial_fails(self) -> None: + """A host with no chassis identifier fails (chassis is always required).""" + machine = _serial_machine(chassis=[]) + check = HardwareSerialCheck(config={"step_output": _serial_output(machines=[machine])}) + check.run() + assert check._passed is False + assert "chassis" in check._error + + def test_required_components_scoping(self) -> None: + """Restricting required_components ignores an unlisted missing identifier.""" + machine = _serial_machine(nic=[]) # NIC has no identifier + check = HardwareSerialCheck( + config={ + "step_output": _serial_output(machines=[machine]), + "required_components": ["chassis", "cpu"], + } + ) + check.run() + assert check._passed is True + + def test_step_failure_fails(self) -> None: + """A failed step output fails the check.""" + check = HardwareSerialCheck(config={"step_output": _serial_output(success=False, error="boom")}) + check.run() + assert check._passed is False + assert "boom" in check._error + + def test_missing_machines_list_fails(self) -> None: + """Step output without a machines list fails.""" + check = HardwareSerialCheck(config={"step_output": {"success": True}}) + check.run() + assert check._passed is False + + def test_min_machines_enforced(self) -> None: + """Fewer machines than min_machines fails.""" + check = HardwareSerialCheck(config={"step_output": _serial_output(machines=[]), "min_machines": 1}) + check.run() + assert check._passed is False diff --git a/isvtest/tests/test_topology.py b/isvtest/tests/test_topology.py new file mode 100644 index 00000000..318e747a --- /dev/null +++ b/isvtest/tests/test_topology.py @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for failure-domain topology observability validations (STG05-01).""" + +from __future__ import annotations + +from typing import Any + +from isvtest.validations.topology import FailureDomainObservabilityCheck + + +def _host(host_id: str, failure_domain: str) -> dict[str, Any]: + """Build one per-host topology record.""" + return {"host_id": host_id, "failure_domain": failure_domain} + + +def _topology_output( + *, + success: bool = True, + hosts: list[dict[str, Any]] | None = None, + error: str = "", +) -> dict[str, Any]: + """Build a minimal topology step output.""" + if hosts is None: + hosts = [_host("m-1", "rack-A"), _host("m-2", "rack-B")] + return { + "success": success, + "platform": "nico", + "site_id": "test-site-001", + "hosts_checked": len(hosts), + "hosts": hosts, + "error": error, + } + + +class TestFailureDomainObservabilityCheck: + """Tests for FailureDomainObservabilityCheck (STG05-01).""" + + def test_all_hosts_mapped_passes(self) -> None: + """Every host reporting a failure domain passes.""" + check = FailureDomainObservabilityCheck(config={"step_output": _topology_output()}) + check.run() + assert check._passed is True + + def test_single_domain_still_observable(self) -> None: + """A single failure domain is observable and passes the default floor of 1.""" + hosts = [_host("m-1", "rack-A"), _host("m-2", "rack-A")] + check = FailureDomainObservabilityCheck(config={"step_output": _topology_output(hosts=hosts)}) + check.run() + assert check._passed is True + + def test_unmapped_host_fails(self) -> None: + """A host with no failure domain fails when all hosts must be mapped.""" + hosts = [_host("m-1", "rack-A"), _host("m-2", "")] + check = FailureDomainObservabilityCheck(config={"step_output": _topology_output(hosts=hosts)}) + check.run() + assert check._passed is False + assert "m-2" in check._error + + def test_unmapped_allowed_when_not_required(self) -> None: + """An unmapped host passes when require_all_hosts_mapped is disabled.""" + hosts = [_host("m-1", "rack-A"), _host("m-2", "")] + check = FailureDomainObservabilityCheck( + config={"step_output": _topology_output(hosts=hosts), "require_all_hosts_mapped": False} + ) + check.run() + assert check._passed is True + + def test_min_failure_domains_enforced(self) -> None: + """A site with fewer distinct domains than required fails.""" + hosts = [_host("m-1", "rack-A"), _host("m-2", "rack-A")] + check = FailureDomainObservabilityCheck( + config={"step_output": _topology_output(hosts=hosts), "min_failure_domains": 2} + ) + check.run() + assert check._passed is False + assert "physical diversity" in check._error + + def test_min_failure_domains_met(self) -> None: + """Two distinct domains satisfy a diversity floor of 2.""" + check = FailureDomainObservabilityCheck(config={"step_output": _topology_output(), "min_failure_domains": 2}) + check.run() + assert check._passed is True + + def test_step_failure_fails(self) -> None: + """A failed step output fails the check.""" + check = FailureDomainObservabilityCheck(config={"step_output": _topology_output(success=False, error="boom")}) + check.run() + assert check._passed is False + assert "boom" in check._error + + def test_missing_hosts_list_fails(self) -> None: + """Step output without a hosts list fails.""" + check = FailureDomainObservabilityCheck(config={"step_output": {"success": True}}) + check.run() + assert check._passed is False + + def test_min_hosts_enforced(self) -> None: + """Fewer hosts than min_hosts fails.""" + check = FailureDomainObservabilityCheck(config={"step_output": _topology_output(hosts=[]), "min_hosts": 1}) + check.run() + assert check._passed is False