From 65928200c8b0bfffbe9e853611a596a175e6ad25 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 13 Jul 2026 21:48:42 +0000 Subject: [PATCH 1/5] feat(nico): add STG02/03/04 storage-infra observability validations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement three NICo bare-metal checks grounded in the public Machine REST API, following the query-script → neutral-JSON → validation pattern from PR #539: - STG02-01 SkipSanitizationBreakfixCheck: extends the SEC21 sanitization audit with tenancy-preserving maintenance skips (in_use -> maintenance -> in_use without Reset). - STG03-01 StableStorageNodeIpCheck: asserts stable admin IPs are queryable via machineInterfaces[].ipAddresses. - STG04-01 OobFailureDetectionCheck: asserts BMC/out-of-band health probes expose failure-detection coverage (BmcSensor baseline). All three ship unreleased; exercise with ISVTEST_INCLUDE_UNRELEASED=1. Closes #358 Closes #359 Closes #360 Signed-off-by: Cursor Agent Co-authored-by: Alexandre Begnoche --- docs/test-plan.yaml | 9 + .../providers/nico/config/bare_metal.yaml | 46 +++- .../nico/scripts/health/query_oob_health.py | 196 +++++++++++++ .../sanitization/query_sanitization.py | 37 +++ .../nico/scripts/storage/query_stable_ips.py | 165 +++++++++++ isvctl/configs/suites/README.md | 4 +- isvctl/configs/suites/bare_metal.yaml | 33 +++ .../providers/nico/test_nico_provider.py | 226 +++++++++++++++ isvtest/src/isvtest/validations/__init__.py | 8 + .../src/isvtest/validations/sanitization.py | 88 ++++++ .../src/isvtest/validations/storage_infra.py | 258 ++++++++++++++++++ isvtest/tests/test_breakfix_sanitization.py | 96 +++++++ isvtest/tests/test_storage_infra.py | 128 +++++++++ 13 files changed, 1290 insertions(+), 4 deletions(-) create mode 100644 isvctl/configs/providers/nico/scripts/health/query_oob_health.py create mode 100644 isvctl/configs/providers/nico/scripts/storage/query_stable_ips.py create mode 100644 isvtest/src/isvtest/validations/storage_infra.py create mode 100644 isvtest/tests/test_breakfix_sanitization.py create mode 100644 isvtest/tests/test_storage_infra.py diff --git a/docs/test-plan.yaml b/docs/test-plan.yaml index ebf7c25b3..e16352165 100644 --- a/docs/test-plan.yaml +++ b/docs/test-plan.yaml @@ -2252,7 +2252,10 @@ domains: tests: - summary: Verify optional skip-sanitization flag during break/fix where tenancy doesn't change labels: + - bare_metal - min_req + - sanitization + - sds_controller priority: P1 milestone: M5 notes: "" @@ -2263,7 +2266,10 @@ domains: status: pending - summary: Verify stable IP assignment for storage nodes across lifecycle operations labels: + - bare_metal - min_req + - network + - sds_controller priority: P1 milestone: M5 notes: "" @@ -2274,7 +2280,10 @@ domains: status: pending - summary: Verify out-of-band failure detection (device, network, memory, drive) labels: + - bare_metal + - health - min_req + - sds_controller priority: P1 milestone: M5 notes: "" diff --git a/isvctl/configs/providers/nico/config/bare_metal.yaml b/isvctl/configs/providers/nico/config/bare_metal.yaml index 228a11d2d..37696bede 100644 --- a/isvctl/configs/providers/nico/config/bare_metal.yaml +++ b/isvctl/configs/providers/nico/config/bare_metal.yaml @@ -44,9 +44,15 @@ # - query_sanitization.py maps each machine's status + statusHistory into a # neutral lifecycle token sequence; MemorySanitizationCheck (SEC21-04), # GpuMemorySanitizationCheck (SEC21-05), FirmwareResetCheck (SEC21-06/SEC22), -# and DiskSanitizationCheck (SEC21-02) assert hosts are sanitized between -# tenants. The same Reset stage performs the NVMe/HDD secure erase, so the -# disk check gates on the same lifecycle signal as the memory check. +# DiskSanitizationCheck (SEC21-02), and SkipSanitizationBreakfixCheck +# (STG02-01) assert hosts are sanitized between tenants and that optional +# maintenance skips preserve tenancy. The same Reset stage performs the +# NVMe/HDD secure erase, so the disk check gates on the same lifecycle signal. +# - query_stable_ips.py reads machineInterfaces[].ipAddresses; StableStorageNodeIpCheck +# (STG03-01) asserts static admin IPs are queryable per host. +# - query_oob_health.py maps BMC health probes into STG04 failure-category +# observability; OobFailureDetectionCheck (STG04-01) asserts out-of-band +# failure detection is exposed via the per-host health API. # - query_attestation.py maps control-plane SPDM and measured-boot status from # nico-admin-cli into the provider-neutral attestation contract (SEC22-01 / # CNP09-02). It requires operator/admin CLI credentials in addition to the @@ -253,6 +259,40 @@ commands: - "{{nico_api_base}}" timeout: 120 + # ─── Query Stable Admin IPs (STG03-01) ───────────────────────── + # Reads each machine's machineInterfaces[].ipAddresses (primary + # interface first) so StableStorageNodeIpCheck can assert static admin + # IPs are queryable across lifecycle operations. + - name: query_stable_ips + phase: test + continue_on_failure: true + command: "python ../scripts/storage/query_stable_ips.py" + args: + - "--org" + - "{{org}}" + - "--site-id" + - "{{site_id}}" + - "--api-base" + - "{{nico_api_base}}" + timeout: 120 + + # ─── Query Out-of-Band Failure Detection (STG04-01) ──────────── + # Maps each machine's BMC health probes into STG04 failure-category + # observability (device / network / memory / drive). OobFailureDetectionCheck + # asserts the OOB health API is present and exposes BMC sensor coverage. + - name: query_oob_health + phase: test + continue_on_failure: true + command: "python ../scripts/health/query_oob_health.py" + args: + - "--org" + - "{{org}}" + - "--site-id" + - "{{site_id}}" + - "--api-base" + - "{{nico_api_base}}" + timeout: 120 + # ─── Query Attestation (SEC22-01 / CNP09-02) ──────────────────── # Enumerates site machines via tenant REST, then reads SPDM attestation # and measured-boot status from the Forge control-plane API via diff --git a/isvctl/configs/providers/nico/scripts/health/query_oob_health.py b/isvctl/configs/providers/nico/scripts/health/query_oob_health.py new file mode 100644 index 000000000..36eacb8ef --- /dev/null +++ b/isvctl/configs/providers/nico/scripts/health/query_oob_health.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Query out-of-band (BMC) failure-detection coverage for NICo machines (STG04-01). + +STG04 requires the platform to detect hardware failures out-of-band — device, +network, memory, and drive issues surfaced before or without relying on the +tenant OS. NICo aggregates BMC sensor data into the machine ``health`` report +(``BmcSensor``, ``BmcLeakDetection``, ...). This script maps those probes into +provider-neutral per-category observability records so +``OobFailureDetectionCheck`` can assert the OOB health API is present and +covers the required failure classes. + +NICo API endpoints used: + GET /v2/org/{org}/carbide/machine?siteId={site_id}&includeMetadata=true + +Auth: + - NICO_BEARER_TOKEN, or + - OIDC client_credentials via NICO_SSA_ISSUER, + NICO_CLIENT_ID, NICO_CLIENT_SECRET, and optional NICO_OIDC_SCOPE. + +Required JSON output fields: + { + "success": true, + "platform": "nico", + "site_id": "...", + "hosts_checked": 1, + "hosts": [ + { + "host_id": "...", + "oob_health_present": true, + "bmc_probe_ids": ["BmcSensor"], + "failure_categories": { + "device": {"observable": true, "probe_ids": ["BmcSensor"]}, + "network": {"observable": false, "probe_ids": []}, + "memory": {"observable": false, "probe_ids": []}, + "drive": {"observable": false, "probe_ids": []} + } + } + ] + } + +A site with no ingested machines emits a structured skip (``skipped`` / +``skip_reason``) so the validation does not hard-fail a site with no hardware +discovered yet. + +Usage: + NICO_BEARER_TOKEN= python query_oob_health.py \ + --org --site-id --api-base + + Wired via the bare_metal suite: + uv run isvctl test run -f isvctl/configs/providers/nico/config/bare_metal.yaml + +Reference: + Probe IDs: infra-controller docs/architecture/health/health_probe_ids.md + OpenAPI spec: rest-api/openapi/spec.yaml (MachineHealth / MachineHealthProbe*) +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +# Allow importing from sibling common/ directory +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from common.nico_client import NicoAuthError, forge_get_all, resolve_auth + +# BMC / out-of-band probe identifiers NICo reports on the machine health API. +BMC_PROBE_PREFIXES = ("bmc",) + +# Failure classes STG04-01 calls out, mapped by probe id/target/message keywords. +FAILURE_CATEGORY_KEYWORDS: dict[str, tuple[str, ...]] = { + "device": ("power", "fan", "temp", "thermal", "voltage", "gpu", "cpu", "sensor", "psu"), + "network": ("nic", "link", "ethernet", "infiniband", "network", "bgp", "mlx"), + "memory": ("memory", "ecc", "dimm", "rowremap", "row_remap", "hbm", "sram"), + "drive": ("disk", "nvme", "hdd", "storage", "drive", "boss", "raid"), +} + + +def _probe_text(probe: dict[str, Any]) -> str: + """Return lowercased probe id + target + message for keyword matching.""" + parts = [probe.get("id"), probe.get("target"), probe.get("message")] + return " ".join(p for p in parts if isinstance(p, str)).lower() + + +def _is_bmc_probe(probe: dict[str, Any]) -> bool: + """Return whether a probe is BMC/out-of-band sourced.""" + probe_id = str(probe.get("id") or "").lower() + return probe_id.startswith(BMC_PROBE_PREFIXES) + + +def _category_observability(health: dict[str, Any]) -> dict[str, dict[str, Any]]: + """Map NICo health probes into STG04 failure-category observability.""" + probes = [p for p in (*(health.get("successes") or []), *(health.get("alerts") or [])) if isinstance(p, dict)] + probe_texts = [(probe, _probe_text(probe)) for probe in probes] + + categories: dict[str, dict[str, Any]] = {} + for category, keywords in FAILURE_CATEGORY_KEYWORDS.items(): + matched_ids = sorted( + { + str(probe.get("id") or "") + for probe, text in probe_texts + if probe.get("id") and any(keyword in text for keyword in keywords) + } + ) + categories[category] = { + "observable": bool(matched_ids), + "probe_ids": matched_ids, + } + return categories + + +def host_record(machine: dict[str, Any]) -> dict[str, Any]: + """Build the provider-neutral OOB failure-detection record for one machine.""" + health = machine.get("health") or {} + probes = [p for p in (*(health.get("successes") or []), *(health.get("alerts") or [])) if isinstance(p, dict)] + + bmc_probe_ids = sorted({str(p.get("id") or "") for p in probes if p.get("id") and _is_bmc_probe(p)}) + oob_present = bool(bmc_probe_ids or health.get("observedAt")) + + return { + "host_id": machine.get("id", ""), + "oob_health_present": oob_present, + "bmc_probe_ids": bmc_probe_ids, + "failure_categories": _category_observability(health), + } + + +def main() -> int: + """Query NICo machine health and print per-host OOB failure-detection JSON.""" + parser = argparse.ArgumentParser(description="Query NICo out-of-band failure-detection coverage") + parser.add_argument("--org", required=True, help="NGC org name") + parser.add_argument("--site-id", required=True, help="NICo site UUID") + parser.add_argument("--api-base", required=True, help="NICo API base URL") + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "nico", + "site_id": args.site_id, + "hosts_checked": 0, + "hosts": [], + } + + try: + auth = resolve_auth() + + machines = forge_get_all( + args.org, + "machine", + auth.token, + base_url=args.api_base, + params={"siteId": args.site_id, "includeMetadata": "true"}, + result_key="machines", + ) + + if not machines: + result["success"] = True + result["skipped"] = True + result["skip_reason"] = "No machines found at site; no hosts to report OOB failure detection for" + print(json.dumps(result, indent=2)) + return 0 + + result["hosts"] = [host_record(machine) for machine in machines] + result["hosts_checked"] = len(result["hosts"]) + result["success"] = True + + except NicoAuthError as e: + result["error_type"] = "auth" + result["error"] = str(e) + except Exception as e: + result["error"] = f"{type(e).__name__}: {e}" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/providers/nico/scripts/sanitization/query_sanitization.py b/isvctl/configs/providers/nico/scripts/sanitization/query_sanitization.py index 57436abf8..0b472d802 100644 --- a/isvctl/configs/providers/nico/scripts/sanitization/query_sanitization.py +++ b/isvctl/configs/providers/nico/scripts/sanitization/query_sanitization.py @@ -106,12 +106,14 @@ IN_USE = "in_use" SANITIZING = "sanitizing" AVAILABLE = "available" +MAINTENANCE = "maintenance" # NICo MachineStatus -> neutral token. Statuses not listed are lowercased. STATUS_TOKENS: dict[str, str] = { "InUse": IN_USE, "Reset": SANITIZING, "Ready": AVAILABLE, + "Maintenance": MAINTENANCE, } # Cap the diagnostic transitions list so a long-lived host does not bloat output. @@ -144,6 +146,32 @@ def ordered_history_statuses(machine: dict[str, Any]) -> list[str]: return statuses +def breakfix_skip_observed(tokens: list[str]) -> bool: + """Return whether the lifecycle shows a tenancy-preserving maintenance skip. + + A valid break/fix skip is ``in_use -> maintenance -> in_use`` with no + intervening ``sanitizing`` stage. Pool maintenance (``maintenance`` while + the host was never ``in_use``) does not count. + """ + active_tenancy = False + maintenance_after_tenancy = False + for token in tokens: + if token == IN_USE: + if maintenance_after_tenancy: + return True + active_tenancy = True + maintenance_after_tenancy = False + elif token == MAINTENANCE and active_tenancy: + maintenance_after_tenancy = True + elif token == SANITIZING: + active_tenancy = False + maintenance_after_tenancy = False + elif token == AVAILABLE: + active_tenancy = False + maintenance_after_tenancy = False + return False + + def evaluate_transitions(tokens: list[str]) -> tuple[bool, bool]: """Return ``(served_tenant, sanitized)`` from a neutral token sequence. @@ -183,14 +211,23 @@ def machine_record(machine: dict[str, Any]) -> dict[str, Any]: dmi = (machine.get("metadata") or {}).get("dmiData") or {} + skip_observed = breakfix_skip_observed(tokens) + instance_bound = bool(machine.get("instanceId")) or bool(machine.get("tenantId")) + # Tenancy is preserved when the host is still (or again) bound after a + # maintenance skip, or is actively in Maintenance while assigned. + tenancy_preserved = not skip_observed or instance_bound or current in {"InUse", "Maintenance"} + return { "machine_id": machine.get("id", ""), "status": status_token(current), "available": is_ready and usable and not assigned, "in_use": current == "InUse", + "instance_bound": instance_bound, "has_gpu": has_gpu(machine), "served_tenant": served, "sanitized": sanitized, + "breakfix_skip_observed": skip_observed, + "tenancy_preserved": tenancy_preserved, # Offered to new tenants (Ready + usable) while still bound to a prior # tenant's instance -- a hard sanitization failure if it ever occurs. "stale_tenant_binding": is_ready and usable and assigned, diff --git a/isvctl/configs/providers/nico/scripts/storage/query_stable_ips.py b/isvctl/configs/providers/nico/scripts/storage/query_stable_ips.py new file mode 100644 index 000000000..6e9660faf --- /dev/null +++ b/isvctl/configs/providers/nico/scripts/storage/query_stable_ips.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Query stable admin IP assignments for NICo machines (STG03-01). + +Storage nodes require static IP addressing that remains stable across host +lifecycle operations and maintenance events. NICo assigns admin IPs through its +IPAM service and exposes them on each machine's ``machineInterfaces`` array. +This script reads the per-interface ``ipAddresses`` (preferring the primary +admin interface) and emits a provider-neutral record so +``StableStorageNodeIpCheck`` can assert every host reports at least one stable +admin IP. + +NICo API endpoints used: + GET /v2/org/{org}/carbide/machine?siteId={site_id} + +Auth: + - NICO_BEARER_TOKEN, or + - OIDC client_credentials via NICO_SSA_ISSUER, + NICO_CLIENT_ID, NICO_CLIENT_SECRET, and optional NICO_OIDC_SCOPE. + +Required JSON output fields: + { + "success": true, + "platform": "nico", + "site_id": "...", + "hosts_checked": 1, + "hosts": [ + { + "host_id": "...", + "hw_sku_device_type": "storage", + "primary_ip_addresses": ["192.156.7.23"], + "has_stable_ip": true + } + ] + } + +A site with no ingested machines emits a structured skip (``skipped`` / +``skip_reason``) so the validation does not hard-fail a site with no hardware +discovered yet. + +Usage: + NICO_BEARER_TOKEN= python query_stable_ips.py \ + --org --site-id --api-base + + Wired via the bare_metal suite: + uv run isvctl test run -f isvctl/configs/providers/nico/config/bare_metal.yaml + +Reference: + OpenAPI spec: rest-api/openapi/spec.yaml (Machine.machineInterfaces / + MachineInterface.ipAddresses / MachineInterface.isPrimary) +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +# Allow importing from sibling common/ directory +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from common.nico_client import NicoAuthError, forge_get_all, resolve_auth + + +def _dedupe_ips(addresses: list[str]) -> list[str]: + """Drop blank values and duplicates while preserving first-seen order.""" + seen: dict[str, None] = {} + for address in addresses: + cleaned = address.strip() if isinstance(address, str) else "" + if cleaned and cleaned not in seen: + seen[cleaned] = None + return list(seen) + + +def _interface_ips(iface: dict[str, Any]) -> list[str]: + """Return non-empty IP addresses reported on one machine interface.""" + return _dedupe_ips([ip for ip in (iface.get("ipAddresses") or []) if isinstance(ip, str)]) + + +def host_record(machine: dict[str, Any]) -> dict[str, Any]: + """Build the provider-neutral stable-IP record for one NICo machine.""" + interfaces = [iface for iface in (machine.get("machineInterfaces") or []) if isinstance(iface, dict)] + primary = [iface for iface in interfaces if iface.get("isPrimary")] + targets = primary or interfaces + + primary_ips: list[str] = [] + for iface in targets: + primary_ips.extend(_interface_ips(iface)) + primary_ips = _dedupe_ips(primary_ips) + + return { + "host_id": machine.get("id", ""), + "hw_sku_device_type": machine.get("hwSkuDeviceType") or "", + "primary_ip_addresses": primary_ips, + "has_stable_ip": bool(primary_ips), + } + + +def main() -> int: + """Query NICo machines and print per-host stable admin IP JSON.""" + parser = argparse.ArgumentParser(description="Query NICo stable machine admin IPs") + parser.add_argument("--org", required=True, help="NGC org name") + parser.add_argument("--site-id", required=True, help="NICo site UUID") + parser.add_argument("--api-base", required=True, help="NICo API base URL") + args = parser.parse_args() + + result: dict[str, Any] = { + "success": False, + "platform": "nico", + "site_id": args.site_id, + "hosts_checked": 0, + "hosts": [], + } + + try: + auth = resolve_auth() + + machines = forge_get_all( + args.org, + "machine", + auth.token, + base_url=args.api_base, + params={"siteId": args.site_id}, + result_key="machines", + ) + + if not machines: + result["success"] = True + result["skipped"] = True + result["skip_reason"] = "No machines found at site; no hosts to report stable admin IPs for" + print(json.dumps(result, indent=2)) + return 0 + + result["hosts"] = [host_record(machine) for machine in machines] + result["hosts_checked"] = len(result["hosts"]) + result["success"] = True + + except NicoAuthError as e: + result["error_type"] = "auth" + result["error"] = str(e) + except Exception as e: + result["error"] = f"{type(e).__name__}: {e}" + + print(json.dumps(result, indent=2)) + return 0 if result["success"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index da9ca3dff..a1589d005 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -124,7 +124,9 @@ For the domain / script-count / AWS-reference overview see the | `query_health_aggregation` | test | `providers/nico/scripts/health/query_health_aggregation.py` | `aggregation_level`, `groups[].{total,healthy,unhealthy,status,unhealthy_hosts}` | | `query_ib_tenant_isolation` | test | `providers/nico/scripts/infiniband/query_ib_tenant_isolation.py` | `partitions_checked`, `partitions[].{name,partition_key,tenant_id,status}` | | `query_ib_keys` | test | `providers/nico/scripts/infiniband/query_ib_keys.py` | `partitions_with_pkey`, `keys..{configured,source,detail}` | -| `query_sanitization` | test | `providers/nico/scripts/sanitization/query_sanitization.py` | `machines_checked`, `machines[].{available,in_use,has_gpu,served_tenant,sanitized,stale_tenant_binding,vendor,product_name,bios_version,transitions}` | +| `query_sanitization` | test | `providers/nico/scripts/sanitization/query_sanitization.py` | `machines_checked`, `machines[].{available,in_use,instance_bound,has_gpu,served_tenant,sanitized,breakfix_skip_observed,tenancy_preserved,stale_tenant_binding,vendor,product_name,bios_version,transitions}` | +| `query_stable_ips` | test | `providers/nico/scripts/storage/query_stable_ips.py` | `hosts_checked`, `hosts[].{host_id,hw_sku_device_type,primary_ip_addresses,has_stable_ip}` | +| `query_oob_health` | test | `providers/nico/scripts/health/query_oob_health.py` | `hosts_checked`, `hosts[].{oob_health_present,bmc_probe_ids,failure_categories..{observable,probe_ids}}` | | `query_attestation` | test | `providers/nico/scripts/attestation/query_attestation.py` | `machines_checked`, `machines[].{attestation_supported,nonce_verified,attestation_signature_valid,secure_boot_enabled,boot_measurements_attested,measured_boot_state}` | | `query_serial_numbers` | test | `providers/nico/scripts/hardware_inventory/query_serial_numbers.py` | `machines_checked`, `machines[].components.{chassis,baseboard,cpu,gpu,nic}.{present,identifiers}` | | `query_topology` | test | `providers/nico/scripts/topology/query_topology.py` | `hosts_checked`, `hosts[].{host_id,failure_domain}` | diff --git a/isvctl/configs/suites/bare_metal.yaml b/isvctl/configs/suites/bare_metal.yaml index 0901e9eeb..a1c030da3 100644 --- a/isvctl/configs/suites/bare_metal.yaml +++ b/isvctl/configs/suites/bare_metal.yaml @@ -599,6 +599,39 @@ tests: labels: ["bare_metal", "sds_controller", "min_req"] min_failure_domains: 1 + # Break/fix skip-sanitization policy (STG02-01). Extends the SEC21 + # tenant-transition audit: a host may skip the sanitizing (Reset) stage when + # it enters maintenance while still serving the same tenant. Tenant releases + # must still pass through sanitizing before returning to the pool. Only runs + # for providers that implement the query_sanitization step. + breakfix_skip_sanitization: + step: query_sanitization + checks: + SkipSanitizationBreakfixCheck: + test_id: "STG02-01" + labels: ["bare_metal", "min_req", "sanitization", "sds_controller"] + + # Stable admin IP observability (STG03-01). Asserts each host reports at + # least one IP on its primary machineInterfaces entry so storage nodes have + # queryable static addressing across lifecycle operations. Only runs for + # providers that implement the query_stable_ips step. + stable_storage_ips: + step: query_stable_ips + checks: + StableStorageNodeIpCheck: + test_id: "STG03-01" + labels: ["bare_metal", "min_req", "network", "sds_controller"] + + # Out-of-band failure detection (STG04-01). Asserts the per-host health API + # exposes BMC/out-of-band probes covering device failures at minimum. Only + # runs for providers that implement the query_oob_health step. + oob_failure_detection: + step: query_oob_health + checks: + OobFailureDetectionCheck: + test_id: "STG04-01" + labels: ["bare_metal", "health", "min_req", "sds_controller"] + # Hardware/firmware attestation (SEC22 / CNP09). Each check asserts a host # establishes a hardware root of trust. Only runs for providers that # implement the query_attestation step. diff --git a/isvctl/tests/providers/nico/test_nico_provider.py b/isvctl/tests/providers/nico/test_nico_provider.py index 4a1eedf28..26cae593c 100644 --- a/isvctl/tests/providers/nico/test_nico_provider.py +++ b/isvctl/tests/providers/nico/test_nico_provider.py @@ -39,8 +39,10 @@ DiskSanitizationCheck, GpuMemorySanitizationCheck, MemorySanitizationCheck, + SkipSanitizationBreakfixCheck, ) from isvtest.validations.topology import FailureDomainObservabilityCheck +from isvtest.validations.storage_infra import OobFailureDetectionCheck, StableStorageNodeIpCheck from isvctl.config.merger import merge_yaml_files from isvctl.config.schema import RunConfig @@ -354,6 +356,8 @@ def fake_urlopen(request: Any, timeout: int = 30) -> _Response: "query_ib_tenant_isolation", "query_ib_keys", "query_sanitization", + "query_stable_ips", + "query_oob_health", ], ) def test_nico_bare_metal_config_exposes_api_base_setting(step_name: str) -> None: @@ -1003,6 +1007,8 @@ def test_nico_network_inventory_scripts_skip_when_site_has_no_network_inventory( ("query_ib_tenant_isolation.py", _load_ib_tenant_isolation_script), ("query_ib_keys.py", _load_ib_keys_script), ("query_sanitization.py", _load_sanitization_script), + ("query_stable_ips.py", lambda: _load_nico_script("storage/query_stable_ips.py", "test_query_stable_ips_api")), + ("query_oob_health.py", lambda: _load_nico_script("health/query_oob_health.py", "test_query_oob_health_api")), ], ) def test_nico_scripts_require_api_base( @@ -2595,6 +2601,225 @@ def test_sanitization_script_output_satisfies_disk_check( assert "without sanitization" in sub["message"] +def test_sanitization_breakfix_skip_detection( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """InUse -> Maintenance -> InUse without Reset is a tenancy-preserving skip.""" + machine = _sanitization_machine( + status="InUse", + history_statuses=["Ready", "InUse", "Maintenance", "InUse"], + instance_id="59bdaaff-3998-4fd9-a140-8749beeb605e", + is_usable=False, + ) + payload = _run_sanitization(monkeypatch, capsys, [machine]) + + record = payload["machines"][0] + assert record["breakfix_skip_observed"] is True + assert record["tenancy_preserved"] is True + assert record["instance_bound"] is True + + +def test_sanitization_breakfix_skip_output_satisfies_check( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """End-to-end: valid maintenance skip passes; unsanitized tenant release fails.""" + good = _run_sanitization( + monkeypatch, + capsys, + [ + _sanitization_machine( + status="InUse", + history_statuses=["Ready", "InUse", "Maintenance", "InUse"], + instance_id="59bdaaff-3998-4fd9-a140-8749beeb605e", + is_usable=False, + ) + ], + ) + check = SkipSanitizationBreakfixCheck(config={"step_output": good}) + check.run() + assert check._passed is True, check._error + assert "maintenance skip" in check._output + + dirty = _run_sanitization(monkeypatch, capsys, [_sanitization_machine(history_statuses=["InUse", "Ready"])]) + bad = SkipSanitizationBreakfixCheck(config={"step_output": dirty}) + bad.run() + assert bad._passed is False + + +# --------------------------------------------------------------------------- +# query_stable_ips (STG03-01) script +# --------------------------------------------------------------------------- + + +def _load_stable_ips_script() -> ModuleType: + """Load the query_stable_ips script as a module for direct unit testing.""" + return _load_nico_script("storage/query_stable_ips.py", "test_query_stable_ips") + + +def _stable_ip_machine( + machine_id: str = "m-1", + *, + hw_sku_device_type: str = "storage", + interfaces: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Build a raw NICo machine payload with admin interface IPs.""" + if interfaces is None: + interfaces = [ + { + "id": "iface-1", + "isPrimary": True, + "ipAddresses": ["192.156.7.23", "202.88.37.112"], + "macAddress": "00:00:5e:00:53:af", + } + ] + return { + "id": machine_id, + "hwSkuDeviceType": hw_sku_device_type, + "machineInterfaces": interfaces, + } + + +def _run_stable_ips( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + machines: list[dict[str, Any]], +) -> dict[str, Any]: + """Drive the stable IP script with mocked auth/API and return its JSON output.""" + module = _load_stable_ips_script() + return _run_script(module, monkeypatch, capsys, script_name="query_stable_ips.py", machines=machines) + + +def test_stable_ips_script_reports_primary_addresses( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Primary interface IPs are emitted in provider-neutral form.""" + payload = _run_stable_ips(monkeypatch, capsys, [_stable_ip_machine()]) + + assert payload["success"] is True + host = payload["hosts"][0] + assert host["has_stable_ip"] is True + assert host["primary_ip_addresses"] == ["192.156.7.23", "202.88.37.112"] + + +def test_stable_ips_script_empty_site_skips( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """A site with no machines emits a structured skip.""" + payload = _run_stable_ips(monkeypatch, capsys, []) + + assert payload["success"] is True + assert payload["skipped"] is True + assert "No machines found" in payload["skip_reason"] + + +def test_stable_ips_script_output_satisfies_check( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """End-to-end: hosts with IPs pass; a host with no IPs fails.""" + good = _run_stable_ips(monkeypatch, capsys, [_stable_ip_machine()]) + check = StableStorageNodeIpCheck(config={"step_output": good}) + check.run() + assert check._passed is True, check._error + + no_ip = _stable_ip_machine(interfaces=[{"id": "iface-1", "isPrimary": True, "ipAddresses": []}]) + bad_payload = _run_stable_ips(monkeypatch, capsys, [no_ip]) + bad = StableStorageNodeIpCheck(config={"step_output": bad_payload}) + bad.run() + assert bad._passed is False + assert "m-1" in bad._error + + +# --------------------------------------------------------------------------- +# query_oob_health (STG04-01) script +# --------------------------------------------------------------------------- + + +def _load_oob_health_script() -> ModuleType: + """Load the query_oob_health script as a module for direct unit testing.""" + return _load_nico_script("health/query_oob_health.py", "test_query_oob_health") + + +def _oob_machine( + machine_id: str = "m-1", + *, + successes: list[dict[str, Any]] | None = None, + alerts: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Build a raw NICo machine payload with BMC health probes.""" + if successes is None: + successes = [ + {"id": "BmcSensor", "target": "CPU1 Temp", "message": "temperature"}, + {"id": "BmcSensor", "target": "PS1 Status", "message": "power_supply"}, + ] + return { + "id": machine_id, + "health": { + "source": "aggregate-host-health", + "observedAt": "2026-07-13T12:00:00Z", + "successes": successes, + "alerts": alerts or [], + }, + } + + +def _run_oob_health( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + machines: list[dict[str, Any]], +) -> dict[str, Any]: + """Drive the OOB health script with mocked auth/API and return its JSON output.""" + module = _load_oob_health_script() + return _run_script(module, monkeypatch, capsys, script_name="query_oob_health.py", machines=machines) + + +def test_oob_health_script_maps_bmc_categories( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """BMC sensor probes surface device-category observability.""" + payload = _run_oob_health(monkeypatch, capsys, [_oob_machine()]) + + host = payload["hosts"][0] + assert host["oob_health_present"] is True + assert "BmcSensor" in host["bmc_probe_ids"] + assert host["failure_categories"]["device"]["observable"] is True + + +def test_oob_health_script_empty_site_skips( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """A site with no machines emits a structured skip.""" + payload = _run_oob_health(monkeypatch, capsys, []) + + assert payload["success"] is True + assert payload["skipped"] is True + assert "No machines found" in payload["skip_reason"] + + +def test_oob_health_script_output_satisfies_check( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """End-to-end: BMC coverage passes; missing BmcSensor fails.""" + good = _run_oob_health(monkeypatch, capsys, [_oob_machine()]) + check = OobFailureDetectionCheck(config={"step_output": good}) + check.run() + assert check._passed is True, check._error + + no_bmc = _oob_machine(successes=[{"id": "BgpDaemonEnabled", "target": None}]) + bad_payload = _run_oob_health(monkeypatch, capsys, [no_bmc]) + bad = OobFailureDetectionCheck(config={"step_output": bad_payload}) + bad.run() + assert bad._passed is False + assert "BmcSensor" in bad._error + + # --------------------------------------------------------------------------- # query_serial_numbers (BFX03-01) script # --------------------------------------------------------------------------- @@ -2805,3 +3030,4 @@ def test_topology_script_output_satisfies_check( bad.run() assert bad._passed is False assert "m-1" in bad._error + diff --git a/isvtest/src/isvtest/validations/__init__.py b/isvtest/src/isvtest/validations/__init__.py index 81324934f..4a817cf29 100644 --- a/isvtest/src/isvtest/validations/__init__.py +++ b/isvtest/src/isvtest/validations/__init__.py @@ -123,6 +123,7 @@ FirmwareResetCheck, GpuMemorySanitizationCheck, MemorySanitizationCheck, + SkipSanitizationBreakfixCheck, ) from isvtest.validations.security import ( ApiEndpointIsolationCheck, @@ -147,6 +148,10 @@ TenantIsolationCheck, VirtualDeviceHardeningCheck, ) +from isvtest.validations.storage_infra import ( + OobFailureDetectionCheck, + StableStorageNodeIpCheck, +) __all__ = [ "AccessKeyAuthenticatedCheck", @@ -212,6 +217,7 @@ "NonceAttestationCheck", "NvlinkDomainCheck", "OidcUserAuthCheck", + "OobFailureDetectionCheck", "PerformanceCheck", "SchemaValidation", "SdnFilterAuditTrailCheck", @@ -227,8 +233,10 @@ "SgSubnetScopingCheck", "SgWorkloadScopingCheck", "ShortLivedCredentialsCheck", + "SkipSanitizationBreakfixCheck", "StableIdentifierCheck", "StablePrivateIpCheck", + "StableStorageNodeIpCheck", "StepSuccessCheck", "StorageL3RoutingCheck", "SubnetConfigCheck", diff --git a/isvtest/src/isvtest/validations/sanitization.py b/isvtest/src/isvtest/validations/sanitization.py index aa645864b..3f4d261f4 100644 --- a/isvtest/src/isvtest/validations/sanitization.py +++ b/isvtest/src/isvtest/validations/sanitization.py @@ -157,6 +157,94 @@ def run(self) -> None: self.set_passed(f"{self.subject} verified on {total} machine(s) ({served} with a prior tenancy audited)") +class SkipSanitizationBreakfixCheck(_TenantSanitizationCheck): + """Validate optional skip-sanitization during break/fix preserves tenancy (STG02-01). + + Extends the SEC21 tenant-transition audit with the STG02 break/fix policy: a + host may skip the sanitizing (``Reset``) stage when it enters ``maintenance`` + while still serving the same tenant (``in_use -> maintenance -> in_use`` with + ``instanceId`` / ``tenantId`` preserved). Tenant releases must still pass + through the sanitizing stage before returning to the allocatable pool. + + Config: + step_output: Step output containing per-machine sanitization records + (see ``MemorySanitizationCheck``), plus ``breakfix_skip_observed``, + ``tenancy_preserved``, and ``instance_bound``. + + Step output (from query_sanitization.py): + machines[].breakfix_skip_observed: bool -- maintenance skip without Reset + machines[].tenancy_preserved: bool -- tenant binding intact after skip + machines[].instance_bound: bool -- host currently bound to an instance + """ + + catalog_exclude: ClassVar[bool] = False + description: ClassVar[str] = "Check optional skip-sanitization during break/fix preserves tenancy" + subject: ClassVar[str] = "Break/fix skip-sanitization policy" + subtest_prefix: ClassVar[str] = "breakfix" + + def run(self) -> None: + """Audit tenant-transition sanitization and any tenancy-preserving maintenance skips.""" + step_output = self.config.get("step_output", {}) + + if not step_output.get("success"): + self.set_failed(f"Sanitization step failed: {step_output.get('error', 'Unknown error')}") + return + + machines = step_output.get("machines") + if not isinstance(machines, list): + self.set_failed("Sanitization step output is missing the 'machines' list") + return + + if not machines: + self.set_failed("No machines found in step output") + return + + failed: dict[str, str] = {} + breakfix_events = 0 + + for machine in machines: + label = _machine_label(machine) + + passed, message = evaluate_sanitization(machine) + self.report_subtest(f"tenant_transition_{label}", passed=passed, message=message) + if not passed: + failed[label] = message + continue + + if machine.get("breakfix_skip_observed"): + breakfix_events += 1 + if machine.get("tenancy_preserved"): + self.report_subtest( + f"breakfix_skip_{label}", + passed=True, + message=f"{label}: tenancy-preserving maintenance skip observed", + ) + else: + reason = f"{label}: maintenance skip observed but tenancy was not preserved" + self.report_subtest(f"breakfix_skip_{label}", passed=False, message=reason) + failed[label] = reason + + total = len(machines) + if failed: + sample = ", ".join(list(failed)[:3]) + more = len(failed) - min(len(failed), 3) + summary = f"{sample} (+{more} more)" if more else sample + self.set_failed(f"Break/fix sanitization policy failed for {len(failed)}/{total} machine(s): {summary}") + return + + if breakfix_events: + summary = ( + f"Break/fix skip-sanitization policy verified on {total} machine(s) " + f"({breakfix_events} tenancy-preserving maintenance skip(s) observed)" + ) + else: + summary = ( + f"Break/fix skip-sanitization policy auditable on {total} machine(s) " + "(no tenancy-preserving maintenance skips in history yet)" + ) + self.set_passed(summary) + + class MemorySanitizationCheck(_TenantSanitizationCheck): """Validate host memory is sanitized between tenants (SEC21-04). diff --git a/isvtest/src/isvtest/validations/storage_infra.py b/isvtest/src/isvtest/validations/storage_infra.py new file mode 100644 index 000000000..fbb4180a9 --- /dev/null +++ b/isvtest/src/isvtest/validations/storage_infra.py @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Storage-infrastructure observability validations (STG03 / STG04). + +Provider-agnostic checks for NICo bare-metal storage controller requirements +that are observable through the machine REST API without a live SDS backend. +""" + +from __future__ import annotations + +from typing import Any, ClassVar + +from isvtest.core.validation import BaseValidation + + +def _host_label(host: dict[str, Any]) -> str: + """Human-facing identifier for a host record.""" + return host.get("host_id") or "unknown" + + +class StableStorageNodeIpCheck(BaseValidation): + """Validate stable admin IP assignment is queryable per host (STG03-01). + + Asserts every host reports at least one non-empty IP on its primary admin + interface (``machineInterfaces[].ipAddresses`` on the primary interface, or + the first interface when no primary is flagged). This is the provider-neutral + signal that static IP assignments are visible and stable across lifecycle + operations. + + Config: + step_output: Step output containing per-host stable IP records. + min_hosts: Minimum number of hosts expected (default: 1). + storage_only: When true, only hosts whose ``hw_sku_device_type`` is + ``storage`` are in scope (default: false -- all ingested hosts). + + Step output (from query_stable_ips.py): + success: bool + platform: str + site_id: str + hosts_checked: int + hosts: list[dict]: + host_id: str + hw_sku_device_type: str + primary_ip_addresses: list[str] + has_stable_ip: bool + """ + + description: ClassVar[str] = "Check stable admin IP assignment is queryable for storage nodes" + timeout: ClassVar[int] = 120 + + def run(self) -> None: + """Validate each in-scope host exposes at least one stable admin IP.""" + step_output = self.config.get("step_output", {}) + + if not step_output.get("success"): + self.set_failed(f"Stable IP query step failed: {step_output.get('error', 'Unknown error')}") + return + + hosts = step_output.get("hosts") + if not isinstance(hosts, list): + self.set_failed("Stable IP step output is missing the 'hosts' list") + return + + min_hosts = self._parse_positive_int("min_hosts", default=1) + if min_hosts is None: + return + + storage_only = self.config.get("storage_only", False) + scoped = [h for h in hosts if not storage_only or (h.get("hw_sku_device_type") or "").lower() == "storage"] + if not scoped: + scope = "storage " if storage_only else "" + self.set_failed(f"No {scope}hosts found in step output") + return + + if len(scoped) < min_hosts: + self.set_failed(f"Expected at least {min_hosts} host(s) with stable IP data, got {len(scoped)}") + return + + missing: list[str] = [] + for host in scoped: + label = _host_label(host) + ips = [ip for ip in (host.get("primary_ip_addresses") or []) if ip] + if host.get("has_stable_ip") and ips: + self.report_subtest( + f"stable_ip_{label}", + passed=True, + message=f"{label}: admin IP(s) {', '.join(ips)}", + ) + else: + missing.append(label) + self.report_subtest( + f"stable_ip_{label}", + passed=False, + message=f"{label}: no stable admin IP reported on machineInterfaces", + ) + + if missing: + sample = ", ".join(missing[:3]) + more = len(missing) - min(len(missing), 3) + summary = f"{sample} (+{more} more)" if more else sample + self.set_failed(f"{len(missing)}/{len(scoped)} host(s) missing stable admin IPs: {summary}") + return + + self.set_passed(f"Stable admin IPs queryable for {len(scoped)} host(s)") + + +class OobFailureDetectionCheck(BaseValidation): + """Validate out-of-band failure detection is observable per host (STG04-01). + + Asserts the per-host health API exposes BMC/out-of-band probes and that the + STG04 failure classes (device, network, memory, drive) are observable through + those probes. By default the check requires ``BmcSensor`` (the baseline BMC + path) and at least ``min_failure_categories`` observable categories. + + Config: + step_output: Step output containing per-host OOB health records. + min_hosts: Minimum number of hosts expected (default: 1). + require_oob_report: Whether each host must return an OOB health report + (default: true). + require_bmc_probes: Probe IDs that must be present (default: + ``["BmcSensor"]``). + require_failure_categories: Categories that must be observable per host + (default: ``["device"]`` -- the minimum BMC sensor surface). + min_failure_categories: Minimum count of observable categories when + ``require_failure_categories`` is omitted (default: 1). + + Step output (from query_oob_health.py): + success: bool + platform: str + site_id: str + hosts_checked: int + hosts: list[dict]: + host_id: str + oob_health_present: bool + bmc_probe_ids: list[str] + failure_categories: dict[str, dict]: + : {observable: bool, probe_ids: list[str]} + """ + + description: ClassVar[str] = "Check out-of-band failure detection is observable via BMC health probes" + timeout: ClassVar[int] = 120 + + def run(self) -> None: + """Validate each host exposes OOB probes covering the required failure classes.""" + step_output = self.config.get("step_output", {}) + + if not step_output.get("success"): + self.set_failed(f"OOB health query step failed: {step_output.get('error', 'Unknown error')}") + return + + hosts = step_output.get("hosts") + if not isinstance(hosts, list): + self.set_failed("OOB health step output is missing the 'hosts' list") + return + + min_hosts = self._parse_positive_int("min_hosts", default=1) + if min_hosts is None: + return + + if len(hosts) < min_hosts: + self.set_failed(f"Expected at least {min_hosts} host(s) with OOB health data, got {len(hosts)}") + return + + require_report = self.config.get("require_oob_report", True) + require_bmc_probes = self.config.get("require_bmc_probes", ["BmcSensor"]) + required_categories = self.config.get("require_failure_categories") + min_categories = self._parse_positive_int("min_failure_categories", default=1) + if min_categories is None: + return + + failed: dict[str, str] = {} + + for host in hosts: + label = _host_label(host) + + if require_report and not host.get("oob_health_present"): + self.report_subtest( + f"oob_report_{label}", + passed=False, + message=f"{label}: OOB health API returned no BMC report", + ) + failed.setdefault(label, "no OOB report") + continue + self.report_subtest( + f"oob_report_{label}", + passed=True, + message=f"{label}: OOB health report present", + ) + + present = set(host.get("bmc_probe_ids") or []) + missing_probes = [probe for probe in require_bmc_probes if probe not in present] + if missing_probes: + self.report_subtest( + f"oob_probes_{label}", + passed=False, + message=f"{label}: missing required BMC probe(s): {', '.join(missing_probes)}", + ) + failed.setdefault(label, f"missing probes {', '.join(missing_probes)}") + else: + self.report_subtest( + f"oob_probes_{label}", + passed=True, + message=f"{label}: required BMC probe(s) present: {', '.join(require_bmc_probes)}", + ) + + categories = host.get("failure_categories") or {} + observable = [ + name for name, data in categories.items() if isinstance(data, dict) and data.get("observable") + ] + if required_categories is not None: + missing_categories = [name for name in required_categories if name not in observable] + if missing_categories: + self.report_subtest( + f"oob_categories_{label}", + passed=False, + message=f"{label}: missing observable failure categories: {', '.join(missing_categories)}", + ) + failed.setdefault(label, f"missing categories {', '.join(missing_categories)}") + else: + self.report_subtest( + f"oob_categories_{label}", + passed=True, + message=f"{label}: categories observable: {', '.join(required_categories)}", + ) + elif len(observable) < min_categories: + self.report_subtest( + f"oob_categories_{label}", + passed=False, + message=f"{label}: {len(observable)} observable categories, requires {min_categories}", + ) + failed.setdefault(label, f"only {len(observable)} categories") + else: + self.report_subtest( + f"oob_categories_{label}", + passed=True, + message=f"{label}: {len(observable)} failure categories observable ({', '.join(observable)})", + ) + + total = len(hosts) + if failed: + failed_desc = ", ".join(f"{lbl} ({reason})" for lbl, reason in failed.items()) + self.set_failed(f"OOB failure-detection gaps on {len(failed)}/{total} host(s): {failed_desc}") + return + + self.set_passed(f"Out-of-band failure detection observable for all {total} host(s)") diff --git a/isvtest/tests/test_breakfix_sanitization.py b/isvtest/tests/test_breakfix_sanitization.py new file mode 100644 index 000000000..e5799ddc8 --- /dev/null +++ b/isvtest/tests/test_breakfix_sanitization.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for STG02 break/fix skip-sanitization validation.""" + +from __future__ import annotations + +from typing import Any + +from isvtest.validations.sanitization import SkipSanitizationBreakfixCheck + + +def _machine( + machine_id: str = "m-001", + *, + served_tenant: bool = True, + sanitized: bool = True, + breakfix_skip_observed: bool = False, + tenancy_preserved: bool = True, + stale_tenant_binding: bool = False, +) -> dict[str, Any]: + """Build a provider-neutral sanitization record with STG02 fields.""" + return { + "machine_id": machine_id, + "status": "in_use", + "available": False, + "in_use": True, + "instance_bound": True, + "has_gpu": False, + "served_tenant": served_tenant, + "sanitized": sanitized, + "breakfix_skip_observed": breakfix_skip_observed, + "tenancy_preserved": tenancy_preserved, + "stale_tenant_binding": stale_tenant_binding, + "transitions": ["in_use", "maintenance", "in_use"], + } + + +def _output(*, machines: list[dict[str, Any]] | None = None) -> dict[str, Any]: + """Build a sanitization step output.""" + if machines is None: + machines = [_machine()] + return { + "success": True, + "platform": "nico", + "site_id": "site-1", + "machines_checked": len(machines), + "machines": machines, + } + + +class TestSkipSanitizationBreakfixCheck: + """Tests for SkipSanitizationBreakfixCheck validation (STG02-01).""" + + def test_valid_breakfix_skip_passes(self) -> None: + """A tenancy-preserving maintenance skip passes.""" + check = SkipSanitizationBreakfixCheck(config={"step_output": _output()}) + check.run() + assert check._passed is True, check._error + assert "maintenance skip" in check._output + + def test_no_breakfix_history_passes(self) -> None: + """Sites with no maintenance skips still pass as auditable.""" + machine = _machine(breakfix_skip_observed=False) + check = SkipSanitizationBreakfixCheck(config={"step_output": _output(machines=[machine])}) + check.run() + assert check._passed is True, check._error + assert "no tenancy-preserving maintenance skips" in check._output + + def test_unsanitized_tenant_release_fails(self) -> None: + """Tenant release without sanitizing still fails.""" + machine = _machine(sanitized=False, breakfix_skip_observed=False) + check = SkipSanitizationBreakfixCheck(config={"step_output": _output(machines=[machine])}) + check.run() + assert check._passed is False + + def test_breakfix_without_tenancy_preservation_fails(self) -> None: + """A maintenance skip that drops tenant binding fails.""" + machine = _machine(breakfix_skip_observed=True, tenancy_preserved=False) + check = SkipSanitizationBreakfixCheck(config={"step_output": _output(machines=[machine])}) + check.run() + assert check._passed is False + sub = next(r for r in check._subtest_results if r["name"] == "breakfix_skip_m-001") + assert "tenancy was not preserved" in sub["message"] diff --git a/isvtest/tests/test_storage_infra.py b/isvtest/tests/test_storage_infra.py new file mode 100644 index 000000000..6c64c7e25 --- /dev/null +++ b/isvtest/tests/test_storage_infra.py @@ -0,0 +1,128 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for storage-infrastructure observability validations (STG03/STG04).""" + +from __future__ import annotations + +from typing import Any + +from isvtest.validations.storage_infra import OobFailureDetectionCheck, StableStorageNodeIpCheck + + +def _host( + host_id: str = "m-001", + *, + has_stable_ip: bool = True, + primary_ip_addresses: list[str] | None = None, + hw_sku_device_type: str = "storage", +) -> dict[str, Any]: + """Build a provider-neutral stable-IP host record.""" + ips = primary_ip_addresses if primary_ip_addresses is not None else (["10.0.0.5"] if has_stable_ip else []) + return { + "host_id": host_id, + "hw_sku_device_type": hw_sku_device_type, + "primary_ip_addresses": ips, + "has_stable_ip": has_stable_ip, + } + + +def _oob_host( + host_id: str = "m-001", + *, + oob_health_present: bool = True, + bmc_probe_ids: list[str] | None = None, + failure_categories: dict[str, dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Build a provider-neutral OOB health host record.""" + probes = bmc_probe_ids if bmc_probe_ids is not None else ["BmcSensor"] + categories = failure_categories + if categories is None: + categories = { + "device": {"observable": True, "probe_ids": ["BmcSensor"]}, + "network": {"observable": False, "probe_ids": []}, + "memory": {"observable": False, "probe_ids": []}, + "drive": {"observable": False, "probe_ids": []}, + } + return { + "host_id": host_id, + "oob_health_present": oob_health_present, + "bmc_probe_ids": probes, + "failure_categories": categories, + } + + +def _output(*, hosts: list[dict[str, Any]] | None = None, success: bool = True, error: str = "") -> dict[str, Any]: + """Build a step output envelope.""" + if hosts is None: + hosts = [_host()] + return { + "success": success, + "platform": "nico", + "site_id": "site-1", + "hosts_checked": len(hosts), + "hosts": hosts, + "error": error, + } + + +class TestStableStorageNodeIpCheck: + """Tests for StableStorageNodeIpCheck validation (STG03-01).""" + + def test_hosts_with_ips_pass(self) -> None: + """Every host reporting admin IPs passes.""" + check = StableStorageNodeIpCheck(config={"step_output": _output()}) + check.run() + assert check._passed is True, check._error + + def test_missing_ip_fails(self) -> None: + """A host with no admin IPs fails.""" + host = _host(has_stable_ip=False, primary_ip_addresses=[]) + check = StableStorageNodeIpCheck(config={"step_output": _output(hosts=[host])}) + check.run() + assert check._passed is False + assert "m-001" in check._error + + def test_storage_only_filter(self) -> None: + """storage_only scopes validation to storage SKU hosts.""" + hosts = [_host(hw_sku_device_type="cpu"), _host(host_id="m-002", hw_sku_device_type="storage")] + check = StableStorageNodeIpCheck(config={"step_output": _output(hosts=hosts), "storage_only": True}) + check.run() + assert check._passed is True, check._error + + +class TestOobFailureDetectionCheck: + """Tests for OobFailureDetectionCheck validation (STG04-01).""" + + def test_bmc_coverage_passes(self) -> None: + """Hosts with BmcSensor and device observability pass.""" + check = OobFailureDetectionCheck(config={"step_output": _output(hosts=[_oob_host()])}) + check.run() + assert check._passed is True, check._error + + def test_missing_bmc_probe_fails(self) -> None: + """Missing required BMC probes fails the host.""" + host = _oob_host(bmc_probe_ids=["BgpDaemonEnabled"]) + check = OobFailureDetectionCheck(config={"step_output": _output(hosts=[host])}) + check.run() + assert check._passed is False + assert "BmcSensor" in check._error + + def test_missing_oob_report_fails(self) -> None: + """Absent OOB health report fails when required.""" + host = _oob_host(oob_health_present=False, bmc_probe_ids=[]) + check = OobFailureDetectionCheck(config={"step_output": _output(hosts=[host])}) + check.run() + assert check._passed is False From acef7a58d20bc074da9c2a1417c74147adc2db66 Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Tue, 14 Jul 2026 09:30:28 -0400 Subject: [PATCH 2/5] refactor(nico): simplify STG02/03/04 validations and scripts Apply post-review cleanups to the storage-infra observability work: - Extend _TenantSanitizationCheck with _extra_machine_failure and _passed_summary hooks so SkipSanitizationBreakfixCheck no longer re-implements the base audit loop (and its subtest_prefix/subject attributes are load-bearing again). - Share probe_text() via scripts/common/nico_client.py instead of duplicating it across query_host_health.py and query_oob_health.py; build the probes list once per machine in query_oob_health.py. - Drop mirror/unread output fields per the JSON-contract minimalism rule: has_stable_ip (== bool(primary_ip_addresses)) and instance_bound (only an intermediate for tenancy_preserved). - Collapse OobFailureDetectionCheck's dual category knobs into the single documented require_failure_categories (default ["device"]). - Single-pass IP dedupe in query_stable_ips.py; merge identical lifecycle branches in breakfix_skip_observed(); fold the empty-scope guard into the min_hosts guard in StableStorageNodeIpCheck. - Merge test_breakfix_sanitization.py into test_sanitization.py, reusing its builders, and make the valid-skip test actually set breakfix_skip_observed=True; reuse named script loaders in the nico provider parametrize rows. Co-Authored-By: Claude Fable 5 Signed-off-by: Alexandre Begnoche --- .../nico/scripts/common/nico_client.py | 11 ++ .../nico/scripts/health/query_host_health.py | 17 +-- .../nico/scripts/health/query_oob_health.py | 19 +-- .../sanitization/query_sanitization.py | 10 +- .../nico/scripts/storage/query_stable_ips.py | 14 +-- isvctl/configs/suites/README.md | 4 +- .../providers/nico/test_nico_provider.py | 26 ++-- .../src/isvtest/validations/sanitization.py | 115 ++++++++---------- .../src/isvtest/validations/storage_infra.py | 42 ++----- isvtest/tests/test_breakfix_sanitization.py | 96 --------------- isvtest/tests/test_sanitization.py | 51 +++++++- isvtest/tests/test_storage_infra.py | 12 +- 12 files changed, 151 insertions(+), 266 deletions(-) delete mode 100644 isvtest/tests/test_breakfix_sanitization.py diff --git a/isvctl/configs/providers/nico/scripts/common/nico_client.py b/isvctl/configs/providers/nico/scripts/common/nico_client.py index d0fa025b8..6081fbd32 100644 --- a/isvctl/configs/providers/nico/scripts/common/nico_client.py +++ b/isvctl/configs/providers/nico/scripts/common/nico_client.py @@ -285,6 +285,17 @@ def classify_health(health: dict[str, Any]) -> str: return "unhealthy" if alerts else "healthy" +def probe_text(probe: dict[str, Any]) -> str: + """Return the lowercased ``id`` + ``target`` + ``message`` text for matching. + + NICo reports BMC sensors under a single ``BmcSensor`` probe id and carries + the sensor identity in ``target`` and the entity type in ``message`` (e.g. + ``power_supply``, ``temperature``), so all three fields are searched. + """ + parts = [probe.get("id"), probe.get("target"), probe.get("message")] + return " ".join(p for p in parts if isinstance(p, str)).lower() + + def sum_capabilities(capabilities: list[dict[str, Any]], cap_type: str) -> int: """Sum the count field for capabilities of a given type. diff --git a/isvctl/configs/providers/nico/scripts/health/query_host_health.py b/isvctl/configs/providers/nico/scripts/health/query_host_health.py index 669fdbd0c..7415a6798 100755 --- a/isvctl/configs/providers/nico/scripts/health/query_host_health.py +++ b/isvctl/configs/providers/nico/scripts/health/query_host_health.py @@ -90,7 +90,7 @@ # Allow importing from sibling common/ directory sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from common.nico_client import NicoAuthError, forge_get_all, resolve_auth +from common.nico_client import NicoAuthError, forge_get_all, probe_text, resolve_auth # Substring keywords that map a NICo health probe (by id/target/message) into an # informational hardware component bucket. NICo does not model GPU/thermal/memory @@ -106,17 +106,6 @@ } -def _probe_text(probe: dict[str, Any]) -> str: - """Return the lowercased ``id`` + ``target`` + ``message`` text for matching. - - NICo reports BMC sensors under a single ``BmcSensor`` probe id and carries - the sensor identity in ``target`` and the entity type in ``message`` (e.g. - ``power_supply``, ``temperature``), so all three fields are searched. - """ - parts = [probe.get("id"), probe.get("target"), probe.get("message")] - return " ".join(p for p in parts if isinstance(p, str)).lower() - - def _matches_keywords(text: str, keywords: tuple[str, ...]) -> bool: """Check whether precomputed probe text contains any component keyword.""" return any(keyword in text for keyword in keywords) @@ -144,8 +133,8 @@ def component_breakdown(health: dict[str, Any]) -> dict[str, dict[str, Any]]: # Compute each probe's match text once, then reuse it across every component # rather than rebuilding the lowercased string per (probe, component) pair. - success_texts = [(s, _probe_text(s)) for s in successes] - alert_texts = [(a, _probe_text(a)) for a in alerts] + success_texts = [(s, probe_text(s)) for s in successes] + alert_texts = [(a, probe_text(a)) for a in alerts] components: dict[str, dict[str, Any]] = {} for component, keywords in COMPONENT_KEYWORDS.items(): diff --git a/isvctl/configs/providers/nico/scripts/health/query_oob_health.py b/isvctl/configs/providers/nico/scripts/health/query_oob_health.py index 36eacb8ef..2c6f761be 100644 --- a/isvctl/configs/providers/nico/scripts/health/query_oob_health.py +++ b/isvctl/configs/providers/nico/scripts/health/query_oob_health.py @@ -80,10 +80,10 @@ # Allow importing from sibling common/ directory sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from common.nico_client import NicoAuthError, forge_get_all, resolve_auth +from common.nico_client import NicoAuthError, forge_get_all, probe_text, resolve_auth # BMC / out-of-band probe identifiers NICo reports on the machine health API. -BMC_PROBE_PREFIXES = ("bmc",) +BMC_PROBE_PREFIX = "bmc" # Failure classes STG04-01 calls out, mapped by probe id/target/message keywords. FAILURE_CATEGORY_KEYWORDS: dict[str, tuple[str, ...]] = { @@ -94,22 +94,15 @@ } -def _probe_text(probe: dict[str, Any]) -> str: - """Return lowercased probe id + target + message for keyword matching.""" - parts = [probe.get("id"), probe.get("target"), probe.get("message")] - return " ".join(p for p in parts if isinstance(p, str)).lower() - - def _is_bmc_probe(probe: dict[str, Any]) -> bool: """Return whether a probe is BMC/out-of-band sourced.""" probe_id = str(probe.get("id") or "").lower() - return probe_id.startswith(BMC_PROBE_PREFIXES) + return probe_id.startswith(BMC_PROBE_PREFIX) -def _category_observability(health: dict[str, Any]) -> dict[str, dict[str, Any]]: +def _category_observability(probes: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: """Map NICo health probes into STG04 failure-category observability.""" - probes = [p for p in (*(health.get("successes") or []), *(health.get("alerts") or [])) if isinstance(p, dict)] - probe_texts = [(probe, _probe_text(probe)) for probe in probes] + probe_texts = [(probe, probe_text(probe)) for probe in probes] categories: dict[str, dict[str, Any]] = {} for category, keywords in FAILURE_CATEGORY_KEYWORDS.items(): @@ -139,7 +132,7 @@ def host_record(machine: dict[str, Any]) -> dict[str, Any]: "host_id": machine.get("id", ""), "oob_health_present": oob_present, "bmc_probe_ids": bmc_probe_ids, - "failure_categories": _category_observability(health), + "failure_categories": _category_observability(probes), } diff --git a/isvctl/configs/providers/nico/scripts/sanitization/query_sanitization.py b/isvctl/configs/providers/nico/scripts/sanitization/query_sanitization.py index 0b472d802..f9dca6082 100644 --- a/isvctl/configs/providers/nico/scripts/sanitization/query_sanitization.py +++ b/isvctl/configs/providers/nico/scripts/sanitization/query_sanitization.py @@ -160,13 +160,9 @@ def breakfix_skip_observed(tokens: list[str]) -> bool: if maintenance_after_tenancy: return True active_tenancy = True - maintenance_after_tenancy = False elif token == MAINTENANCE and active_tenancy: maintenance_after_tenancy = True - elif token == SANITIZING: - active_tenancy = False - maintenance_after_tenancy = False - elif token == AVAILABLE: + elif token in (SANITIZING, AVAILABLE): active_tenancy = False maintenance_after_tenancy = False return False @@ -212,17 +208,15 @@ def machine_record(machine: dict[str, Any]) -> dict[str, Any]: dmi = (machine.get("metadata") or {}).get("dmiData") or {} skip_observed = breakfix_skip_observed(tokens) - instance_bound = bool(machine.get("instanceId")) or bool(machine.get("tenantId")) # Tenancy is preserved when the host is still (or again) bound after a # maintenance skip, or is actively in Maintenance while assigned. - tenancy_preserved = not skip_observed or instance_bound or current in {"InUse", "Maintenance"} + tenancy_preserved = not skip_observed or assigned or current in {"InUse", "Maintenance"} return { "machine_id": machine.get("id", ""), "status": status_token(current), "available": is_ready and usable and not assigned, "in_use": current == "InUse", - "instance_bound": instance_bound, "has_gpu": has_gpu(machine), "served_tenant": served, "sanitized": sanitized, diff --git a/isvctl/configs/providers/nico/scripts/storage/query_stable_ips.py b/isvctl/configs/providers/nico/scripts/storage/query_stable_ips.py index 6e9660faf..8da3e1930 100644 --- a/isvctl/configs/providers/nico/scripts/storage/query_stable_ips.py +++ b/isvctl/configs/providers/nico/scripts/storage/query_stable_ips.py @@ -42,8 +42,7 @@ { "host_id": "...", "hw_sku_device_type": "storage", - "primary_ip_addresses": ["192.156.7.23"], - "has_stable_ip": true + "primary_ip_addresses": ["192.156.7.23"] } ] } @@ -88,27 +87,18 @@ def _dedupe_ips(addresses: list[str]) -> list[str]: return list(seen) -def _interface_ips(iface: dict[str, Any]) -> list[str]: - """Return non-empty IP addresses reported on one machine interface.""" - return _dedupe_ips([ip for ip in (iface.get("ipAddresses") or []) if isinstance(ip, str)]) - - def host_record(machine: dict[str, Any]) -> dict[str, Any]: """Build the provider-neutral stable-IP record for one NICo machine.""" interfaces = [iface for iface in (machine.get("machineInterfaces") or []) if isinstance(iface, dict)] primary = [iface for iface in interfaces if iface.get("isPrimary")] targets = primary or interfaces - primary_ips: list[str] = [] - for iface in targets: - primary_ips.extend(_interface_ips(iface)) - primary_ips = _dedupe_ips(primary_ips) + primary_ips = _dedupe_ips([ip for iface in targets for ip in (iface.get("ipAddresses") or [])]) return { "host_id": machine.get("id", ""), "hw_sku_device_type": machine.get("hwSkuDeviceType") or "", "primary_ip_addresses": primary_ips, - "has_stable_ip": bool(primary_ips), } diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index a1589d005..d5d303ad5 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -124,8 +124,8 @@ For the domain / script-count / AWS-reference overview see the | `query_health_aggregation` | test | `providers/nico/scripts/health/query_health_aggregation.py` | `aggregation_level`, `groups[].{total,healthy,unhealthy,status,unhealthy_hosts}` | | `query_ib_tenant_isolation` | test | `providers/nico/scripts/infiniband/query_ib_tenant_isolation.py` | `partitions_checked`, `partitions[].{name,partition_key,tenant_id,status}` | | `query_ib_keys` | test | `providers/nico/scripts/infiniband/query_ib_keys.py` | `partitions_with_pkey`, `keys..{configured,source,detail}` | -| `query_sanitization` | test | `providers/nico/scripts/sanitization/query_sanitization.py` | `machines_checked`, `machines[].{available,in_use,instance_bound,has_gpu,served_tenant,sanitized,breakfix_skip_observed,tenancy_preserved,stale_tenant_binding,vendor,product_name,bios_version,transitions}` | -| `query_stable_ips` | test | `providers/nico/scripts/storage/query_stable_ips.py` | `hosts_checked`, `hosts[].{host_id,hw_sku_device_type,primary_ip_addresses,has_stable_ip}` | +| `query_sanitization` | test | `providers/nico/scripts/sanitization/query_sanitization.py` | `machines_checked`, `machines[].{available,in_use,has_gpu,served_tenant,sanitized,breakfix_skip_observed,tenancy_preserved,stale_tenant_binding,vendor,product_name,bios_version,transitions}` | +| `query_stable_ips` | test | `providers/nico/scripts/storage/query_stable_ips.py` | `hosts_checked`, `hosts[].{host_id,hw_sku_device_type,primary_ip_addresses}` | | `query_oob_health` | test | `providers/nico/scripts/health/query_oob_health.py` | `hosts_checked`, `hosts[].{oob_health_present,bmc_probe_ids,failure_categories..{observable,probe_ids}}` | | `query_attestation` | test | `providers/nico/scripts/attestation/query_attestation.py` | `machines_checked`, `machines[].{attestation_supported,nonce_verified,attestation_signature_valid,secure_boot_enabled,boot_measurements_attested,measured_boot_state}` | | `query_serial_numbers` | test | `providers/nico/scripts/hardware_inventory/query_serial_numbers.py` | `machines_checked`, `machines[].components.{chassis,baseboard,cpu,gpu,nic}.{present,identifiers}` | diff --git a/isvctl/tests/providers/nico/test_nico_provider.py b/isvctl/tests/providers/nico/test_nico_provider.py index 26cae593c..d443657dc 100644 --- a/isvctl/tests/providers/nico/test_nico_provider.py +++ b/isvctl/tests/providers/nico/test_nico_provider.py @@ -220,6 +220,16 @@ def _load_sanitization_script() -> ModuleType: return module +def _load_stable_ips_script() -> ModuleType: + """Load the query_stable_ips script as a module for direct unit testing.""" + return _load_nico_script("storage/query_stable_ips.py", "test_query_stable_ips") + + +def _load_oob_health_script() -> ModuleType: + """Load the query_oob_health script as a module for direct unit testing.""" + return _load_nico_script("health/query_oob_health.py", "test_query_oob_health") + + def test_nico_auth_prefers_explicit_bearer_token(monkeypatch: pytest.MonkeyPatch) -> None: """A locally supplied NICo bearer token should be the simplest auth path.""" module = _load_nico_client() @@ -1007,8 +1017,8 @@ def test_nico_network_inventory_scripts_skip_when_site_has_no_network_inventory( ("query_ib_tenant_isolation.py", _load_ib_tenant_isolation_script), ("query_ib_keys.py", _load_ib_keys_script), ("query_sanitization.py", _load_sanitization_script), - ("query_stable_ips.py", lambda: _load_nico_script("storage/query_stable_ips.py", "test_query_stable_ips_api")), - ("query_oob_health.py", lambda: _load_nico_script("health/query_oob_health.py", "test_query_oob_health_api")), + ("query_stable_ips.py", _load_stable_ips_script), + ("query_oob_health.py", _load_oob_health_script), ], ) def test_nico_scripts_require_api_base( @@ -2617,7 +2627,6 @@ def test_sanitization_breakfix_skip_detection( record = payload["machines"][0] assert record["breakfix_skip_observed"] is True assert record["tenancy_preserved"] is True - assert record["instance_bound"] is True def test_sanitization_breakfix_skip_output_satisfies_check( @@ -2653,11 +2662,6 @@ def test_sanitization_breakfix_skip_output_satisfies_check( # --------------------------------------------------------------------------- -def _load_stable_ips_script() -> ModuleType: - """Load the query_stable_ips script as a module for direct unit testing.""" - return _load_nico_script("storage/query_stable_ips.py", "test_query_stable_ips") - - def _stable_ip_machine( machine_id: str = "m-1", *, @@ -2700,7 +2704,6 @@ def test_stable_ips_script_reports_primary_addresses( assert payload["success"] is True host = payload["hosts"][0] - assert host["has_stable_ip"] is True assert host["primary_ip_addresses"] == ["192.156.7.23", "202.88.37.112"] @@ -2739,11 +2742,6 @@ def test_stable_ips_script_output_satisfies_check( # --------------------------------------------------------------------------- -def _load_oob_health_script() -> ModuleType: - """Load the query_oob_health script as a module for direct unit testing.""" - return _load_nico_script("health/query_oob_health.py", "test_query_oob_health") - - def _oob_machine( machine_id: str = "m-1", *, diff --git a/isvtest/src/isvtest/validations/sanitization.py b/isvtest/src/isvtest/validations/sanitization.py index 3f4d261f4..35c79487b 100644 --- a/isvtest/src/isvtest/validations/sanitization.py +++ b/isvtest/src/isvtest/validations/sanitization.py @@ -115,6 +115,18 @@ def _select_machines(self, machines: list[dict[str, Any]]) -> list[dict[str, Any return [m for m in machines if m.get("has_gpu")] return machines + def _extra_machine_failure(self, machine: dict[str, Any], label: str) -> str | None: + """Hook: extra per-machine checks, run only when the sanitization gate passed. + + Subclasses may report their own subtests here; a returned reason marks + the machine failed in the summary, ``None`` leaves it passing. + """ + return None + + def _passed_summary(self, total: int, served: int) -> str: + """Hook: summary message when every in-scope machine passed.""" + return f"{self.subject} verified on {total} machine(s) ({served} with a prior tenancy audited)" + def run(self) -> None: """Validate that every in-scope machine was sanitized between tenancies.""" step_output = self.config.get("step_output", {}) @@ -139,10 +151,15 @@ def run(self) -> None: for machine in scoped: if machine.get("served_tenant"): served += 1 + label = _machine_label(machine) passed, message = evaluate_sanitization(machine) - self.report_subtest(f"{self.subtest_prefix}_{_machine_label(machine)}", passed=passed, message=message) + self.report_subtest(f"{self.subtest_prefix}_{label}", passed=passed, message=message) if not passed: - failed[_machine_label(machine)] = message + failed[label] = message + continue + reason = self._extra_machine_failure(machine, label) + if reason: + failed[label] = reason total = len(scoped) if failed: @@ -154,7 +171,7 @@ def run(self) -> None: self.set_failed(f"{self.subject} failed for {len(failed)}/{total} machine(s): {summary}") return - self.set_passed(f"{self.subject} verified on {total} machine(s) ({served} with a prior tenancy audited)") + self.set_passed(self._passed_summary(total, served)) class SkipSanitizationBreakfixCheck(_TenantSanitizationCheck): @@ -168,81 +185,49 @@ class SkipSanitizationBreakfixCheck(_TenantSanitizationCheck): Config: step_output: Step output containing per-machine sanitization records - (see ``MemorySanitizationCheck``), plus ``breakfix_skip_observed``, - ``tenancy_preserved``, and ``instance_bound``. + (see ``MemorySanitizationCheck``), plus ``breakfix_skip_observed`` + and ``tenancy_preserved``. Step output (from query_sanitization.py): machines[].breakfix_skip_observed: bool -- maintenance skip without Reset machines[].tenancy_preserved: bool -- tenant binding intact after skip - machines[].instance_bound: bool -- host currently bound to an instance """ catalog_exclude: ClassVar[bool] = False description: ClassVar[str] = "Check optional skip-sanitization during break/fix preserves tenancy" subject: ClassVar[str] = "Break/fix skip-sanitization policy" - subtest_prefix: ClassVar[str] = "breakfix" - - def run(self) -> None: - """Audit tenant-transition sanitization and any tenancy-preserving maintenance skips.""" - step_output = self.config.get("step_output", {}) - - if not step_output.get("success"): - self.set_failed(f"Sanitization step failed: {step_output.get('error', 'Unknown error')}") - return - - machines = step_output.get("machines") - if not isinstance(machines, list): - self.set_failed("Sanitization step output is missing the 'machines' list") - return + subtest_prefix: ClassVar[str] = "tenant_transition" - if not machines: - self.set_failed("No machines found in step output") - return + # Per-run counter; incrementing on the instance shadows this class default. + _breakfix_events: int = 0 - failed: dict[str, str] = {} - breakfix_events = 0 - - for machine in machines: - label = _machine_label(machine) - - passed, message = evaluate_sanitization(machine) - self.report_subtest(f"tenant_transition_{label}", passed=passed, message=message) - if not passed: - failed[label] = message - continue - - if machine.get("breakfix_skip_observed"): - breakfix_events += 1 - if machine.get("tenancy_preserved"): - self.report_subtest( - f"breakfix_skip_{label}", - passed=True, - message=f"{label}: tenancy-preserving maintenance skip observed", - ) - else: - reason = f"{label}: maintenance skip observed but tenancy was not preserved" - self.report_subtest(f"breakfix_skip_{label}", passed=False, message=reason) - failed[label] = reason - - total = len(machines) - if failed: - sample = ", ".join(list(failed)[:3]) - more = len(failed) - min(len(failed), 3) - summary = f"{sample} (+{more} more)" if more else sample - self.set_failed(f"Break/fix sanitization policy failed for {len(failed)}/{total} machine(s): {summary}") - return - - if breakfix_events: - summary = ( - f"Break/fix skip-sanitization policy verified on {total} machine(s) " - f"({breakfix_events} tenancy-preserving maintenance skip(s) observed)" + def _extra_machine_failure(self, machine: dict[str, Any], label: str) -> str | None: + """Audit any tenancy-preserving maintenance skip on a sanitized machine.""" + if not machine.get("breakfix_skip_observed"): + return None + self._breakfix_events += 1 + if machine.get("tenancy_preserved"): + self.report_subtest( + f"breakfix_skip_{label}", + passed=True, + message=f"{label}: tenancy-preserving maintenance skip observed", ) - else: - summary = ( - f"Break/fix skip-sanitization policy auditable on {total} machine(s) " - "(no tenancy-preserving maintenance skips in history yet)" + return None + reason = f"{label}: maintenance skip observed but tenancy was not preserved" + self.report_subtest(f"breakfix_skip_{label}", passed=False, message=reason) + return reason + + def _passed_summary(self, total: int, served: int) -> str: + """Summarize how many tenancy-preserving maintenance skips were audited.""" + if self._breakfix_events: + return ( + f"Break/fix skip-sanitization policy verified on {total} machine(s) " + f"({self._breakfix_events} tenancy-preserving maintenance skip(s) observed)" ) - self.set_passed(summary) + return ( + f"Break/fix skip-sanitization policy auditable on {total} machine(s) " + "(no tenancy-preserving maintenance skips in history yet)" + ) class MemorySanitizationCheck(_TenantSanitizationCheck): diff --git a/isvtest/src/isvtest/validations/storage_infra.py b/isvtest/src/isvtest/validations/storage_infra.py index fbb4180a9..15013a8ae 100644 --- a/isvtest/src/isvtest/validations/storage_infra.py +++ b/isvtest/src/isvtest/validations/storage_infra.py @@ -55,7 +55,6 @@ class StableStorageNodeIpCheck(BaseValidation): host_id: str hw_sku_device_type: str primary_ip_addresses: list[str] - has_stable_ip: bool """ description: ClassVar[str] = "Check stable admin IP assignment is queryable for storage nodes" @@ -80,20 +79,16 @@ def run(self) -> None: storage_only = self.config.get("storage_only", False) scoped = [h for h in hosts if not storage_only or (h.get("hw_sku_device_type") or "").lower() == "storage"] - if not scoped: - scope = "storage " if storage_only else "" - self.set_failed(f"No {scope}hosts found in step output") - return - if len(scoped) < min_hosts: - self.set_failed(f"Expected at least {min_hosts} host(s) with stable IP data, got {len(scoped)}") + scope = "storage " if storage_only else "" + self.set_failed(f"Expected at least {min_hosts} {scope}host(s) with stable IP data, got {len(scoped)}") return missing: list[str] = [] for host in scoped: label = _host_label(host) ips = [ip for ip in (host.get("primary_ip_addresses") or []) if ip] - if host.get("has_stable_ip") and ips: + if ips: self.report_subtest( f"stable_ip_{label}", passed=True, @@ -123,7 +118,7 @@ class OobFailureDetectionCheck(BaseValidation): Asserts the per-host health API exposes BMC/out-of-band probes and that the STG04 failure classes (device, network, memory, drive) are observable through those probes. By default the check requires ``BmcSensor`` (the baseline BMC - path) and at least ``min_failure_categories`` observable categories. + path) and the ``device`` category to be observable. Config: step_output: Step output containing per-host OOB health records. @@ -134,8 +129,6 @@ class OobFailureDetectionCheck(BaseValidation): ``["BmcSensor"]``). require_failure_categories: Categories that must be observable per host (default: ``["device"]`` -- the minimum BMC sensor surface). - min_failure_categories: Minimum count of observable categories when - ``require_failure_categories`` is omitted (default: 1). Step output (from query_oob_health.py): success: bool @@ -176,10 +169,7 @@ def run(self) -> None: require_report = self.config.get("require_oob_report", True) require_bmc_probes = self.config.get("require_bmc_probes", ["BmcSensor"]) - required_categories = self.config.get("require_failure_categories") - min_categories = self._parse_positive_int("min_failure_categories", default=1) - if min_categories is None: - return + required_categories = self.config.get("require_failure_categories", ["device"]) failed: dict[str, str] = {} @@ -220,28 +210,14 @@ def run(self) -> None: observable = [ name for name, data in categories.items() if isinstance(data, dict) and data.get("observable") ] - if required_categories is not None: - missing_categories = [name for name in required_categories if name not in observable] - if missing_categories: - self.report_subtest( - f"oob_categories_{label}", - passed=False, - message=f"{label}: missing observable failure categories: {', '.join(missing_categories)}", - ) - failed.setdefault(label, f"missing categories {', '.join(missing_categories)}") - else: - self.report_subtest( - f"oob_categories_{label}", - passed=True, - message=f"{label}: categories observable: {', '.join(required_categories)}", - ) - elif len(observable) < min_categories: + missing_categories = [name for name in required_categories if name not in observable] + if missing_categories: self.report_subtest( f"oob_categories_{label}", passed=False, - message=f"{label}: {len(observable)} observable categories, requires {min_categories}", + message=f"{label}: missing observable failure categories: {', '.join(missing_categories)}", ) - failed.setdefault(label, f"only {len(observable)} categories") + failed.setdefault(label, f"missing categories {', '.join(missing_categories)}") else: self.report_subtest( f"oob_categories_{label}", diff --git a/isvtest/tests/test_breakfix_sanitization.py b/isvtest/tests/test_breakfix_sanitization.py deleted file mode 100644 index e5799ddc8..000000000 --- a/isvtest/tests/test_breakfix_sanitization.py +++ /dev/null @@ -1,96 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Tests for STG02 break/fix skip-sanitization validation.""" - -from __future__ import annotations - -from typing import Any - -from isvtest.validations.sanitization import SkipSanitizationBreakfixCheck - - -def _machine( - machine_id: str = "m-001", - *, - served_tenant: bool = True, - sanitized: bool = True, - breakfix_skip_observed: bool = False, - tenancy_preserved: bool = True, - stale_tenant_binding: bool = False, -) -> dict[str, Any]: - """Build a provider-neutral sanitization record with STG02 fields.""" - return { - "machine_id": machine_id, - "status": "in_use", - "available": False, - "in_use": True, - "instance_bound": True, - "has_gpu": False, - "served_tenant": served_tenant, - "sanitized": sanitized, - "breakfix_skip_observed": breakfix_skip_observed, - "tenancy_preserved": tenancy_preserved, - "stale_tenant_binding": stale_tenant_binding, - "transitions": ["in_use", "maintenance", "in_use"], - } - - -def _output(*, machines: list[dict[str, Any]] | None = None) -> dict[str, Any]: - """Build a sanitization step output.""" - if machines is None: - machines = [_machine()] - return { - "success": True, - "platform": "nico", - "site_id": "site-1", - "machines_checked": len(machines), - "machines": machines, - } - - -class TestSkipSanitizationBreakfixCheck: - """Tests for SkipSanitizationBreakfixCheck validation (STG02-01).""" - - def test_valid_breakfix_skip_passes(self) -> None: - """A tenancy-preserving maintenance skip passes.""" - check = SkipSanitizationBreakfixCheck(config={"step_output": _output()}) - check.run() - assert check._passed is True, check._error - assert "maintenance skip" in check._output - - def test_no_breakfix_history_passes(self) -> None: - """Sites with no maintenance skips still pass as auditable.""" - machine = _machine(breakfix_skip_observed=False) - check = SkipSanitizationBreakfixCheck(config={"step_output": _output(machines=[machine])}) - check.run() - assert check._passed is True, check._error - assert "no tenancy-preserving maintenance skips" in check._output - - def test_unsanitized_tenant_release_fails(self) -> None: - """Tenant release without sanitizing still fails.""" - machine = _machine(sanitized=False, breakfix_skip_observed=False) - check = SkipSanitizationBreakfixCheck(config={"step_output": _output(machines=[machine])}) - check.run() - assert check._passed is False - - def test_breakfix_without_tenancy_preservation_fails(self) -> None: - """A maintenance skip that drops tenant binding fails.""" - machine = _machine(breakfix_skip_observed=True, tenancy_preserved=False) - check = SkipSanitizationBreakfixCheck(config={"step_output": _output(machines=[machine])}) - check.run() - assert check._passed is False - sub = next(r for r in check._subtest_results if r["name"] == "breakfix_skip_m-001") - assert "tenancy was not preserved" in sub["message"] diff --git a/isvtest/tests/test_sanitization.py b/isvtest/tests/test_sanitization.py index 3e8d85e91..6f2d7002e 100644 --- a/isvtest/tests/test_sanitization.py +++ b/isvtest/tests/test_sanitization.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Tests for the tenant-transition sanitization validations (SEC21-02/04/05/06).""" +"""Tests for the tenant-transition sanitization validations (SEC21-02/04/05/06, STG02-01).""" from __future__ import annotations @@ -24,6 +24,7 @@ FirmwareResetCheck, GpuMemorySanitizationCheck, MemorySanitizationCheck, + SkipSanitizationBreakfixCheck, ) @@ -36,6 +37,8 @@ def _machine( has_gpu: bool = True, served_tenant: bool = True, sanitized: bool = True, + breakfix_skip_observed: bool = False, + tenancy_preserved: bool = True, stale_tenant_binding: bool = False, vendor: str = "Lenovo", product_name: str = "ThinkSystem SR670 V2", @@ -51,6 +54,8 @@ def _machine( "has_gpu": has_gpu, "served_tenant": served_tenant, "sanitized": sanitized, + "breakfix_skip_observed": breakfix_skip_observed, + "tenancy_preserved": tenancy_preserved, "stale_tenant_binding": stale_tenant_binding, "vendor": vendor, "product_name": product_name, @@ -289,3 +294,47 @@ def test_step_failure(self) -> None: check.run() assert check._passed is False assert "API timeout" in check._error + + +# =========================================================================== +# SkipSanitizationBreakfixCheck (STG02-01) +# =========================================================================== + + +class TestSkipSanitizationBreakfixCheck: + """Tests for SkipSanitizationBreakfixCheck validation (STG02-01).""" + + def test_valid_breakfix_skip_passes(self) -> None: + """A tenancy-preserving maintenance skip passes.""" + machine = _machine(breakfix_skip_observed=True, transitions=["in_use", "maintenance", "in_use"]) + check = SkipSanitizationBreakfixCheck(config={"step_output": _output(machines=[machine])}) + check.run() + assert check._passed is True, check._error + assert "1 tenancy-preserving maintenance skip(s) observed" in check._output + + def test_no_breakfix_history_passes(self) -> None: + """Sites with no maintenance skips still pass as auditable.""" + check = SkipSanitizationBreakfixCheck(config={"step_output": _output()}) + check.run() + assert check._passed is True, check._error + assert "no tenancy-preserving maintenance skips" in check._output + + def test_unsanitized_tenant_release_fails(self) -> None: + """Tenant release without sanitizing still fails.""" + machine = _machine(sanitized=False, transitions=["in_use", "available"]) + check = SkipSanitizationBreakfixCheck(config={"step_output": _output(machines=[machine])}) + check.run() + assert check._passed is False + + def test_breakfix_without_tenancy_preservation_fails(self) -> None: + """A maintenance skip that drops tenant binding fails.""" + machine = _machine( + breakfix_skip_observed=True, + tenancy_preserved=False, + transitions=["in_use", "maintenance", "in_use"], + ) + check = SkipSanitizationBreakfixCheck(config={"step_output": _output(machines=[machine])}) + check.run() + assert check._passed is False + sub = next(r for r in check._subtest_results if r["name"] == "breakfix_skip_m-001") + assert "tenancy was not preserved" in sub["message"] diff --git a/isvtest/tests/test_storage_infra.py b/isvtest/tests/test_storage_infra.py index 6c64c7e25..199d159d5 100644 --- a/isvtest/tests/test_storage_infra.py +++ b/isvtest/tests/test_storage_infra.py @@ -25,17 +25,14 @@ def _host( host_id: str = "m-001", *, - has_stable_ip: bool = True, primary_ip_addresses: list[str] | None = None, hw_sku_device_type: str = "storage", ) -> dict[str, Any]: """Build a provider-neutral stable-IP host record.""" - ips = primary_ip_addresses if primary_ip_addresses is not None else (["10.0.0.5"] if has_stable_ip else []) return { "host_id": host_id, "hw_sku_device_type": hw_sku_device_type, - "primary_ip_addresses": ips, - "has_stable_ip": has_stable_ip, + "primary_ip_addresses": primary_ip_addresses if primary_ip_addresses is not None else ["10.0.0.5"], } @@ -64,17 +61,16 @@ def _oob_host( } -def _output(*, hosts: list[dict[str, Any]] | None = None, success: bool = True, error: str = "") -> dict[str, Any]: +def _output(*, hosts: list[dict[str, Any]] | None = None) -> dict[str, Any]: """Build a step output envelope.""" if hosts is None: hosts = [_host()] return { - "success": success, + "success": True, "platform": "nico", "site_id": "site-1", "hosts_checked": len(hosts), "hosts": hosts, - "error": error, } @@ -89,7 +85,7 @@ def test_hosts_with_ips_pass(self) -> None: def test_missing_ip_fails(self) -> None: """A host with no admin IPs fails.""" - host = _host(has_stable_ip=False, primary_ip_addresses=[]) + host = _host(primary_ip_addresses=[]) check = StableStorageNodeIpCheck(config={"step_output": _output(hosts=[host])}) check.run() assert check._passed is False From 62fe2716eb983f99cdbd3e76d25961669a756c2f Mon Sep 17 00:00:00 2001 From: Alexandre Begnoche Date: Tue, 14 Jul 2026 09:47:50 -0400 Subject: [PATCH 3/5] fix(nico): address CodeRabbit review on STG03/04 scripts Scope OOB failure_categories to BMC probes only, request includeMetadata for stable-IP machine enumeration, and escape GFM table pipes in the suites README. Signed-off-by: Alexandre Begnoche --- .../nico/scripts/health/query_oob_health.py | 5 +++-- .../nico/scripts/storage/query_stable_ips.py | 4 ++-- isvctl/configs/suites/README.md | 2 +- .../providers/nico/test_nico_provider.py | 19 +++++++++++++++++++ 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/isvctl/configs/providers/nico/scripts/health/query_oob_health.py b/isvctl/configs/providers/nico/scripts/health/query_oob_health.py index 2c6f761be..8b9f52a6d 100644 --- a/isvctl/configs/providers/nico/scripts/health/query_oob_health.py +++ b/isvctl/configs/providers/nico/scripts/health/query_oob_health.py @@ -125,14 +125,15 @@ def host_record(machine: dict[str, Any]) -> dict[str, Any]: health = machine.get("health") or {} probes = [p for p in (*(health.get("successes") or []), *(health.get("alerts") or [])) if isinstance(p, dict)] - bmc_probe_ids = sorted({str(p.get("id") or "") for p in probes if p.get("id") and _is_bmc_probe(p)}) + bmc_probes = [p for p in probes if _is_bmc_probe(p)] + bmc_probe_ids = sorted({str(p.get("id") or "") for p in bmc_probes if p.get("id")}) oob_present = bool(bmc_probe_ids or health.get("observedAt")) return { "host_id": machine.get("id", ""), "oob_health_present": oob_present, "bmc_probe_ids": bmc_probe_ids, - "failure_categories": _category_observability(probes), + "failure_categories": _category_observability(bmc_probes), } diff --git a/isvctl/configs/providers/nico/scripts/storage/query_stable_ips.py b/isvctl/configs/providers/nico/scripts/storage/query_stable_ips.py index 8da3e1930..021105a29 100644 --- a/isvctl/configs/providers/nico/scripts/storage/query_stable_ips.py +++ b/isvctl/configs/providers/nico/scripts/storage/query_stable_ips.py @@ -25,7 +25,7 @@ admin IP. NICo API endpoints used: - GET /v2/org/{org}/carbide/machine?siteId={site_id} + GET /v2/org/{org}/carbide/machine?siteId={site_id}&includeMetadata=true Auth: - NICO_BEARER_TOKEN, or @@ -126,7 +126,7 @@ def main() -> int: "machine", auth.token, base_url=args.api_base, - params={"siteId": args.site_id}, + params={"siteId": args.site_id, "includeMetadata": "true"}, result_key="machines", ) diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index d5d303ad5..0a2c14fd1 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -126,7 +126,7 @@ For the domain / script-count / AWS-reference overview see the | `query_ib_keys` | test | `providers/nico/scripts/infiniband/query_ib_keys.py` | `partitions_with_pkey`, `keys..{configured,source,detail}` | | `query_sanitization` | test | `providers/nico/scripts/sanitization/query_sanitization.py` | `machines_checked`, `machines[].{available,in_use,has_gpu,served_tenant,sanitized,breakfix_skip_observed,tenancy_preserved,stale_tenant_binding,vendor,product_name,bios_version,transitions}` | | `query_stable_ips` | test | `providers/nico/scripts/storage/query_stable_ips.py` | `hosts_checked`, `hosts[].{host_id,hw_sku_device_type,primary_ip_addresses}` | -| `query_oob_health` | test | `providers/nico/scripts/health/query_oob_health.py` | `hosts_checked`, `hosts[].{oob_health_present,bmc_probe_ids,failure_categories..{observable,probe_ids}}` | +| `query_oob_health` | test | `providers/nico/scripts/health/query_oob_health.py` | `hosts_checked`, `hosts[].{oob_health_present,bmc_probe_ids,failure_categories..{observable,probe_ids}}` | | `query_attestation` | test | `providers/nico/scripts/attestation/query_attestation.py` | `machines_checked`, `machines[].{attestation_supported,nonce_verified,attestation_signature_valid,secure_boot_enabled,boot_measurements_attested,measured_boot_state}` | | `query_serial_numbers` | test | `providers/nico/scripts/hardware_inventory/query_serial_numbers.py` | `machines_checked`, `machines[].components.{chassis,baseboard,cpu,gpu,nic}.{present,identifiers}` | | `query_topology` | test | `providers/nico/scripts/topology/query_topology.py` | `hosts_checked`, `hosts[].{host_id,failure_domain}` | diff --git a/isvctl/tests/providers/nico/test_nico_provider.py b/isvctl/tests/providers/nico/test_nico_provider.py index d443657dc..e375b458d 100644 --- a/isvctl/tests/providers/nico/test_nico_provider.py +++ b/isvctl/tests/providers/nico/test_nico_provider.py @@ -2788,6 +2788,25 @@ def test_oob_health_script_maps_bmc_categories( assert host["failure_categories"]["device"]["observable"] is True +def test_oob_health_script_ignores_non_bmc_probes_for_categories( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + """Non-BMC probes must not inflate STG04 failure-category observability.""" + machine = _oob_machine( + successes=[ + {"id": "BmcSensor", "target": "CPU1 Temp", "message": "temperature"}, + {"id": "BgpDaemonEnabled", "target": "mlx5_0", "message": "network link up"}, + ], + ) + payload = _run_oob_health(monkeypatch, capsys, [machine]) + + categories = payload["hosts"][0]["failure_categories"] + assert categories["device"]["observable"] is True + assert categories["network"]["observable"] is False + assert categories["network"]["probe_ids"] == [] + + def test_oob_health_script_empty_site_skips( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], From c57c42e39e1f1be796c53ac0471c5f5f59265e31 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 14 Jul 2026 15:34:48 +0000 Subject: [PATCH 4/5] fix(ci): apply pre-commit fixes to nico provider tests Signed-off-by: Cursor Agent Co-authored-by: Alexandre Begnoche --- isvctl/tests/providers/nico/test_nico_provider.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/isvctl/tests/providers/nico/test_nico_provider.py b/isvctl/tests/providers/nico/test_nico_provider.py index e375b458d..69b226106 100644 --- a/isvctl/tests/providers/nico/test_nico_provider.py +++ b/isvctl/tests/providers/nico/test_nico_provider.py @@ -41,8 +41,8 @@ MemorySanitizationCheck, SkipSanitizationBreakfixCheck, ) -from isvtest.validations.topology import FailureDomainObservabilityCheck from isvtest.validations.storage_infra import OobFailureDetectionCheck, StableStorageNodeIpCheck +from isvtest.validations.topology import FailureDomainObservabilityCheck from isvctl.config.merger import merge_yaml_files from isvctl.config.schema import RunConfig @@ -3047,4 +3047,3 @@ def test_topology_script_output_satisfies_check( bad.run() assert bad._passed is False assert "m-1" in bad._error - From 49b414ed78e887c46c5b9447b2c1a73b09155cc1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 14 Jul 2026 15:36:02 +0000 Subject: [PATCH 5/5] docs(nico): document host_id in query_oob_health README row Signed-off-by: Cursor Agent Co-authored-by: Alexandre Begnoche --- isvctl/configs/suites/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/isvctl/configs/suites/README.md b/isvctl/configs/suites/README.md index 0a2c14fd1..09c19cafe 100644 --- a/isvctl/configs/suites/README.md +++ b/isvctl/configs/suites/README.md @@ -126,7 +126,7 @@ For the domain / script-count / AWS-reference overview see the | `query_ib_keys` | test | `providers/nico/scripts/infiniband/query_ib_keys.py` | `partitions_with_pkey`, `keys..{configured,source,detail}` | | `query_sanitization` | test | `providers/nico/scripts/sanitization/query_sanitization.py` | `machines_checked`, `machines[].{available,in_use,has_gpu,served_tenant,sanitized,breakfix_skip_observed,tenancy_preserved,stale_tenant_binding,vendor,product_name,bios_version,transitions}` | | `query_stable_ips` | test | `providers/nico/scripts/storage/query_stable_ips.py` | `hosts_checked`, `hosts[].{host_id,hw_sku_device_type,primary_ip_addresses}` | -| `query_oob_health` | test | `providers/nico/scripts/health/query_oob_health.py` | `hosts_checked`, `hosts[].{oob_health_present,bmc_probe_ids,failure_categories..{observable,probe_ids}}` | +| `query_oob_health` | test | `providers/nico/scripts/health/query_oob_health.py` | `hosts_checked`, `hosts[].{host_id,oob_health_present,bmc_probe_ids,failure_categories..{observable,probe_ids}}` | | `query_attestation` | test | `providers/nico/scripts/attestation/query_attestation.py` | `machines_checked`, `machines[].{attestation_supported,nonce_verified,attestation_signature_valid,secure_boot_enabled,boot_measurements_attested,measured_boot_state}` | | `query_serial_numbers` | test | `providers/nico/scripts/hardware_inventory/query_serial_numbers.py` | `machines_checked`, `machines[].components.{chassis,baseboard,cpu,gpu,nic}.{present,identifiers}` | | `query_topology` | test | `providers/nico/scripts/topology/query_topology.py` | `hosts_checked`, `hosts[].{host_id,failure_domain}` |