Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/test-plan.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,8 @@ domains:
notes: ""
github_issues:
- "#248"
labels:
- vm
- name: Identity and Asset Mgmt (IAM)
capabilities:
- description: Manages users, service identities, roles, and permissions across tenants and services. Forms the security foundation of the platform.
Expand Down
31 changes: 24 additions & 7 deletions isvctl/configs/providers/aws/config/vm.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,16 @@
# 3. verify_tags - boto3: Verifies user-defined tags (test)
# 4. serial_console - boto3: Retrieves serial console output (test)
# 5. console_rbac - boto3: Simulates EC2 serial console IAM access (test)
# 6. stop_instance - boto3: Stops EC2, verifies stopped state (test)
# 7. start_instance - boto3: Starts stopped EC2, verifies recovery (test)
# 8. reboot_instance - boto3: Reboots EC2, validates recovery (test)
# 9. describe_instance - boto3: Describes current state, passes SSH info (test)
# 6. component_key_access - boto3: AUTH03 specified-key SOL access (test)
# 7. virtual_device_hardening - guest/hypervisor device hardening (test)
# 8. stop_instance - boto3: Stops EC2, verifies stopped state (test)
# 9. start_instance - boto3: Starts stopped EC2, verifies recovery (test)
# 10. reboot_instance - boto3: Reboots EC2, validates recovery (test)
# 11. describe_instance - boto3: Describes current state, passes SSH info (test)
# (runs after reboot so IP is current; validation anchor)
# 10. deploy_nim - paramiko: Deploys NIM container via SSH (test)
# 11. teardown_nim - paramiko: Stops NIM container via SSH (teardown)
# 12. teardown - boto3: Terminates EC2 (teardown)
# 12. deploy_nim - paramiko: Deploys NIM container via SSH (test)
# 13. teardown_nim - paramiko: Stops NIM container via SSH (teardown)
# 14. teardown - boto3: Terminates EC2 (teardown)
#
# Usage:
# uv run isvctl test run -f isvctl/configs/providers/aws/config/vm.yaml
Expand Down Expand Up @@ -105,6 +107,21 @@ commands:
- "{{region}}"
timeout: 120

# AWS-specific: Specified key access to SOL (AUTH03); network devices provider-hidden
- name: component_key_access
phase: test
command: "python3 ../scripts/vm/component_key_access.py"
args:
- "--instance-id"
- "{{steps.launch_instance.instance_id}}"
- "--key-file"
- "{{steps.launch_instance.key_file}}"
- "--key-name"
- "{{steps.launch_instance.key_name}}"
- "--region"
- "{{region}}"
timeout: 120

# AWS-specific: Virtual device hardening evidence
- name: virtual_device_hardening
phase: test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
from botocore.exceptions import ClientError

RETENTION_EVIDENCE_LIMITATION = "EC2 GetConsoleOutput does not prove one-month log retention"
SERIAL_ACCESS_DISABLED_SKIP_REASON = "EC2 serial console access is disabled for this account or region"
SERIAL_CONSOLE_RETENTION_DAYS_REQUIRED = 30


Expand Down
20 changes: 10 additions & 10 deletions isvctl/configs/providers/aws/scripts/iam/test_credentials.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@
"account_id": "123456789",
"arn": "arn:aws:iam::...",
"tests": {
"sts_identity": {"passed": true},
"iam_access": {"passed": true}
"identity": {"passed": true},
"access": {"passed": true}
}
}
"""
Expand Down Expand Up @@ -69,9 +69,9 @@ def test_sts_identity(session: boto3.Session, result: dict) -> bool:
result["account_id"] = identity["Account"]
result["arn"] = identity["Arn"]
result["user_id"] = identity["UserId"]
result["tests"]["sts_identity"] = {"passed": True}
result["tests"]["identity"] = {"passed": True}
if attempt > 0:
result["tests"]["sts_identity"]["retries"] = attempt
result["tests"]["identity"]["retries"] = attempt
return True
except ClientError as e:
last_error = e
Expand All @@ -88,7 +88,7 @@ def test_sts_identity(session: boto3.Session, result: dict) -> bool:
error_type, error_msg = classify_aws_error(last_error)
result["error_type"] = error_type
result["error"] = error_msg
result["tests"]["sts_identity"] = {"passed": False, "error": error_msg}
result["tests"]["identity"] = {"passed": False, "error": error_msg}
return False


Expand Down Expand Up @@ -117,19 +117,19 @@ def main() -> int:
print(json.dumps(result, indent=2))
return 1

# Test IAM access
# Test authorized resource access (IAM GetUser, or AccessDenied proving authz)
try:
iam = session.client("iam")
iam.get_user()
result["tests"]["iam_access"] = {"passed": True}
result["tests"]["access"] = {"passed": True}
except ClientError as e:
if e.response["Error"]["Code"] == "AccessDenied":
# Access denied means credentials work but limited permissions
result["tests"]["iam_access"] = {"passed": True, "note": "Access denied (expected)"}
result["tests"]["access"] = {"passed": True, "note": "Access denied (expected)"}
else:
result["tests"]["iam_access"] = {"passed": False, "error": str(e)}
result["tests"]["access"] = {"passed": False, "error": str(e)}

result["success"] = result["tests"]["sts_identity"]["passed"]
result["success"] = result["tests"]["identity"]["passed"] and result["tests"]["access"]["passed"]
Comment thread
coderabbitai[bot] marked this conversation as resolved.
print(json.dumps(result, indent=2))
return 0 if result["success"] else 1

Expand Down
156 changes: 156 additions & 0 deletions isvctl/configs/providers/aws/scripts/vm/component_key_access.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Prove a specified SSH key can access SOL (serial console) on AWS EC2.

AUTH03-01: after AUTH02 launches an instance with a requested key, push that
key's public material via EC2 Instance Connect serial-console authorization.
Tenant-visible network-device SSH is not offered on AWS, so that probe is
marked provider-hidden.

Usage:
python component_key_access.py --instance-id i-xxx --key-file /tmp/key.pem \\
--key-name isv-test-key --region us-west-2
"""

from __future__ import annotations

import argparse
import json
import os
import sys
from pathlib import Path
from typing import Any

sys.path.insert(0, str(Path(__file__).parent.parent))

import boto3
from botocore.exceptions import ClientError
from common.errors import handle_aws_errors
from common.serial_console import SERIAL_ACCESS_DISABLED_SKIP_REASON, check_serial_access
from paramiko import ECDSAKey, Ed25519Key, RSAKey
from paramiko.ssh_exception import SSHException


def _load_openssh_public_key(key_file: str) -> str:
"""Derive an OpenSSH public-key line from a private key PEM file."""
path = Path(key_file)
if not path.is_file():
msg = f"Key file not found: {key_file}"
raise FileNotFoundError(msg)

loaders = (RSAKey, ECDSAKey, Ed25519Key)
last_error: Exception | None = None
for loader in loaders:
try:
private_key = loader.from_private_key_file(str(path))
return f"{private_key.get_name()} {private_key.get_base64()}"
except (SSHException, OSError, ValueError) as exc:
last_error = exc

msg = f"Unable to load private key from {key_file}: {last_error}"
raise ValueError(msg)


def _probe_sol_access(ec2: Any, eic: Any, instance_id: str, ssh_public_key: str) -> dict[str, Any]:
"""Authorize the specified public key for EC2 serial-console SSH."""
serial_access = check_serial_access(ec2)
if serial_access.get("error"):
return {"passed": False, "error": serial_access["error"], "probes": ["serial_console_access"]}
if serial_access.get("enabled") is not True:
return {
"passed": False,
"skipped": True,
"skip_reason": SERIAL_ACCESS_DISABLED_SKIP_REASON,
"probes": ["serial_console_access"],
}

try:
response = eic.send_serial_console_ssh_public_key(
InstanceId=instance_id,
SSHPublicKey=ssh_public_key,
SerialPort=0,
)
except ClientError as exc:
return {"passed": False, "error": str(exc), "probes": ["send_serial_console_ssh_public_key"]}

return {
"passed": bool(response.get("Success")),
"message": "Authorized specified key for EC2 serial console SSH",
"probes": ["serial_console_access", "send_serial_console_ssh_public_key"],
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.


def _probe_network_device_access() -> dict[str, Any]:
"""AWS has no tenant-visible network-device key path; mark provider-hidden."""
return {
"passed": True,
"provider_hidden": True,
"message": "AWS does not expose tenant-visible network-device SSH for key-based access",
"probes": ["network_device_ssh"],
}


@handle_aws_errors
def main() -> int:
"""Run AUTH03 key-based component access probes and emit JSON."""
parser = argparse.ArgumentParser(description="Prove specified key access to SOL / network devices")
parser.add_argument("--instance-id", required=True, help="EC2 instance ID")
parser.add_argument("--key-file", required=True, help="Path to the instance private key PEM")
parser.add_argument("--key-name", required=True, help="Key pair name requested at launch (AUTH02)")
parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-west-2"))
args = parser.parse_args()

result: dict[str, Any] = {
"success": False,
"platform": "vm",
"test_name": "component_key_access",
"instance_id": args.instance_id,
"key_name": args.key_name,
"tests": {},
}

try:
ssh_public_key = _load_openssh_public_key(args.key_file)
except (OSError, ValueError) as exc:
result["error"] = str(exc)
result["tests"]["sol_access"] = {"passed": False, "error": str(exc)}
print(json.dumps(result, indent=2))
return 1

ec2 = boto3.client("ec2", region_name=args.region)
eic = boto3.client("ec2-instance-connect", region_name=args.region)

sol = _probe_sol_access(ec2, eic, args.instance_id, ssh_public_key)
if sol.get("skipped") is True:
result["success"] = True
result["skipped"] = True
result["skip_reason"] = sol["skip_reason"]
print(json.dumps(result, indent=2))
return 0

result["tests"]["sol_access"] = sol
result["tests"]["network_device_access"] = _probe_network_device_access()
result["success"] = all(test["passed"] for test in result["tests"].values())
if not result["success"] and sol.get("error"):
result["error"] = sol["error"]

print(json.dumps(result, indent=2))
return 0 if result["success"] else 1


if __name__ == "__main__":
sys.exit(main())
3 changes: 1 addition & 2 deletions isvctl/configs/providers/aws/scripts/vm/console_rbac.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,12 @@
import botocore.session
from botocore.exceptions import ClientError
from common.errors import handle_aws_errors
from common.serial_console import check_serial_access
from common.serial_console import SERIAL_ACCESS_DISABLED_SKIP_REASON, check_serial_access

CONSOLE_ACTION = "ec2-instance-connect:SendSerialConsoleSSHPublicKey"
TEST_USER_PREFIX = "isv-console-rbac-test-"
ALLOW_POLICY_NAME = "IsvConsoleRbacScopedAccess"
OWNER_TAG = {"Key": "CreatedBy", "Value": "isvtest"}
SERIAL_ACCESS_DISABLED_SKIP_REASON = "EC2 serial console access is disabled for this account or region"
REQUIRED_TESTS = (
"serial_console_access_enabled",
"denied_principal_cannot_access_console",
Expand Down
24 changes: 20 additions & 4 deletions isvctl/configs/providers/my-isv/config/vm.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
# (InstanceStateCheck, InstanceCreatedCheck, InstanceListCheck,
# InstanceTagCheck, InstanceStopCheck, InstanceStartCheck,
# StableIdentifierCheck, InstanceRebootCheck, SerialConsoleCheck,
# ConsoleRbacCheck, StepSuccessCheck). SSH-based host checks (ConnectivityCheck,
# ConsoleRbacCheck, ComponentKeyAccessCheck, StepSuccessCheck). SSH-based host checks (ConnectivityCheck,
# GpuCheck, DriverCheck, CpuInfoCheck, NIM checks, etc.) are excluded
# via `exclude.labels: [ssh]` because a dummy-success stub cannot
# spin up a real host to SSH into. The deploy_nim / teardown_nim
Expand All @@ -45,10 +45,12 @@
# 6. reboot_instance - Reboot it
# 7. serial_console - Confirm serial console access
# 8. console_rbac - Verify console access is RBAC-scoped
# 9. deploy_nim - SKIPPED (needs real SSH)
# 9. component_key_access - AUTH03 specified-key SOL / network device access
# 10. virtual_device_hardening - Confirm unused virtio devices are absent
# 11. deploy_nim - SKIPPED (needs real SSH)
# TEARDOWN
# 10. teardown_nim - SKIPPED (needs real SSH)
# 11. teardown - Terminate the VM
# 12. teardown_nim - SKIPPED (needs real SSH)
# 13. teardown - Terminate the VM
#
# Usage:
# uv run isvctl test run -f isvctl/configs/providers/my-isv/config/vm.yaml
Expand Down Expand Up @@ -154,6 +156,20 @@ commands:
- "{{region}}"
timeout: 60

- name: component_key_access
phase: test
command: "python ../scripts/vm/component_key_access.py"
args:
- "--instance-id"
- "{{steps.launch_instance.instance_id}}"
- "--key-file"
- "{{steps.launch_instance.key_file}}"
- "--key-name"
- "{{steps.launch_instance.key_name}}"
- "--region"
- "{{region}}"
timeout: 60

- name: virtual_device_hardening
phase: test
command: "python ../scripts/vm/virtual_device_hardening.py"
Expand Down
2 changes: 1 addition & 1 deletion isvctl/configs/providers/my-isv/scripts/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ template, then fill in the TODOs.
|--------|---------|----------|---------------|---------------|
| `iam/` | 3 | [`suites/iam.yaml`](../../../suites/iam.yaml) | [`config/iam.yaml`](../config/iam.yaml) | [`providers/aws/scripts/iam/`](../../aws/scripts/iam/) |
| `control-plane/` | 10 | [`suites/control-plane.yaml`](../../../suites/control-plane.yaml) | [`config/control-plane.yaml`](../config/control-plane.yaml) | [`providers/aws/scripts/control-plane/`](../../aws/scripts/control-plane/) |
| `vm/` | 9 | [`suites/vm.yaml`](../../../suites/vm.yaml) | [`config/vm.yaml`](../config/vm.yaml) | [`providers/aws/scripts/vm/`](../../aws/scripts/vm/) |
| `vm/` | 10 | [`suites/vm.yaml`](../../../suites/vm.yaml) | [`config/vm.yaml`](../config/vm.yaml) | [`providers/aws/scripts/vm/`](../../aws/scripts/vm/) |
| `bare_metal/` | 12 | [`suites/bare_metal.yaml`](../../../suites/bare_metal.yaml) | [`config/bare_metal.yaml`](../config/bare_metal.yaml) | [`providers/aws/scripts/bare_metal/`](../../aws/scripts/bare_metal/) |
| `storage/` | 17 | [`suites/storage.yaml`](../../../suites/storage.yaml) | [`config/storage.yaml`](../config/storage.yaml) | [`providers/aws/scripts/storage/`](../../aws/scripts/storage/) |
| `network/` | 18 | [`suites/network.yaml`](../../../suites/network.yaml) | [`config/network.yaml`](../config/network.yaml) | [`providers/aws/scripts/network/`](../../aws/scripts/network/) |
Expand Down
Loading