feat(security): implement SEC13-01 mutual TLS validation

docs/test-plan.yaml

@@ -3727,6 +3727,8 @@ domains:
               - summary: Verify mTLS (or equivalent) for all east-west and north-south traffic
                 labels:
                   - min_req
+                  - network
+                  - security
                 priority: P1
                 milestone: M5
                 notes: ""

isvctl/configs/providers/aws/config/security.yaml

@@ -31,20 +31,21 @@
 #   3. BMC Protocol Security   - Verify CNP10-01 BMC protocol posture
 #   4. BMC Bastion Access      - Verify BMC reachable only via hardened bastion
 #   5. API Endpoint Isolation  - Verify management APIs are not publicly exposed
-#   6. Insecure Protocols     - Verify SSLv3/TLS1.0/TLS1.1/plain HTTP disabled (SEC13-02)
-#   7. MFA Enforcement         - Verify admin interfaces (UI, CLI, API) require MFA
-#   8. Cert Rotation Cycle     - Verify cert rotation cycle / auto-renewal (SEC09-01)
-#   9. KMS Options             - Verify provider- and customer-managed keys (SEC09-02)
-#  10. Centralized KMS         - Verify encrypted resources resolve to KMS (SEC09-03)
-#  11. Customer Managed Key    - Verify customer-managed key encryption (SEC09-04)
-#  12. Least Privilege         - Verify scoped user/resource/network policies (SEC04-01/02)
-#  13. Audit Logging           - Verify management-event logging and retention (SEC08-01/02)
-#  14. SA Credential Auth      - Verify IAM user + long-lived access key auth
-#  15. OIDC User Auth          - Probe configured OIDC-protected platform endpoint (SEC01-01)
-#  16. Short-Lived Credentials - Verify STS issues node + workload creds with bounded TTL (SEC02-01)
-#  17. Tenant Isolation        - Verify hard tenant isolation across network, data, compute and storage (SEC11-01)
-#  18. Capacity Grouping       - Verify reservations are grouped and tenant-pinned (CAP04-01)
-#  19. Topology Block          - Verify topology block atomic allocation boundaries (CAP04-02)
+#   6. Mutual TLS              - Verify mTLS north-south/east-west (SEC13-01)
+#   7. Insecure Protocols      - Verify SSLv3/TLS1.0/TLS1.1/plain HTTP disabled (SEC13-02)
+#   8. MFA Enforcement         - Verify admin interfaces (UI, CLI, API) require MFA
+#   9. Cert Rotation Cycle     - Verify cert rotation cycle / auto-renewal (SEC09-01)
+#  10. KMS Options             - Verify provider- and customer-managed keys (SEC09-02)
+#  11. Centralized KMS         - Verify encrypted resources resolve to KMS (SEC09-03)
+#  12. Customer Managed Key    - Verify customer-managed key encryption (SEC09-04)
+#  13. Least Privilege         - Verify scoped user/resource/network policies (SEC04-01/02)
+#  14. Audit Logging           - Verify management-event logging and retention (SEC08-01/02)
+#  15. SA Credential Auth      - Verify IAM user + long-lived access key auth
+#  16. OIDC User Auth          - Probe configured OIDC-protected platform endpoint (SEC01-01)
+#  17. Short-Lived Credentials - Verify STS issues node + workload creds with bounded TTL (SEC02-01)
+#  18. Tenant Isolation        - Verify hard tenant isolation across network, data, compute and storage (SEC11-01)
+#  19. Capacity Grouping       - Verify reservations are grouped and tenant-pinned (CAP04-01)
+#  20. Topology Block          - Verify topology block atomic allocation boundaries (CAP04-02)
 #
 # Note: SEC12-* (BMC) tests cannot be fully validated on AWS because the
 # Nitro/hypervisor BMC plane is provider-hidden. The references inspect the
@@ -105,7 +106,25 @@ commands:
         args: ["--region", "{{region}}"]
         timeout: 60
 
-      # Test 6: Insecure Protocols (SEC13-02)
+      # Test 6a: Mutual TLS (SEC13-01)
+      # Probes EDGE_ENDPOINTS / EAST_WEST_ENDPOINTS with MTLS_* cert material.
+      # Without endpoints the shared script emits a structured skip. East-west
+      # is provider-hidden on AWS unless EAST_WEST_ENDPOINTS is set (intra-VPC
+      # service mTLS is customer-owned mesh/sidecar), expressed via the
+      # provider-hidden message below.
+      - name: mutual_tls_test
+        phase: test
+        command: "python3 ../../shared/mutual_tls_test.py"
+        args:
+          - >-
+            --east-west-provider-hidden-message=east_west_mtls_enforced:
+            AWS EC2/EKS tenants do not receive a customer-provable east-west
+            mTLS surface in region {{region}}; intra-VPC service encryption is
+            customer-owned (mesh/sidecar). Set EAST_WEST_ENDPOINTS to probe
+            platform-specific east-west endpoints when available.
+        timeout: 120
+
+      # Test 6b: Insecure Protocols (SEC13-02)
       # Provisions a temporary TLS-only ALB fixture, probes it for legacy
       # protocol refusal, then tears it down. When the orchestrator principal
       # lacks fixture permissions the script emits a structured skip.

isvctl/configs/providers/aws/scripts/security/tenant_isolation_test.py

@@ -263,6 +263,40 @@ def _launch_instance(ec2: Any, tenant: Tenant, ami_id: str) -> None:
     tenant.created["instance"] = True
 
 
+def _wait_instance_terminated(
+    ec2: Any,
+    instance_id: str,
+    *,
+    delay: float = 5.0,
+    max_attempts: int = 60,
+) -> None:
+    """Poll until ``instance_id`` is terminated (or already gone).
+
+    Unlike boto3's ``instance_terminated`` waiter, treat ``pending`` as
+    in-progress. Terminating a still-booting instance briefly stays
+    ``pending`` before ``shutting-down``; the stock waiter fails that race
+    immediately and leaves ENIs holding the SG/subnet/VPC.
+    """
+    for _ in range(max_attempts):
+        try:
+            response = ec2.describe_instances(InstanceIds=[instance_id])
+        except ClientError as e:
+            if e.response.get("Error", {}).get("Code") == "InvalidInstanceID.NotFound":
+                return
+            raise
+        states = [
+            instance.get("State", {}).get("Name")
+            for reservation in response.get("Reservations", [])
+            for instance in reservation.get("Instances", [])
+        ]
+        # all() on an empty list is True: no reservations means already gone.
+        if all(state == "terminated" for state in states):
+            return
+        time.sleep(delay)
+    msg = f"instance {instance_id} did not reach terminated within {delay * max_attempts:.0f}s"
+    raise TimeoutError(msg)
+
+
 def _create_volume(ec2: Any, tenant: Tenant) -> None:
     """Create a 1 GiB gp3 EBS volume in the tenant's AZ."""
     response = ec2.create_volume(
@@ -377,22 +411,16 @@ def _teardown_tenant(
     errors: list[str] = []
 
     if tenant.created.get("instance") and tenant.instance_id:
-        # NB: catch WaiterError too -- the InstanceTerminated waiter treats
-        # "pending" as a terminal failure, which fires when setup raised
-        # before the instance reached running. Letting it propagate would
-        # mask the original error (and skip the rest of cleanup); the
-        # safety-net teardown step picks up any leftover instance.
+        # Catch wait failures too so a stuck terminate does not skip the rest
+        # of cleanup; the safety-net teardown step picks up leftovers.
         try:
             ec2.terminate_instances(InstanceIds=[tenant.instance_id])
-        except ClientError as e:
+        except (ClientError, BotoCoreError) as e:
             errors.append(f"terminate instance {tenant.instance_id}: {e}")
         else:
             try:
-                ec2.get_waiter("instance_terminated").wait(
-                    InstanceIds=[tenant.instance_id],
-                    WaiterConfig={"Delay": 5, "MaxAttempts": 60},
-                )
-            except (ClientError, WaiterError) as e:
+                _wait_instance_terminated(ec2, tenant.instance_id)
+            except (ClientError, BotoCoreError, TimeoutError) as e:
                 errors.append(f"wait terminated {tenant.instance_id}: {e}")
 
     if tenant.created.get("volume") and tenant.volume_id:
@@ -863,8 +891,8 @@ def main() -> int:
             result["success"] = all(t.get("passed") for t in result["tests"].values())
     except (ClientError, WaiterError, BotoCoreError) as exc:
         # Capture the FIRST error before cleanup runs, otherwise a downstream
-        # WaiterError from terminating a still-pending instance would mask
-        # the real cause (IAM limit, VPC limit, ResourceLimitExceeded, ...).
+        # cleanup error would mask the real cause (IAM limit, VPC limit,
+        # ResourceLimitExceeded, ...).
         error_type, error_msg = classify_aws_error(exc)
         result["error"] = f"[{error_type}] {error_msg}"
         result["success"] = False

isvctl/configs/providers/my-isv/config/security.yaml

@@ -91,6 +91,17 @@ commands:
         args: ["--region", "{{region}}"]
         timeout: 60
 
+      - name: mutual_tls_test
+        phase: test
+        command: "python ../../shared/mutual_tls_test.py"
+        args:
+          - "--north-south-endpoints={{mutual_tls_north_south_endpoints}}"
+          - "--east-west-endpoints={{mutual_tls_east_west_endpoints}}"
+          - "--ca-cert={{mutual_tls_ca_cert}}"
+          - "--client-cert={{mutual_tls_client_cert}}"
+          - "--client-key={{mutual_tls_client_key}}"
+        timeout: 60
+
       - name: insecure_protocols_test
         phase: test
         command: "python ../../shared/insecure_protocols_test.py"
@@ -208,6 +219,11 @@ tests:
   settings:
     region: "my-isv-region-1"
     insecure_protocols_endpoints: "{{env.EDGE_ENDPOINTS | default('', true)}}"
+    mutual_tls_north_south_endpoints: "{{env.EDGE_ENDPOINTS | default('', true)}}"
+    mutual_tls_east_west_endpoints: "{{env.EAST_WEST_ENDPOINTS | default('', true)}}"
+    mutual_tls_ca_cert: "{{env.MTLS_CA_CERT_PATH | default('', true)}}"
+    mutual_tls_client_cert: "{{env.MTLS_CLIENT_CERT_PATH | default('', true)}}"
+    mutual_tls_client_key: "{{env.MTLS_CLIENT_KEY_PATH | default('', true)}}"
     tenant_id: "{{env.MY_ISV_TENANT_ID | default('demo-tenant', true)}}"
     reservation_id: "demo-capacity-reservation"
     reservation_count: 2