Service mesh integration with mutual

deploy/service-mesh/destination-rules.yaml

@@ -0,0 +1,25 @@
+apiVersion: networking.istio.io/v1beta1
+kind: DestinationRule
+metadata:
+  name: utility-contracts-default-mtls
+  namespace: utility-contracts
+  labels:
+    app.kubernetes.io/part-of: utility-contracts
+spec:
+  host: "*.utility-contracts.svc.cluster.local"
+  trafficPolicy:
+    tls:
+      mode: ISTIO_MUTUAL
+    connectionPool:
+      tcp:
+        maxConnections: 1000
+        connectTimeout: 100ms
+      http:
+        http1MaxPendingRequests: 1024
+        http2MaxRequests: 4096
+        maxRequestsPerConnection: 100
+    outlierDetection:
+      consecutive5xxErrors: 5
+      interval: 5s
+      baseEjectionTime: 30s
+      maxEjectionPercent: 50

deploy/service-mesh/monitoring.yaml

@@ -0,0 +1,37 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: utility-contracts-mesh-slo
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/part-of: utility-contracts
+    prometheus: kube-prometheus
+    role: alert-rules
+spec:
+  groups:
+    - name: utility-contracts.mesh.slo
+      rules:
+        - alert: UtilityContractsMeshP99LatencyHigh
+          expr: histogram_quantile(0.99, sum(rate(istio_request_duration_milliseconds_bucket{destination_workload_namespace="utility-contracts"}[5m])) by (le, destination_workload)) > 100
+          for: 10m
+          labels:
+            severity: page
+          annotations:
+            summary: "Utility Contracts service mesh P99 latency exceeds 100ms"
+            description: "{{ $labels.destination_workload }} has P99 latency above the critical path target."
+        - alert: UtilityContractsMeshAvailabilityLow
+          expr: 100 * sum(rate(istio_requests_total{destination_workload_namespace="utility-contracts",response_code!~"5.."}[30m])) / sum(rate(istio_requests_total{destination_workload_namespace="utility-contracts"}[30m])) < 99.99
+          for: 10m
+          labels:
+            severity: page
+          annotations:
+            summary: "Utility Contracts mesh availability is below 99.99%"
+            description: "Successful request ratio is below the service availability target."
+        - alert: UtilityContractsMTLSPolicyDrift
+          expr: sum(istio_requests_total{destination_workload_namespace="utility-contracts",connection_security_policy!="mutual_tls"}) > 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Non-mTLS traffic detected in Utility Contracts namespace"
+            description: "All in-mesh service traffic must use mutual TLS."

deploy/service-mesh/mtls-policy.yaml

@@ -0,0 +1,33 @@
+apiVersion: security.istio.io/v1beta1
+kind: PeerAuthentication
+metadata:
+  name: utility-contracts-strict-mtls
+  namespace: utility-contracts
+  labels:
+    app.kubernetes.io/part-of: utility-contracts
+spec:
+  mtls:
+    mode: STRICT
+---
+apiVersion: security.istio.io/v1beta1
+kind: AuthorizationPolicy
+metadata:
+  name: utility-contracts-service-allowlist
+  namespace: utility-contracts
+  labels:
+    app.kubernetes.io/part-of: utility-contracts
+spec:
+  action: ALLOW
+  rules:
+    - from:
+        - source:
+            namespaces: ["utility-contracts"]
+      to:
+        - operation:
+            ports: ["8080", "9090"]
+    - from:
+        - source:
+            namespaces: ["istio-system", "monitoring"]
+      to:
+        - operation:
+            ports: ["15020", "9090"]

deploy/service-mesh/traffic-policy.yaml

@@ -0,0 +1,65 @@
+apiVersion: networking.istio.io/v1beta1
+kind: VirtualService
+metadata:
+  name: utility-contracts-blue-green
+  namespace: utility-contracts
+  labels:
+    app.kubernetes.io/part-of: utility-contracts
+spec:
+  hosts:
+    - api.utility-contracts.example.com
+    - utility-api.utility-contracts.svc.cluster.local
+  gateways:
+    - utility-contracts-gateway
+    - mesh
+  http:
+    - name: canary
+      match:
+        - headers:
+            x-canary:
+              exact: "true"
+      route:
+        - destination:
+            host: utility-api.utility-contracts.svc.cluster.local
+            subset: green
+          weight: 100
+      timeout: 100ms
+      retries:
+        attempts: 2
+        perTryTimeout: 40ms
+        retryOn: connect-failure,refused-stream,unavailable,cancelled,5xx
+    - name: primary
+      route:
+        - destination:
+            host: utility-api.utility-contracts.svc.cluster.local
+            subset: blue
+          weight: 100
+        - destination:
+            host: utility-api.utility-contracts.svc.cluster.local
+            subset: green
+          weight: 0
+      timeout: 100ms
+      retries:
+        attempts: 2
+        perTryTimeout: 40ms
+        retryOn: connect-failure,refused-stream,unavailable,cancelled,5xx
+---
+apiVersion: networking.istio.io/v1beta1
+kind: DestinationRule
+metadata:
+  name: utility-api-subsets
+  namespace: utility-contracts
+  labels:
+    app.kubernetes.io/part-of: utility-contracts
+spec:
+  host: utility-api.utility-contracts.svc.cluster.local
+  trafficPolicy:
+    tls:
+      mode: ISTIO_MUTUAL
+  subsets:
+    - name: blue
+      labels:
+        deployment-slot: blue
+    - name: green
+      labels:
+        deployment-slot: green

docs/SERVICE_MESH_MTLS.md

@@ -0,0 +1,73 @@
+# Service Mesh Integration with Mutual TLS
+
+This runbook documents the Utility Contracts service mesh architecture, mTLS
+security posture, observability requirements, and blue-green deployment process.
+It is designed for Istio-compatible clusters and keeps critical-path P99 latency
+below 100 ms while preserving the 99.99% availability objective.
+
+## Architecture
+
+- All workloads in the `utility-contracts` namespace are injected with an Istio
+  sidecar and communicate through the mesh.
+- `PeerAuthentication/utility-contracts-strict-mtls` enforces strict mutual TLS
+  for every in-namespace service connection.
+- `DestinationRule/utility-contracts-default-mtls` originates `ISTIO_MUTUAL` TLS
+  for service-to-service calls and applies connection pools plus outlier
+  detection to bound tail latency.
+- `VirtualService/utility-contracts-blue-green` routes normal traffic to the
+  active `blue` subset, supports `green` canary traffic with the `x-canary: true`
+  header, and caps request timeouts at 100 ms.
+- Prometheus alerts detect P99 latency violations, availability drops below
+  99.99%, and policy drift where traffic is not protected by mutual TLS.
+
+## Deployment Procedure
+
+1. Label the namespace for sidecar injection:
+   `kubectl label namespace utility-contracts istio-injection=enabled --overwrite`.
+2. Apply the mesh manifests:
+   `kubectl apply -f deploy/service-mesh/`.
+3. Deploy the inactive color (`green` for a `blue` active deployment) with the
+   `deployment-slot=green` label.
+4. Send smoke traffic with `x-canary: true` and verify latency, error rate, and
+   mTLS policy metrics.
+5. Increase the `green` route weight in controlled steps: 1%, 5%, 25%, 50%, and
+   100%. Hold each step for at least 15 minutes or one business-critical batch,
+   whichever is longer.
+6. Keep the previous color healthy for fast rollback until the new color has met
+   the SLOs for one full observation window.
+
+## Canary Analysis Gates
+
+Promote the canary only when all gates pass:
+
+- P99 request latency is at or below 100 ms for critical paths.
+- Successful request ratio is at least 99.99% over the observation window.
+- `UtilityContractsMTLSPolicyDrift` is inactive.
+- No new 5xx spike or outlier-ejection pattern appears in mesh telemetry.
+- Security review confirms namespace, authorization, and TLS policies match the
+  intended allowlist.
+
+## Rollback
+
+1. Set the `green` route weight to `0` and the last known-good `blue` route to
+   `100`.
+2. Confirm `istio_requests_total` returns to the pre-deployment error budget
+   burn rate.
+3. Keep the failed color running only long enough to collect diagnostics, then
+   scale it down.
+4. Open a post-incident review and update this runbook if any gate or alert did
+   not trigger as expected.
+
+## Validation
+
+Run the offline manifest validator before review and deployment:
+
+```bash
+python3 scripts/validate_service_mesh.py
+```
+
+Run the pytest coverage when Python test dependencies are available:
+
+```bash
+python3 -m pytest tests/test_service_mesh_manifests.py
+```

scripts/validate_service_mesh.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""Validate Utility Contracts service mesh manifests without cluster access."""
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+MANIFEST_DIR = ROOT / "deploy" / "service-mesh"
+REQUIRED_FILES = (
+    "mtls-policy.yaml",
+    "destination-rules.yaml",
+    "traffic-policy.yaml",
+    "monitoring.yaml",
+)
+
+
+def read_manifest(name: str) -> str:
+    return (MANIFEST_DIR / name).read_text(encoding="utf-8")
+
+
+def require(pattern: str, text: str, message: str) -> None:
+    if not re.search(pattern, text, re.MULTILINE):
+        raise AssertionError(message)
+
+
+def assert_strict_mtls(text: str) -> None:
+    require(r"kind:\s*PeerAuthentication", text, "PeerAuthentication is required")
+    require(r"name:\s*utility-contracts-strict-mtls", text, "strict mTLS policy must be named")
+    require(r"mode:\s*STRICT", text, "PeerAuthentication must enforce STRICT mTLS")
+    require(r"kind:\s*AuthorizationPolicy", text, "AuthorizationPolicy is required")
+    require(r"action:\s*ALLOW", text, "authorization policy must be an explicit allowlist")
+
+
+def assert_destination_rules(text: str) -> None:
+    require(r"kind:\s*DestinationRule", text, "default DestinationRule is required")
+    require(r"host:\s*\"\*\.utility-contracts\.svc\.cluster\.local\"", text, "default host wildcard is required")
+    require(r"mode:\s*ISTIO_MUTUAL", text, "DestinationRule must originate ISTIO_MUTUAL TLS")
+    require(r"connectTimeout:\s*100ms", text, "connect timeout must preserve the 100ms P99 budget")
+    require(r"outlierDetection:", text, "outlier detection must be configured")
+
+
+def assert_blue_green_and_canary(text: str) -> None:
+    require(r"kind:\s*VirtualService", text, "VirtualService is required")
+    require(r"name:\s*canary", text, "canary route is required")
+    require(r"x-canary:", text, "canary header match is required")
+    require(r"name:\s*primary", text, "primary route is required")
+    require(r"timeout:\s*100ms", text, "routes must cap request timeout at 100ms")
+    require(r"name:\s*blue", text, "blue subset is required")
+    require(r"name:\s*green", text, "green subset is required")
+    require(r"mode:\s*ISTIO_MUTUAL", text, "subset DestinationRule must use ISTIO_MUTUAL")
+
+
+def assert_monitoring_rules(text: str) -> None:
+    require(r"kind:\s*PrometheusRule", text, "PrometheusRule is required")
+    for alert in (
+        "UtilityContractsMeshP99LatencyHigh",
+        "UtilityContractsMeshAvailabilityLow",
+        "UtilityContractsMTLSPolicyDrift",
+    ):
+        require(rf"alert:\s*{alert}", text, f"{alert} alert is required")
+
+
+def main() -> int:
+    for name in REQUIRED_FILES:
+        if not (MANIFEST_DIR / name).is_file():
+            raise FileNotFoundError(MANIFEST_DIR / name)
+    assert_strict_mtls(read_manifest("mtls-policy.yaml"))
+    assert_destination_rules(read_manifest("destination-rules.yaml"))
+    assert_blue_green_and_canary(read_manifest("traffic-policy.yaml"))
+    assert_monitoring_rules(read_manifest("monitoring.yaml"))
+    print("service mesh manifest validation passed")
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except (AssertionError, FileNotFoundError) as error:
+        print(f"service mesh manifest validation failed: {error}", file=sys.stderr)
+        raise SystemExit(1)

tests/test_service_mesh_manifests.py

@@ -0,0 +1,38 @@
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+MANIFEST_DIR = ROOT / "deploy" / "service-mesh"
+
+
+def manifest(name):
+    return (MANIFEST_DIR / name).read_text(encoding="utf-8")
+
+
+def test_strict_mtls_peer_authentication():
+    text = manifest("mtls-policy.yaml")
+    assert "kind: PeerAuthentication" in text
+    assert "name: utility-contracts-strict-mtls" in text
+    assert "mode: STRICT" in text
+
+
+def test_all_destination_rules_use_istio_mutual_tls():
+    text = manifest("destination-rules.yaml") + manifest("traffic-policy.yaml")
+    assert text.count("kind: DestinationRule") >= 2
+    assert text.count("mode: ISTIO_MUTUAL") >= 2
+
+
+def test_virtual_service_preserves_latency_budget_and_blue_green_subsets():
+    text = manifest("traffic-policy.yaml")
+    assert "kind: VirtualService" in text
+    assert "name: canary" in text
+    assert "name: primary" in text
+    assert "timeout: 100ms" in text
+    assert "name: blue" in text
+    assert "name: green" in text
+
+
+def test_monitoring_covers_latency_availability_and_mtls_drift():
+    text = manifest("monitoring.yaml")
+    assert "alert: UtilityContractsMeshP99LatencyHigh" in text
+    assert "alert: UtilityContractsMeshAvailabilityLow" in text
+    assert "alert: UtilityContractsMTLSPolicyDrift" in text