From f848b96a0bee0607bab8db524237cb3ed22c7141 Mon Sep 17 00:00:00 2001 From: gloskull Date: Fri, 17 Jul 2026 18:44:18 +0100 Subject: [PATCH] Add service mesh mTLS integration --- deploy/service-mesh/destination-rules.yaml | 25 +++++++ deploy/service-mesh/monitoring.yaml | 37 ++++++++++ deploy/service-mesh/mtls-policy.yaml | 33 +++++++++ deploy/service-mesh/traffic-policy.yaml | 65 +++++++++++++++++ docs/SERVICE_MESH_MTLS.md | 73 +++++++++++++++++++ scripts/validate_service_mesh.py | 82 ++++++++++++++++++++++ tests/test_service_mesh_manifests.py | 38 ++++++++++ 7 files changed, 353 insertions(+) create mode 100644 deploy/service-mesh/destination-rules.yaml create mode 100644 deploy/service-mesh/monitoring.yaml create mode 100644 deploy/service-mesh/mtls-policy.yaml create mode 100644 deploy/service-mesh/traffic-policy.yaml create mode 100644 docs/SERVICE_MESH_MTLS.md create mode 100755 scripts/validate_service_mesh.py create mode 100644 tests/test_service_mesh_manifests.py diff --git a/deploy/service-mesh/destination-rules.yaml b/deploy/service-mesh/destination-rules.yaml new file mode 100644 index 0000000..a0ff621 --- /dev/null +++ b/deploy/service-mesh/destination-rules.yaml @@ -0,0 +1,25 @@ +apiVersion: networking.istio.io/v1beta1 +kind: DestinationRule +metadata: + name: utility-contracts-default-mtls + namespace: utility-contracts + labels: + app.kubernetes.io/part-of: utility-contracts +spec: + host: "*.utility-contracts.svc.cluster.local" + trafficPolicy: + tls: + mode: ISTIO_MUTUAL + connectionPool: + tcp: + maxConnections: 1000 + connectTimeout: 100ms + http: + http1MaxPendingRequests: 1024 + http2MaxRequests: 4096 + maxRequestsPerConnection: 100 + outlierDetection: + consecutive5xxErrors: 5 + interval: 5s + baseEjectionTime: 30s + maxEjectionPercent: 50 diff --git a/deploy/service-mesh/monitoring.yaml b/deploy/service-mesh/monitoring.yaml new file mode 100644 index 0000000..9dfdc8e --- /dev/null +++ b/deploy/service-mesh/monitoring.yaml @@ -0,0 +1,37 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: utility-contracts-mesh-slo + namespace: monitoring + labels: + app.kubernetes.io/part-of: utility-contracts + prometheus: kube-prometheus + role: alert-rules +spec: + groups: + - name: utility-contracts.mesh.slo + rules: + - alert: UtilityContractsMeshP99LatencyHigh + expr: histogram_quantile(0.99, sum(rate(istio_request_duration_milliseconds_bucket{destination_workload_namespace="utility-contracts"}[5m])) by (le, destination_workload)) > 100 + for: 10m + labels: + severity: page + annotations: + summary: "Utility Contracts service mesh P99 latency exceeds 100ms" + description: "{{ $labels.destination_workload }} has P99 latency above the critical path target." + - alert: UtilityContractsMeshAvailabilityLow + expr: 100 * sum(rate(istio_requests_total{destination_workload_namespace="utility-contracts",response_code!~"5.."}[30m])) / sum(rate(istio_requests_total{destination_workload_namespace="utility-contracts"}[30m])) < 99.99 + for: 10m + labels: + severity: page + annotations: + summary: "Utility Contracts mesh availability is below 99.99%" + description: "Successful request ratio is below the service availability target." + - alert: UtilityContractsMTLSPolicyDrift + expr: sum(istio_requests_total{destination_workload_namespace="utility-contracts",connection_security_policy!="mutual_tls"}) > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Non-mTLS traffic detected in Utility Contracts namespace" + description: "All in-mesh service traffic must use mutual TLS." diff --git a/deploy/service-mesh/mtls-policy.yaml b/deploy/service-mesh/mtls-policy.yaml new file mode 100644 index 0000000..5ae43e2 --- /dev/null +++ b/deploy/service-mesh/mtls-policy.yaml @@ -0,0 +1,33 @@ +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: utility-contracts-strict-mtls + namespace: utility-contracts + labels: + app.kubernetes.io/part-of: utility-contracts +spec: + mtls: + mode: STRICT +--- +apiVersion: security.istio.io/v1beta1 +kind: AuthorizationPolicy +metadata: + name: utility-contracts-service-allowlist + namespace: utility-contracts + labels: + app.kubernetes.io/part-of: utility-contracts +spec: + action: ALLOW + rules: + - from: + - source: + namespaces: ["utility-contracts"] + to: + - operation: + ports: ["8080", "9090"] + - from: + - source: + namespaces: ["istio-system", "monitoring"] + to: + - operation: + ports: ["15020", "9090"] diff --git a/deploy/service-mesh/traffic-policy.yaml b/deploy/service-mesh/traffic-policy.yaml new file mode 100644 index 0000000..ffb8366 --- /dev/null +++ b/deploy/service-mesh/traffic-policy.yaml @@ -0,0 +1,65 @@ +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: utility-contracts-blue-green + namespace: utility-contracts + labels: + app.kubernetes.io/part-of: utility-contracts +spec: + hosts: + - api.utility-contracts.example.com + - utility-api.utility-contracts.svc.cluster.local + gateways: + - utility-contracts-gateway + - mesh + http: + - name: canary + match: + - headers: + x-canary: + exact: "true" + route: + - destination: + host: utility-api.utility-contracts.svc.cluster.local + subset: green + weight: 100 + timeout: 100ms + retries: + attempts: 2 + perTryTimeout: 40ms + retryOn: connect-failure,refused-stream,unavailable,cancelled,5xx + - name: primary + route: + - destination: + host: utility-api.utility-contracts.svc.cluster.local + subset: blue + weight: 100 + - destination: + host: utility-api.utility-contracts.svc.cluster.local + subset: green + weight: 0 + timeout: 100ms + retries: + attempts: 2 + perTryTimeout: 40ms + retryOn: connect-failure,refused-stream,unavailable,cancelled,5xx +--- +apiVersion: networking.istio.io/v1beta1 +kind: DestinationRule +metadata: + name: utility-api-subsets + namespace: utility-contracts + labels: + app.kubernetes.io/part-of: utility-contracts +spec: + host: utility-api.utility-contracts.svc.cluster.local + trafficPolicy: + tls: + mode: ISTIO_MUTUAL + subsets: + - name: blue + labels: + deployment-slot: blue + - name: green + labels: + deployment-slot: green diff --git a/docs/SERVICE_MESH_MTLS.md b/docs/SERVICE_MESH_MTLS.md new file mode 100644 index 0000000..0e73158 --- /dev/null +++ b/docs/SERVICE_MESH_MTLS.md @@ -0,0 +1,73 @@ +# Service Mesh Integration with Mutual TLS + +This runbook documents the Utility Contracts service mesh architecture, mTLS +security posture, observability requirements, and blue-green deployment process. +It is designed for Istio-compatible clusters and keeps critical-path P99 latency +below 100 ms while preserving the 99.99% availability objective. + +## Architecture + +- All workloads in the `utility-contracts` namespace are injected with an Istio + sidecar and communicate through the mesh. +- `PeerAuthentication/utility-contracts-strict-mtls` enforces strict mutual TLS + for every in-namespace service connection. +- `DestinationRule/utility-contracts-default-mtls` originates `ISTIO_MUTUAL` TLS + for service-to-service calls and applies connection pools plus outlier + detection to bound tail latency. +- `VirtualService/utility-contracts-blue-green` routes normal traffic to the + active `blue` subset, supports `green` canary traffic with the `x-canary: true` + header, and caps request timeouts at 100 ms. +- Prometheus alerts detect P99 latency violations, availability drops below + 99.99%, and policy drift where traffic is not protected by mutual TLS. + +## Deployment Procedure + +1. Label the namespace for sidecar injection: + `kubectl label namespace utility-contracts istio-injection=enabled --overwrite`. +2. Apply the mesh manifests: + `kubectl apply -f deploy/service-mesh/`. +3. Deploy the inactive color (`green` for a `blue` active deployment) with the + `deployment-slot=green` label. +4. Send smoke traffic with `x-canary: true` and verify latency, error rate, and + mTLS policy metrics. +5. Increase the `green` route weight in controlled steps: 1%, 5%, 25%, 50%, and + 100%. Hold each step for at least 15 minutes or one business-critical batch, + whichever is longer. +6. Keep the previous color healthy for fast rollback until the new color has met + the SLOs for one full observation window. + +## Canary Analysis Gates + +Promote the canary only when all gates pass: + +- P99 request latency is at or below 100 ms for critical paths. +- Successful request ratio is at least 99.99% over the observation window. +- `UtilityContractsMTLSPolicyDrift` is inactive. +- No new 5xx spike or outlier-ejection pattern appears in mesh telemetry. +- Security review confirms namespace, authorization, and TLS policies match the + intended allowlist. + +## Rollback + +1. Set the `green` route weight to `0` and the last known-good `blue` route to + `100`. +2. Confirm `istio_requests_total` returns to the pre-deployment error budget + burn rate. +3. Keep the failed color running only long enough to collect diagnostics, then + scale it down. +4. Open a post-incident review and update this runbook if any gate or alert did + not trigger as expected. + +## Validation + +Run the offline manifest validator before review and deployment: + +```bash +python3 scripts/validate_service_mesh.py +``` + +Run the pytest coverage when Python test dependencies are available: + +```bash +python3 -m pytest tests/test_service_mesh_manifests.py +``` diff --git a/scripts/validate_service_mesh.py b/scripts/validate_service_mesh.py new file mode 100755 index 0000000..6cedffd --- /dev/null +++ b/scripts/validate_service_mesh.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Validate Utility Contracts service mesh manifests without cluster access.""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MANIFEST_DIR = ROOT / "deploy" / "service-mesh" +REQUIRED_FILES = ( + "mtls-policy.yaml", + "destination-rules.yaml", + "traffic-policy.yaml", + "monitoring.yaml", +) + + +def read_manifest(name: str) -> str: + return (MANIFEST_DIR / name).read_text(encoding="utf-8") + + +def require(pattern: str, text: str, message: str) -> None: + if not re.search(pattern, text, re.MULTILINE): + raise AssertionError(message) + + +def assert_strict_mtls(text: str) -> None: + require(r"kind:\s*PeerAuthentication", text, "PeerAuthentication is required") + require(r"name:\s*utility-contracts-strict-mtls", text, "strict mTLS policy must be named") + require(r"mode:\s*STRICT", text, "PeerAuthentication must enforce STRICT mTLS") + require(r"kind:\s*AuthorizationPolicy", text, "AuthorizationPolicy is required") + require(r"action:\s*ALLOW", text, "authorization policy must be an explicit allowlist") + + +def assert_destination_rules(text: str) -> None: + require(r"kind:\s*DestinationRule", text, "default DestinationRule is required") + require(r"host:\s*\"\*\.utility-contracts\.svc\.cluster\.local\"", text, "default host wildcard is required") + require(r"mode:\s*ISTIO_MUTUAL", text, "DestinationRule must originate ISTIO_MUTUAL TLS") + require(r"connectTimeout:\s*100ms", text, "connect timeout must preserve the 100ms P99 budget") + require(r"outlierDetection:", text, "outlier detection must be configured") + + +def assert_blue_green_and_canary(text: str) -> None: + require(r"kind:\s*VirtualService", text, "VirtualService is required") + require(r"name:\s*canary", text, "canary route is required") + require(r"x-canary:", text, "canary header match is required") + require(r"name:\s*primary", text, "primary route is required") + require(r"timeout:\s*100ms", text, "routes must cap request timeout at 100ms") + require(r"name:\s*blue", text, "blue subset is required") + require(r"name:\s*green", text, "green subset is required") + require(r"mode:\s*ISTIO_MUTUAL", text, "subset DestinationRule must use ISTIO_MUTUAL") + + +def assert_monitoring_rules(text: str) -> None: + require(r"kind:\s*PrometheusRule", text, "PrometheusRule is required") + for alert in ( + "UtilityContractsMeshP99LatencyHigh", + "UtilityContractsMeshAvailabilityLow", + "UtilityContractsMTLSPolicyDrift", + ): + require(rf"alert:\s*{alert}", text, f"{alert} alert is required") + + +def main() -> int: + for name in REQUIRED_FILES: + if not (MANIFEST_DIR / name).is_file(): + raise FileNotFoundError(MANIFEST_DIR / name) + assert_strict_mtls(read_manifest("mtls-policy.yaml")) + assert_destination_rules(read_manifest("destination-rules.yaml")) + assert_blue_green_and_canary(read_manifest("traffic-policy.yaml")) + assert_monitoring_rules(read_manifest("monitoring.yaml")) + print("service mesh manifest validation passed") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (AssertionError, FileNotFoundError) as error: + print(f"service mesh manifest validation failed: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/tests/test_service_mesh_manifests.py b/tests/test_service_mesh_manifests.py new file mode 100644 index 0000000..5adaec6 --- /dev/null +++ b/tests/test_service_mesh_manifests.py @@ -0,0 +1,38 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MANIFEST_DIR = ROOT / "deploy" / "service-mesh" + + +def manifest(name): + return (MANIFEST_DIR / name).read_text(encoding="utf-8") + + +def test_strict_mtls_peer_authentication(): + text = manifest("mtls-policy.yaml") + assert "kind: PeerAuthentication" in text + assert "name: utility-contracts-strict-mtls" in text + assert "mode: STRICT" in text + + +def test_all_destination_rules_use_istio_mutual_tls(): + text = manifest("destination-rules.yaml") + manifest("traffic-policy.yaml") + assert text.count("kind: DestinationRule") >= 2 + assert text.count("mode: ISTIO_MUTUAL") >= 2 + + +def test_virtual_service_preserves_latency_budget_and_blue_green_subsets(): + text = manifest("traffic-policy.yaml") + assert "kind: VirtualService" in text + assert "name: canary" in text + assert "name: primary" in text + assert "timeout: 100ms" in text + assert "name: blue" in text + assert "name: green" in text + + +def test_monitoring_covers_latency_availability_and_mtls_drift(): + text = manifest("monitoring.yaml") + assert "alert: UtilityContractsMeshP99LatencyHigh" in text + assert "alert: UtilityContractsMeshAvailabilityLow" in text + assert "alert: UtilityContractsMTLSPolicyDrift" in text