diff --git a/airflow/dags/run_ogc_process.py b/airflow/dags/run_ogc_process.py index 46b8e521..cb2f9469 100644 --- a/airflow/dags/run_ogc_process.py +++ b/airflow/dags/run_ogc_process.py @@ -19,7 +19,7 @@ from kubernetes.client import models as k8s from unity_sps_utils import POD_LABEL, POD_NAMESPACE, get_affinity -API_HOST = "https://api.dit.maap-project.org/api/" +API_HOST = "https://api.uat.maap-project.org/api/" def fetch_ogc_processes(): @@ -237,7 +237,7 @@ def execute(self, context): description="Select a process to execute.", ), "queue": Param( - "maap-dps-worker-cardamom", + "maap-dps-worker-8gb", type="string", title="Queue", description="The MAAP queue to submit the job to", diff --git a/airflow/helm/values.tmpl.yaml b/airflow/helm/values.tmpl.yaml index 94f9075d..1ed89de1 100644 --- a/airflow/helm/values.tmpl.yaml +++ b/airflow/helm/values.tmpl.yaml @@ -56,10 +56,10 @@ affinity: values: ["on-demand"] - key: "karpenter.k8s.aws/instance-family" operator: "In" - values: ["r5"] + values: ["m5", "m6i", "t3"] - key: "karpenter.k8s.aws/instance-cpu" operator: "In" - values: ["8"] + values: ["4"] topologySpreadConstraints: - maxSkew: 1 @@ -93,11 +93,10 @@ scheduler: values: ["on-demand"] - key: "karpenter.k8s.aws/instance-family" operator: "In" - # values: ["c6i", "c5"] # Choosing compute-optimized instances - values: ["r5"] # Choosing memory-optimized instance + values: ["m5", "m6i", "t3"] - key: "karpenter.k8s.aws/instance-cpu" operator: "In" - values: ["8"] + values: ["4"] topologySpreadConstraints: - maxSkew: 1 topologyKey: "topology.kubernetes.io/zone" @@ -130,11 +129,10 @@ triggerer: values: [ "on-demand" ] - key: "karpenter.k8s.aws/instance-family" operator: "In" - # values: ["c6i", "c5"] # Choosing compute-optimized instances - values: [ "r5" ] # Choosing memory-optimized instance + values: ["m5", "m6i", "t3"] - key: "karpenter.k8s.aws/instance-cpu" operator: "In" - values: [ "8" ] # Scheduler might benefit from higher CPU + values: ["4"] postgresql: enabled: false @@ -154,11 +152,10 @@ pgbouncer: values: [ "on-demand" ] - key: "karpenter.k8s.aws/instance-family" operator: "In" - # values: ["c6i", "c5"] # Choosing compute-optimized instances - values: [ "r5" ] # Choosing memory-optimized instance + values: ["m5", "m6i", "t3"] - key: "karpenter.k8s.aws/instance-cpu" operator: "In" - values: [ "8" ] # Scheduler might benefit from higher CPU + values: ["4"] webserverSecretKeySecretName: ${webserver_secret_name} @@ -186,11 +183,10 @@ webserver: values: ["on-demand"] - key: "karpenter.k8s.aws/instance-family" operator: "In" - # values: ["c6i", "c5"] # Choosing compute-optimized instances - values: ["r5"] # Choosing memory-optimized instance + values: ["m5", "m6i", "t3"] - key: "karpenter.k8s.aws/instance-cpu" operator: "In" - values: ["8"] # Balancing between CPU and memory + values: ["4"] # Balancing between CPU and memory topologySpreadConstraints: - maxSkew: 1 topologyKey: "topology.kubernetes.io/zone" @@ -224,11 +220,10 @@ workers: - matchExpressions: - key: "karpenter.k8s.aws/instance-family" operator: "In" - # values: ["c6i", "c5"] # Choosing compute-optimized instances - values: ["r5"] # Choosing memory-optimized instance + values: ["m5", "m6i", "t3"] - key: "karpenter.k8s.aws/instance-cpu" operator: "In" - values: ["8"] + values: ["4"] topologySpreadConstraints: - maxSkew: 1 topologyKey: "topology.kubernetes.io/zone" @@ -318,11 +313,10 @@ dagProcessor: values: [ "on-demand" ] - key: "karpenter.k8s.aws/instance-family" operator: "In" - # values: ["c6i", "c5"] # Choosing compute-optimized instances - values: [ "r5" ] # Choosing memory-optimized instance + values: ["m5", "m6i", "t3"] - key: "karpenter.k8s.aws/instance-cpu" operator: "In" - values: [ "8" ] # Scheduler might benefit from higher CPU + values: ["4"] env: - name: "AIRFLOW_VAR_KUBERNETES_PIPELINE_NAMESPACE" diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf b/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf index f3513533..a9bfa754 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/locals.tf @@ -28,4 +28,12 @@ locals { }[var.venue] # BASE_URL uses placeholder initially, updated by null_resource after LB is created airflow_base_url = "http://placeholder:${local.load_balancer_port}" + keycloak_client_secret_ssm_param = { + "ops" = "/sps/ops/keycloak/client_secret" + "dev" = "/sps/dev/keycloak/client_secret" + }[lower(var.venue)] + keycloak_client_id = { + "ops" = "airflow-ops" + "dev" = "airflow-dev" + }[lower(var.venue)] } diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf index c22f610a..1ef830de 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/main.tf @@ -63,7 +63,7 @@ resource "kubernetes_secret" "airflow_oidc" { data "aws_ssm_parameter" "keycloak_client_secret" { count = var.enable_oidc_auth ? 1 : 0 - name = var.keycloak_client_secret_ssm_param + name = local.keycloak_client_secret_ssm_param } # TODO evaluate if this role is still necessary @@ -435,7 +435,7 @@ resource "helm_release" "airflow" { webserver_config = indent(4, templatefile("${path.module}/../../../airflow/config/webserver_config.py.tpl", { keycloak_role_mapping = var.keycloak_role_mapping keycloak_provider_url = var.keycloak_provider_url - keycloak_client_id = var.keycloak_client_id + keycloak_client_id = local.keycloak_client_id })) }) ] diff --git a/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf b/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf index e0f5b7fe..99d0aec5 100644 --- a/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-airflow/variables.tf @@ -94,12 +94,6 @@ variable "keycloak_client_id" { default = "airflow" } -variable "keycloak_client_secret_ssm_param" { - description = "SSM parameter path containing Keycloak OIDC client secret" - type = string - default = "/sps/keycloak/client_secret" -} - variable "enable_oidc_auth" { description = "Enable Keycloak OIDC authentication for Airflow" type = bool diff --git a/terraform-unity/modules/terraform-unity-sps-eks/README.md b/terraform-unity/modules/terraform-unity-sps-eks/README.md index 115b3c8c..7c7f801d 100644 --- a/terraform-unity/modules/terraform-unity-sps-eks/README.md +++ b/terraform-unity/modules/terraform-unity-sps-eks/README.md @@ -38,7 +38,7 @@ | [cluster\_version](#input\_cluster\_version) | The EKS cluster version (must be supported by the cs-infra module). | `string` | `"1.33"` | no | | [deployment\_name](#input\_deployment\_name) | The name of the deployment. | `string` | `""` | no | | [installprefix](#input\_installprefix) | The install prefix for the service area (unused) | `string` | `""` | no | -| [nodegroups](#input\_nodegroups) | A map of node group configurations |
map(object({
create_iam_role = optional(bool)
iam_role_arn = optional(string)
ami_id = optional(string)
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
instance_types = optional(list(string))
capacity_type = optional(string)
metadata_options = optional(map(any))
block_device_mappings = optional(map(object({
device_name = string
ebs = object({
volume_size = number
volume_type = string
encrypted = bool
delete_on_termination = bool
})
})))
}))
|
{
"defaultGroup": {
"block_device_mappings": {
"xvda": {
"device_name": "/dev/xvda",
"ebs": {
"delete_on_termination": true,
"encrypted": true,
"volume_size": 100,
"volume_type": "gp2"
}
}
},
"desired_size": 1,
"instance_types": [
"m5.2xlarge"
],
"max_size": 1,
"metadata_options": {
"http_endpoint": "enabled",
"http_put_response_hop_limit": 3
},
"min_size": 1
}
}
| no | +| [nodegroups](#input\_nodegroups) | A map of node group configurations |
map(object({
create_iam_role = optional(bool)
iam_role_arn = optional(string)
ami_id = optional(string)
min_size = optional(number)
max_size = optional(number)
desired_size = optional(number)
instance_types = optional(list(string))
capacity_type = optional(string)
metadata_options = optional(map(any))
block_device_mappings = optional(map(object({
device_name = string
ebs = object({
volume_size = number
volume_type = string
encrypted = bool
delete_on_termination = bool
})
})))
}))
|
{
"defaultGroup": {
"block_device_mappings": {
"xvda": {
"device_name": "/dev/xvda",
"ebs": {
"delete_on_termination": true,
"encrypted": true,
"volume_size": 100,
"volume_type": "gp2"
}
}
},
"desired_size": 1,
"instance_types": [
"m5.xlarge"
],
"max_size": 1,
"metadata_options": {
"http_endpoint": "enabled",
"http_put_response_hop_limit": 3
},
"min_size": 1
}
}
| no | | [project](#input\_project) | The project or mission deploying Unity SPS | `string` | `"unity"` | no | | [release](#input\_release) | The software release version. | `string` | `"25.3"` | no | | [service\_area](#input\_service\_area) | The service area owner of the resources being deployed | `string` | `"sps"` | no | diff --git a/terraform-unity/modules/terraform-unity-sps-eks/variables.tf b/terraform-unity/modules/terraform-unity-sps-eks/variables.tf index 05c4f592..21d5e8bc 100644 --- a/terraform-unity/modules/terraform-unity-sps-eks/variables.tf +++ b/terraform-unity/modules/terraform-unity-sps-eks/variables.tf @@ -72,7 +72,7 @@ variable "nodegroups" { })) default = { defaultGroup = { - instance_types = ["m5.2xlarge"] + instance_types = ["c5.xlarge"] min_size = 1 max_size = 1 desired_size = 1 diff --git a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf index 8912389c..69ee66a0 100644 --- a/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf +++ b/terraform-unity/modules/terraform-unity-sps-ogc-processes-api/main.tf @@ -34,12 +34,12 @@ resource "kubernetes_deployment" "redis" { match_expressions { key = "karpenter.k8s.aws/instance-family" operator = "In" - values = ["r5"] + values = ["m5", "m6i", "t3"] } match_expressions { key = "karpenter.k8s.aws/instance-cpu" operator = "In" - values = ["8"] + values = ["4"] } } } @@ -111,12 +111,12 @@ resource "kubernetes_deployment" "ogc_processes_api" { match_expressions { key = "karpenter.k8s.aws/instance-family" operator = "In" - values = ["r5"] + values = ["m5", "m6i", "t3"] } match_expressions { key = "karpenter.k8s.aws/instance-cpu" operator = "In" - values = ["8"] + values = ["4"] } } } diff --git a/terraform-unity/variables.tf b/terraform-unity/variables.tf index 4e6bbe48..9d590a7c 100644 --- a/terraform-unity/variables.tf +++ b/terraform-unity/variables.tf @@ -169,7 +169,7 @@ variable "karpenter_node_pools" { { key = "karpenter.k8s.aws/instance-family" operator = "In" - values = ["m7i", "m6i", "m5", "m5ad", "t3", "c7i", "c6i", "c6id", "c5", "r7i", "r6i", "r5"] + values = ["m7i", "m6i", "m5", "t3", "c7i", "c6i", "c5"] }, { key = "karpenter.k8s.aws/instance-cpu" @@ -213,7 +213,7 @@ variable "karpenter_node_pools" { { key = "karpenter.k8s.aws/instance-family" operator = "In" - values = ["m7i", "m6i", "m5", "m5ad", "t3", "c7i", "c6i", "c6id", "c5", "r7i", "r6i", "r5"] + values = ["m7i", "m6i", "m5", "t3", "c7i", "c6i", "c5"] }, { key = "karpenter.k8s.aws/instance-cpu" @@ -257,7 +257,7 @@ variable "karpenter_node_pools" { { key = "karpenter.k8s.aws/instance-family" operator = "In" - values = ["m7i", "m6i", "m5", "t3", "c7i", "c6i", "c6id", "c5", "r7i", "r6i", "r5", "m5ad"] + values = ["m7i", "m6i", "m5", "t3", "c7i", "c6i", "c5"] }, { key = "karpenter.k8s.aws/instance-cpu" @@ -300,7 +300,7 @@ variable "karpenter_node_pools" { { key = "karpenter.k8s.aws/instance-family" operator = "In" - values = ["m7i", "m6i", "m5", "t3", "c7i", "c6i", "c6id", "c5", "r7i", "r6i", "r5", "m5ad"] + values = ["m7i", "m6i", "m5", "t3", "c7i", "c6i", "c5"] }, { key = "karpenter.k8s.aws/instance-cpu" @@ -310,7 +310,7 @@ variable "karpenter_node_pools" { { key = "karpenter.k8s.aws/instance-cpu" operator = "Lt" - values = ["17"] // To 16 inclusive + values = ["5"] // To 4 inclusive (m5.xlarge max) }, { key = "karpenter.k8s.aws/instance-memory" @@ -320,7 +320,7 @@ variable "karpenter_node_pools" { { key = "karpenter.k8s.aws/instance-memory" operator = "Lt" - values = ["65537"] // To 64 GB inclusive + values = ["16385"] // To 16 GB inclusive (m5.xlarge max) }, { key = "karpenter.k8s.aws/instance-hypervisor", @@ -378,5 +378,5 @@ variable "installprefix" { variable "db_instance_class" { description = "The SPS RDS database instance class" type = string - default = "db.m5d.xlarge" + default = "db.t3.medium" }