From 265edccea117905998009576e7bf5bacb40eff3a Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 24 Jun 2026 14:35:45 -0700 Subject: [PATCH 1/5] feat(terraform): bring-your-own Data Prepper pipeline + gp3 StorageClass Add extra_helm_values and data_prepper_pipeline_secret_file variables so an operator can supply a custom Data Prepper pipeline through the chart's dataPrepperManageSecret=false gate: terraform renders the file into the data-prepper-pipeline Secret and orders it before the release. Create a gp3 StorageClass (EKS ships only gp2) since the OpenSearch/Cortex storage_class variables default to gp3. Signed-off-by: Kyle Hounslow --- terraform/aws/observability-stack.tf | 58 +++++++++++++++++++++++++++- terraform/aws/variables.tf | 12 ++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/terraform/aws/observability-stack.tf b/terraform/aws/observability-stack.tf index 2490081e..4a1c8d35 100644 --- a/terraform/aws/observability-stack.tf +++ b/terraform/aws/observability-stack.tf @@ -109,6 +109,49 @@ resource "kubernetes_namespace" "observability" { depends_on = [module.eks] } +# EKS ships only a gp2 StorageClass. opensearch_storage_class / cortex_storage_class +# default to gp3 (better IOPS/$ for bulk ingest), so create it via the EBS CSI driver. +# WaitForFirstConsumer matches gp2 so a PVC binds on the node its pod lands on. +resource "kubernetes_storage_class" "gp3" { + metadata { + name = "gp3" + } + storage_provisioner = "ebs.csi.aws.com" + volume_binding_mode = "WaitForFirstConsumer" + allow_volume_expansion = true + parameters = { + type = "gp3" + } + + depends_on = [module.eks] +} + +# Bring-your-own Data Prepper pipeline. When data_prepper_pipeline_secret_file +# is set, create the Secret the chart's dataPrepperManageSecret=false gate +# expects (same name the subchart mounts) and order it before the release so +# Data Prepper finds it at boot. The chart default leaves this empty and +# renders its own pipeline Secret. +locals { + opensearch_password_effective = var.opensearch_password != "" ? var.opensearch_password : "My_password_123!@#" +} + +resource "kubernetes_secret" "data_prepper_pipeline" { + count = var.data_prepper_pipeline_secret_file != "" ? 1 : 0 + + metadata { + name = "data-prepper-pipeline" + namespace = kubernetes_namespace.observability.metadata[0].name + } + + data = { + "pipelines.yaml" = templatefile(var.data_prepper_pipeline_secret_file, { + opensearch_user = "admin" + opensearch_password = local.opensearch_password_effective + trace_flush_interval = var.data_prepper_trace_flush_interval + }) + } +} + resource "helm_release" "observability_stack" { name = "obs-stack" chart = "${path.module}/../../charts/observability-stack" @@ -121,9 +164,20 @@ resource "helm_release" "observability_stack" { values = concat( [file("${path.module}/values-eks.yaml")], - var.anonymous_auth ? [file("${path.module}/../../charts/observability-stack/values-anonymous-auth.yaml")] : [] + var.anonymous_auth ? [file("${path.module}/../../charts/observability-stack/values-anonymous-auth.yaml")] : [], + [for f in var.extra_helm_values : file(f)] ) + # Bring-your-own pipeline: skip the chart's managed Secret so Data Prepper + # mounts the one created above. + dynamic "set" { + for_each = var.data_prepper_pipeline_secret_file != "" ? [1] : [] + content { + name = "dataPrepperManageSecret" + value = "false" + } + } + # --- TLS / Domain (conditional) --- dynamic "set" { for_each = local.enable_tls ? [1] : [] @@ -260,5 +314,7 @@ resource "helm_release" "observability_stack" { depends_on = [ helm_release.aws_lb_controller, + kubernetes_secret.data_prepper_pipeline, + kubernetes_storage_class.gp3, ] } diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf index 72d98860..8cfa6275 100644 --- a/terraform/aws/variables.tf +++ b/terraform/aws/variables.tf @@ -93,6 +93,18 @@ variable "tags" { } } +variable "extra_helm_values" { + description = "Additional Helm values file paths layered onto the release, applied last so they win. Use for deployment-specific overrides without editing chart defaults." + type = list(string) + default = [] +} + +variable "data_prepper_pipeline_secret_file" { + description = "Path to a pipelines.yaml template for a bring-your-own Data Prepper pipeline. When set, terraform creates the data-prepper-pipeline Secret from it and sets dataPrepperManageSecret=false so the chart renders no pipeline Secret. The template is rendered with templatefile and may reference opensearch_user, opensearch_password, and trace_flush_interval. Empty leaves the chart's managed pipeline in place." + type = string + default = "" +} + # ============================================================================ # OpenSearch sizing # ============================================================================ From 009f1b75e9f746ae5dab760dfa7d0bfc19ac67ff Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 24 Jun 2026 22:38:03 -0700 Subject: [PATCH 2/5] fix(collector): default debug exporter to verbosity basic verbosity detailed serializes every span, log, and metric to stdout, which throttles the collector pipeline under sustained load (observed ~6x slower trace ingest at bench scale, with OpenSearch write pools idle). basic logs per-batch counts, which is enough for the debug exporter's purpose. Signed-off-by: Kyle Hounslow --- .../templates/otel-collector-configmap.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/charts/observability-stack/templates/otel-collector-configmap.yaml b/charts/observability-stack/templates/otel-collector-configmap.yaml index c22bf7dd..ba689488 100644 --- a/charts/observability-stack/templates/otel-collector-configmap.yaml +++ b/charts/observability-stack/templates/otel-collector-configmap.yaml @@ -172,7 +172,10 @@ data: - set(body, ToKeyValueString(body)) where IsMap(body) exporters: debug: - verbosity: detailed + # basic logs per-batch counts only. detailed serializes every span/log/ + # metric to stdout, which throttles the pipeline at high throughput + # (observed ~6x slower trace ingest). + verbosity: basic otlp/opensearch: endpoint: "{{ .Release.Name }}-data-prepper:21890" tls: From 15f48c4cb83a125da66f5f4f669547741987abd1 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Wed, 24 Jun 2026 22:40:11 -0700 Subject: [PATCH 3/5] chore(chart): bump version to 0.4.0 Signed-off-by: Kyle Hounslow --- charts/observability-stack/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/observability-stack/Chart.yaml b/charts/observability-stack/Chart.yaml index b4338e93..bfec2443 100644 --- a/charts/observability-stack/Chart.yaml +++ b/charts/observability-stack/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: observability-stack description: OpenTelemetry-native observability platform for microservices, web apps, and AI agents type: application -version: 0.3.0 +version: 0.4.0 appVersion: "3.7.0" home: https://github.com/opensearch-project/observability-stack From a5461ecac86e7a5362cf8df09a7c2bf4f5a3ece8 Mon Sep 17 00:00:00 2001 From: Kyle Hounslow <7102778+kylehounslow@users.noreply.github.com> Date: Thu, 25 Jun 2026 11:54:34 -0700 Subject: [PATCH 4/5] Update comments for StorageClass configuration Clarified comments regarding StorageClass creation for EKS and PVC binding. Signed-off-by: Kyle Hounslow <7102778+kylehounslow@users.noreply.github.com> --- terraform/aws/observability-stack.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/terraform/aws/observability-stack.tf b/terraform/aws/observability-stack.tf index 4a1c8d35..a7440f2e 100644 --- a/terraform/aws/observability-stack.tf +++ b/terraform/aws/observability-stack.tf @@ -109,8 +109,9 @@ resource "kubernetes_namespace" "observability" { depends_on = [module.eks] } -# EKS ships only a gp2 StorageClass. opensearch_storage_class / cortex_storage_class -# default to gp3 (better IOPS/$ for bulk ingest), so create it via the EBS CSI driver. +# EKS pre-creates only gp2 by default, but opensearch_storage_class / cortex_storage_class +# default to gp3 (better IOPS/$ for bulk ingest). Without a matching StorageClass the PVCs +# stay Pending and the release times out, so create it via the EBS CSI driver. # WaitForFirstConsumer matches gp2 so a PVC binds on the node its pod lands on. resource "kubernetes_storage_class" "gp3" { metadata { From 475c89472eef1a1f65ddebfe865f9f584ed253ed Mon Sep 17 00:00:00 2001 From: Kyle Hounslow Date: Fri, 26 Jun 2026 11:13:42 -0700 Subject: [PATCH 5/5] fix(terraform): guard gp3 StorageClass and derive BYO pipeline creds from chart - create_gp3_storage_class (default true) lets operators opt out on clusters that already provide gp3 (EKS Auto Mode, manual, shared), avoiding the 'already exists' apply failure. - BYO pipeline Secret reads opensearchUsername/opensearchPassword defaults from the chart's values.yaml instead of re-hardcoding the literals, so the Secret and the cluster can't drift apart if the chart default changes. - Add var.opensearch_username, wired into the release like opensearch_password. Signed-off-by: Kyle Hounslow --- terraform/aws/observability-stack.tf | 36 +++++++++++++++++++++------- terraform/aws/variables.tf | 14 ++++++++++- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/terraform/aws/observability-stack.tf b/terraform/aws/observability-stack.tf index a7440f2e..c971d45f 100644 --- a/terraform/aws/observability-stack.tf +++ b/terraform/aws/observability-stack.tf @@ -110,10 +110,15 @@ resource "kubernetes_namespace" "observability" { } # EKS pre-creates only gp2 by default, but opensearch_storage_class / cortex_storage_class -# default to gp3 (better IOPS/$ for bulk ingest). Without a matching StorageClass the PVCs +# default to gp3 (better IOPS/$ for bulk ingest). Without a matching StorageClass the PVCs # stay Pending and the release times out, so create it via the EBS CSI driver. # WaitForFirstConsumer matches gp2 so a PVC binds on the node its pod lands on. +# Gated by create_gp3_storage_class: clusters that already ship gp3 (EKS Auto Mode, +# a manually created class, a shared cluster) must opt out or the apply fails with +# "storageclasses.storage.k8s.io \"gp3\" already exists". resource "kubernetes_storage_class" "gp3" { + count = var.create_gp3_storage_class ? 1 : 0 + metadata { name = "gp3" } @@ -127,15 +132,21 @@ resource "kubernetes_storage_class" "gp3" { depends_on = [module.eks] } -# Bring-your-own Data Prepper pipeline. When data_prepper_pipeline_secret_file -# is set, create the Secret the chart's dataPrepperManageSecret=false gate -# expects (same name the subchart mounts) and order it before the release so -# Data Prepper finds it at boot. The chart default leaves this empty and -# renders its own pipeline Secret. +# Credentials passed into the BYO pipeline Secret below. When the corresponding +# var is empty the chart default applies, so read both defaults from the chart's +# values.yaml (its single source of truth) rather than re-hardcoding the literals +# here. That way the rendered Secret and the cluster can never authenticate with +# different credentials if the chart default changes. locals { - opensearch_password_effective = var.opensearch_password != "" ? var.opensearch_password : "My_password_123!@#" + chart_values = yamldecode(file("${path.module}/../../charts/observability-stack/values.yaml")) + opensearch_username_effective = var.opensearch_username != "" ? var.opensearch_username : local.chart_values.opensearchUsername + opensearch_password_effective = var.opensearch_password != "" ? var.opensearch_password : local.chart_values.opensearchPassword } +# Bring-your-own Data Prepper pipeline. When data_prepper_pipeline_secret_file +# is set, create the Secret the chart's dataPrepperManageSecret=false gate expects +# (same name the subchart mounts) and order it before the release so Data Prepper +# finds it at boot. The chart default leaves this empty and renders its own Secret. resource "kubernetes_secret" "data_prepper_pipeline" { count = var.data_prepper_pipeline_secret_file != "" ? 1 : 0 @@ -146,7 +157,7 @@ resource "kubernetes_secret" "data_prepper_pipeline" { data = { "pipelines.yaml" = templatefile(var.data_prepper_pipeline_secret_file, { - opensearch_user = "admin" + opensearch_user = local.opensearch_username_effective opensearch_password = local.opensearch_password_effective trace_flush_interval = var.data_prepper_trace_flush_interval }) @@ -238,7 +249,14 @@ resource "helm_release" "observability_stack" { value = var.enable_otel_demo ? "true" : "false" } - # --- Custom password (conditional) --- + # --- Custom credentials (conditional) --- + dynamic "set" { + for_each = var.opensearch_username != "" ? [1] : [] + content { + name = "opensearchUsername" + value = var.opensearch_username + } + } dynamic "set_sensitive" { for_each = var.opensearch_password != "" ? [1] : [] content { diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf index 8cfa6275..7894092d 100644 --- a/terraform/aws/variables.tf +++ b/terraform/aws/variables.tf @@ -65,8 +65,14 @@ variable "anonymous_auth" { default = false } +variable "opensearch_username" { + description = "OpenSearch admin username. Leave empty to use the chart default (opensearchUsername in values.yaml)." + type = string + default = "" +} + variable "opensearch_password" { - description = "OpenSearch admin password. Leave empty to use chart default." + description = "OpenSearch admin password. Leave empty to use the chart default (opensearchPassword in values.yaml)." type = string default = "" sensitive = true @@ -99,6 +105,12 @@ variable "extra_helm_values" { default = [] } +variable "create_gp3_storage_class" { + description = "Create a gp3 StorageClass via the EBS CSI driver. Set false on clusters that already provide gp3 (EKS Auto Mode, a manually created class, a shared cluster) to avoid an \"already exists\" apply error." + type = bool + default = true +} + variable "data_prepper_pipeline_secret_file" { description = "Path to a pipelines.yaml template for a bring-your-own Data Prepper pipeline. When set, terraform creates the data-prepper-pipeline Secret from it and sets dataPrepperManageSecret=false so the chart renders no pipeline Secret. The template is rendered with templatefile and may reference opensearch_user, opensearch_password, and trace_flush_interval. Empty leaves the chart's managed pipeline in place." type = string