From f7273b71d34b1234d5a1f099304eec767063a31c Mon Sep 17 00:00:00 2001 From: Anshuman Biswas Date: Wed, 23 Sep 2026 20:55:17 -0400 Subject: [PATCH 1/8] Add the database monitoring agent Terraform module Moves the Fargate module for the Elastio database monitoring agent (elastio/blue-stack#17072) here from its old repository, as aws-elastio-database-monitoring-agent 0.1.0. The image defaults to public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.4 and every resource name uses the elastio-dbmon prefix. The variable names are unchanged because the Elastio Portal renders a module call from them. --- database-monitoring/terraform/.module.toml | 6 + database-monitoring/terraform/README.md | 179 ++++++++ .../examples/basic/.terraform.lock.hcl | 47 ++ .../terraform/examples/basic/main.tf | 11 + .../terraform/examples/basic/variables.tf | 31 ++ .../terraform/examples/basic/versions.tf | 3 + database-monitoring/terraform/main.tf | 407 ++++++++++++++++++ database-monitoring/terraform/outputs.tf | 34 ++ .../terraform/tests/module.tftest.hcl | 129 ++++++ database-monitoring/terraform/variables.tf | 76 ++++ database-monitoring/terraform/versions.tf | 14 + 11 files changed, 937 insertions(+) create mode 100644 database-monitoring/terraform/.module.toml create mode 100644 database-monitoring/terraform/README.md create mode 100644 database-monitoring/terraform/examples/basic/.terraform.lock.hcl create mode 100644 database-monitoring/terraform/examples/basic/main.tf create mode 100644 database-monitoring/terraform/examples/basic/variables.tf create mode 100644 database-monitoring/terraform/examples/basic/versions.tf create mode 100644 database-monitoring/terraform/main.tf create mode 100644 database-monitoring/terraform/outputs.tf create mode 100644 database-monitoring/terraform/tests/module.tftest.hcl create mode 100644 database-monitoring/terraform/variables.tf create mode 100644 database-monitoring/terraform/versions.tf diff --git a/database-monitoring/terraform/.module.toml b/database-monitoring/terraform/.module.toml new file mode 100644 index 0000000..2c37ec7 --- /dev/null +++ b/database-monitoring/terraform/.module.toml @@ -0,0 +1,6 @@ +[module] +name = "aws-elastio-database-monitoring-agent" +version = "0.1.0" + +description = "Terraform module that runs the Elastio database monitoring agent on AWS Fargate" +type = "terraform" diff --git a/database-monitoring/terraform/README.md b/database-monitoring/terraform/README.md new file mode 100644 index 0000000..7cb7397 --- /dev/null +++ b/database-monitoring/terraform/README.md @@ -0,0 +1,179 @@ +# `elastio-database-monitoring-agent` module + +This Terraform module runs the Elastio database monitoring agent on AWS Fargate. One agent watches one PostgreSQL database and reports to your Elastio tenant. + +## Installation + +Elastio terraform modules are published to the public Cloudsmith registry. Before you add this module, add this to your [`.terraformrc`](https://developer.hashicorp.com/terraform/cli/config/config-file). The file lives in your home directory (if you are on Linux): + +```hcl +credentials "terraform.cloudsmith.io" { + token = "elastio/public/" +} +``` + +Then add the module to your project: + +```tf +module "elastio_database_monitoring_agent" { + source = "terraform.cloudsmith.io/public/elastio-database-monitoring-agent/aws" + version = "0.1.0" + + name = "orders-db-prod" + server_url = "https://.app.elastio.com" + api_key = var.elastio_api_key # shown by the Elastio Portal when you add the database + database_url = var.database_url # postgres://user:pass@host:5432/dbname + + subnet_ids = ["subnet-0123456789abcdef0", "subnet-0fedcba9876543210"] + security_group_ids = ["sg-0123456789abcdef0"] +} +``` + +You can find the full version of this example in [`examples/basic`](./examples/basic). + +The `image` input defaults to the agent release this module version was tested with. Leave it unset unless Elastio support asks you to pin a different one. + +Pass `api_key` and `database_url` as sensitive variables or from your own secret store. They are stored in two Secrets Manager secrets that the module creates, and ECS reads them when the task starts. They never appear in the task definition. The module also generates a third secret, the agent's hashing key. + +## Prerequisites + +**Run the setup SQL first.** The Elastio Portal shows the exact statements for your database when you add it: a role with `REPLICATION`, a publication, and a replication slot that uses the `pgoutput` plugin. The agent never creates or drops a slot or a publication. It attaches to ones that already exist. If the slot is missing when the task starts, the agent exits and ECS keeps restarting it until the slot exists. + +**The subnets must reach both the database and the internet.** The agent connects to the database on its port, and to your Elastio tenant over HTTPS. Private subnets need a NAT gateway for the second connection. If they have none, set `assign_public_ip = true` and use subnets that have a route to an internet gateway. The subnets must exist before you run `terraform plan`, because the module reads each subnet's availability zone to decide where to put the EFS mount targets. + +**The security groups need these outbound rules:** + +- the database port, to the database; +- TCP 443, to your Elastio tenant; +- TCP 2049, to the VPC, for NFS to the ledger's EFS file system (only needed with the default `persistent_ledger = true`). + +A security group with the usual allow-all egress already allows all three. The database's own security group must allow inbound traffic from these security groups on the database port. + +## What it creates + +- An ECS cluster, service and task family named after `name` (with any characters ECS doesn't allow replaced), with Container Insights turned off. +- A CloudWatch log group named `/elastio-dbmon/agent/`. +- Three Secrets Manager secrets, `elastio-dbmon//api-key-*`, `.../database-url-*` and `.../hash-secret-*`. The hashing key is 32 random bytes, hex-encoded. The agent minimises evidence under this key, so the key must stay the same for the life of the ledger. Never taint or replace it. +- IAM roles `elastio-dbmon--exec-*` and `elastio-dbmon--task-*`. The execution role has `AmazonECSTaskExecutionRolePolicy` and permission to read exactly those three secrets. The agent makes no AWS calls, so the task role carries only the EFS mount permission below. +- With `persistent_ledger = true` (the default), storage for the ledger: + - an encrypted EFS file system tagged `elastio-dbmon--ledger`, using General Purpose performance mode and bursting throughput; + - one mount target per distinct availability zone among `subnet_ids`; + - a security group `elastio-dbmon--efs-*` on the mount targets, which allows TCP 2049 only from `security_group_ids`; + - an access point that maps every client to uid/gid 65532 and roots it at `/elastio-dbmon`, created with mode 0700; + - a task-role policy that allows `elasticfilesystem:ClientMount` and `ClientWrite` on the file system, only through that access point. +- A Fargate task definition: 0.25 vCPU, 0.5 GB, ARM64, with a read-only root filesystem. The task runs as the image's non-root user. +- A service with `desired_count = 1`. Its deployment policy (minimum healthy 0%, maximum 100%) stops the old task before it starts the new one. A replication slot allows only one consumer at a time, and the ledger allows only one writer. Don't raise the maximum. + +## Cost + +In us-east-1 at on-demand rates, the smallest ARM Fargate size costs: + +| | rate | per month (730 h) | +| --------- | ---------------------- | ----------------- | +| 0.25 vCPU | $0.03238 per vCPU-hour | $5.91 | +| 0.5 GB | $0.00356 per GB-hour | $1.30 | +| | | **$7.21** | + +On top of that: + +- **Secrets Manager:** three secrets at $0.40 each, so $1.20 a month. +- **EFS:** the ledger holds only minimised evidence (keys and hashes, never row contents) and is pruned as it goes. It stays at tens of MB, which is cents a month. Mount targets and access points are free. +- **Ephemeral storage and CloudWatch:** the extra GB of ephemeral storage above the free 20 GB, and CloudWatch log ingestion, each cost a few cents. + +If the subnets use a NAT gateway, its hourly charge will be your largest cost by far. You pay that charge whether or not you deploy this module. + +## Operating it + +The agent writes its log to the log group above. `enable_execute_command` is off. + +The ledger is on EFS, so it survives task replacement. That matters for findings that are still standing when the task is replaced, because the agent re-reads a finding's original window from the ledger to decide whether later activity explains it. With `persistent_ledger = false`, the ledger is on the task's ephemeral storage and every replacement starts it empty. Nothing is lost on the Elastio side, but findings that were standing at the time can no longer be explained. + +To upgrade the agent, bump the module `version` and apply. To remove the agent, run `terraform destroy`, then **drop the replication slot** in the database. An abandoned slot keeps write-ahead log until the disk fills. + +## Testing the module + +`tests/module.tftest.hcl` plans the module against mocked providers, so it needs no AWS account: + +```sh +terraform init -backend=false && terraform test +``` + + + +## Requirements + +| Name | Version | +| ------------------------------------------------------------------------ | ------- | +| [terraform](#requirement_terraform) | ~> 1.9 | +| [aws](#requirement_aws) | >= 5.0 | +| [random](#requirement_random) | >= 3.0 | + +## Providers + +| Name | Version | +| --------------------------------------------------------- | ------- | +| [aws](#provider_aws) | >= 5.0 | +| [random](#provider_random) | >= 3.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +| ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | +| [aws_cloudwatch_log_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_ecs_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_cluster) | resource | +| [aws_ecs_service.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource | +| [aws_ecs_task_definition.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource | +| [aws_efs_access_point.ledger](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_access_point) | resource | +| [aws_efs_file_system.ledger](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_file_system) | resource | +| [aws_efs_mount_target.ledger](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_mount_target) | resource | +| [aws_iam_role.execution](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.mount_ledger](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.read_secrets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy_attachment.execution](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_secretsmanager_secret.api_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret) | resource | +| [aws_secretsmanager_secret.database_url](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret) | resource | +| [aws_secretsmanager_secret.hash_secret](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret) | resource | +| [aws_secretsmanager_secret_version.api_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version) | resource | +| [aws_secretsmanager_secret_version.database_url](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version) | resource | +| [aws_secretsmanager_secret_version.hash_secret](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/secretsmanager_secret_version) | resource | +| [aws_security_group.efs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_vpc_security_group_ingress_rule.efs_from_agent](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_security_group_ingress_rule) | resource | +| [random_id.hash_secret](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource | +| [aws_subnet.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +| --------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------------------ | :------: | +| [api_key](#input_api_key) | Bearer token the agent presents to the server. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | +| [assign_public_ip](#input_assign_public_ip) | Give the task a public IP. Needed only when the subnets have no NAT gateway and the server URL is on the internet. | `bool` | `false` | no | +| [database_url](#input_database_url) | postgres:// URL of the database to watch, for a role with REPLICATION. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | +| [image](#input_image) | Agent image. The default is the agent release this module version was tested with. Pin a version; :latest moves. | `string` | `"public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.4"` | no | +| [log_retention_days](#input_log_retention_days) | How long CloudWatch keeps the agent's log. | `number` | `7` | no | +| [name](#input_name) | What the product shows for this agent. Also the ECS cluster, service and task family name, sanitised to the characters ECS allows. | `string` | n/a | yes | +| [persistent_ledger](#input_persistent_ledger) | Keep the agent's ledger on an encrypted EFS file system so it survives task replacement. When false, the ledger is on the task's ephemeral storage and every replacement starts it empty, after which findings standing at the time can never be explained by later activity. | `bool` | `true` | no | +| [publication](#input_publication) | Name of the publication whose tables are streamed. | `string` | `"elastio_monitor"` | no | +| [security_group_ids](#input_security_group_ids) | Security groups attached to the task. The database's own security group must admit them on its port. | `list(string)` | n/a | yes | +| [server_url](#input_server_url) | Base URL of the server the agent reports to. | `string` | n/a | yes | +| [slot](#input_slot) | Name of the existing pgoutput replication slot. The agent never creates or drops one. | `string` | `"elastio_monitor"` | no | +| [subnet_ids](#input_subnet_ids) | Subnets the task runs in. They must reach the database and the server URL. | `list(string)` | n/a | yes | +| [tags](#input_tags) | Tags applied to every resource the module creates. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +| -------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [cluster_arn](#output_cluster_arn) | ARN of the ECS cluster the agent runs in. | +| [execution_role_arn](#output_execution_role_arn) | IAM role ECS uses to pull the image, read the three secrets and write logs. | +| [hash_secret_arn](#output_hash_secret_arn) | ARN of the Secrets Manager secret holding the agent's hashing key. It must stay stable for the life of the ledger. | +| [ledger_file_system_id](#output_ledger_file_system_id) | ID of the EFS file system holding the agent's ledger, or null when persistent_ledger is false. | +| [log_group_name](#output_log_group_name) | CloudWatch log group the agent writes to. | +| [service_name](#output_service_name) | Name of the ECS service. | +| [task_role_arn](#output_task_role_arn) | IAM role the running container assumes. The agent has no AWS code; with a persistent ledger the role may mount the ledger's file system, and nothing else. | + + diff --git a/database-monitoring/terraform/examples/basic/.terraform.lock.hcl b/database-monitoring/terraform/examples/basic/.terraform.lock.hcl new file mode 100644 index 0000000..c602ab1 --- /dev/null +++ b/database-monitoring/terraform/examples/basic/.terraform.lock.hcl @@ -0,0 +1,47 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.66.0" + constraints = ">= 5.0.0" + hashes = [ + "h1:OnLj4nhqJnEcUzyyRKUjp1FgWG00Y8maikJEYSf9Zjw=", + "zh:156fe7164a3d26ef6b35734c43e99fb198df90575ed897d1182b8e930b8cd523", + "zh:1af52b22b35be00f8d16e3ebebff9fa699ec4db2ef69e6032ba5c536f80c03d9", + "zh:2545a8478bd551fdc9694f6cc1a1ad24617f6736f8bde0ad6cae90987c65380f", + "zh:4070db1ee369ccb41cb610bfd887386bc0a9b9ecad60aeb4dbce58443d2519dd", + "zh:53da7d3c1840ef875c7d34e967732502a64fe677af0e78824773d4c15a8fe740", + "zh:576a93a28bf611a4de2a2e6ced697a41d5126b8fd31d30782b16797e410a9706", + "zh:58fed5fa9a033355b9d4f3092c817b70d934100e0d8678d6e4c93f3c9493d4e4", + "zh:6a9ca2f24e2ee9156dd785d159a850b35d190e9cf7eca21cb9582970c2db80cd", + "zh:729edd30f99cc16009deba5c013265b0c81eda261a3d0821cbd011d3287fd230", + "zh:7ae460049b75bd4aefee465ef7c53a01ac2df46d4d3e3ac00824afa8b5cb83fb", + "zh:9051fa85c8034ade8a57a5c6f232fd33da28f3800bb5aa40bc8625dbc5e27632", + "zh:906547e4319805e7acf7fbdf2bac28a4b1a7370790a2a430c7adb1b29bb934eb", + "zh:998f27410a66158a35ee5ed142c27e5b21fe8601941da55da2157f8042d6dcca", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9c1804eff1dda0446dc2d215231015bb65a2fc6c3b7ba24584fe45f1ddd3fa9f", + "zh:b03ff5efdee310502aaaeb460144dc059bce72a0d8217e6b989099ef8aef9283", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.9.1" + constraints = ">= 3.0.0" + hashes = [ + "h1:PlW+UZ4EElQF3NQwf41KQwavFujab3Czc51zu9dyVM8=", + "zh:05f4734c1f0be840b711b3eff259ebc5fca436784c728955b1678078466f48d7", + "zh:0b91bf19371d012434eba1deeb6aab77158def9b39601dcbd94450b3974a2a26", + "zh:0ee6eacd47ec00183d55d726a4b6c4ce951a199f944bf22f1aa58392ebdfa7a2", + "zh:19388a4074b76a89a43a6c8328d7ae8ee2e7de3d346af51e80d3e6d3d12925f1", + "zh:23e74d48c5e2ac2e823fd527f49fee9db37d32a1990c9e3bf126ead697b843eb", + "zh:3cabf7fbd096c520064aae3aba61aba670af83ab91291a71fa1b1332929c2b7f", + "zh:5c0a3b8af0be60be4eca12ddee385cfa8babc1ec8e98cdf9de2f2274c73eabfa", + "zh:60b4f8a8ef18f52bf8e19215229dae408bee732825964092db7c989fd2de4097", + "zh:7359015acfedcbd6366f2329c854cf8d3c8ca5cd0faa89d2d37db358d6eba6c5", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7b38758402f0e13a1071162da28994023cd2ac676e54af350c9ffd8dfa73fa7b", + "zh:7c7fbb8895eb75bb4de1f933e98553bd99c8d048c89a925ddba490aa5a67f7dc", + "zh:8c2b8c6a7ccdec16b73e2fb9f3700ea097f58c592571e4c5de60c93d2301732c", + ] +} diff --git a/database-monitoring/terraform/examples/basic/main.tf b/database-monitoring/terraform/examples/basic/main.tf new file mode 100644 index 0000000..f8df436 --- /dev/null +++ b/database-monitoring/terraform/examples/basic/main.tf @@ -0,0 +1,11 @@ +module "elastio_database_monitoring_agent" { + source = "../../" + + name = "orders-db-prod" + server_url = var.elastio_server_url + api_key = var.elastio_api_key + database_url = var.database_url + + subnet_ids = var.subnet_ids + security_group_ids = var.security_group_ids +} diff --git a/database-monitoring/terraform/examples/basic/variables.tf b/database-monitoring/terraform/examples/basic/variables.tf new file mode 100644 index 0000000..e305201 --- /dev/null +++ b/database-monitoring/terraform/examples/basic/variables.tf @@ -0,0 +1,31 @@ +variable "elastio_server_url" { + description = "URL of your Elastio tenant. For example `https://mycompany.app.elastio.com`" + type = string + nullable = false +} + +variable "elastio_api_key" { + description = "Agent API key shown by the Elastio Portal when you add the database" + sensitive = true + type = string + nullable = false +} + +variable "database_url" { + description = "postgres:// URL of the database to watch, for the role created by the setup SQL" + sensitive = true + type = string + nullable = false +} + +variable "subnet_ids" { + description = "Subnets the agent runs in. They must reach the database and the Elastio tenant" + type = list(string) + nullable = false +} + +variable "security_group_ids" { + description = "Security groups attached to the agent. They need outbound 443 and 2049" + type = list(string) + nullable = false +} diff --git a/database-monitoring/terraform/examples/basic/versions.tf b/database-monitoring/terraform/examples/basic/versions.tf new file mode 100644 index 0000000..705d038 --- /dev/null +++ b/database-monitoring/terraform/examples/basic/versions.tf @@ -0,0 +1,3 @@ +terraform { + required_version = "~> 1.9" +} diff --git a/database-monitoring/terraform/main.tf b/database-monitoring/terraform/main.tf new file mode 100644 index 0000000..09ba3a3 --- /dev/null +++ b/database-monitoring/terraform/main.tf @@ -0,0 +1,407 @@ +# One Elastio database monitoring agent on AWS Fargate, watching one database. +# +# The agent is a single static binary that needs a network path to the +# database and to the server URL, three secrets, and a small writable directory +# for its ledger that outlives the task. Everything below is the least that +# gives it those. + +locals { + # ECS names allow letters, digits, hyphens and underscores, up to 255. The + # product-facing name is passed to the agent untouched as QUELL_AGENT_NAME. + ecs_name = substr(replace(var.name, "/[^A-Za-z0-9_-]/", "-"), 0, 255) + + # A shortened form for name prefixes with tight length limits. An IAM role + # name_prefix is at most 38 characters: "elastio-dbmon-" (14) + 16 + + # "-exec-" (6) = 36. + short_name = substr(local.ecs_name, 0, 16) + + # All resource names start with this. + prefix = "elastio-dbmon" + + # distroless nonroot, and what the image's USER resolves to. + uid = 65532 + gid = 65532 + + # The ledger directory inside the container. It is the agent image's own + # writable volume (owned by the nonroot user), so it must not change. + ledger_dir = "/var/lib/quell" +} + +resource "aws_ecs_cluster" "this" { + name = local.ecs_name + + setting { + name = "containerInsights" + value = "disabled" + } + + tags = var.tags +} + +resource "aws_cloudwatch_log_group" "this" { + name = "/${local.prefix}/agent/${local.ecs_name}" + retention_in_days = var.log_retention_days + tags = var.tags +} + +locals { + # The region the module deploys into, read from an ARN rather than from + # `data.aws_region`, whose name/id attributes are deprecated in AWS provider + # 6 but are the only ones in 5. + region = split(":", aws_cloudwatch_log_group.this.arn)[3] +} + +# The three secrets. They reach the container through the task definition's +# `secrets` block, which the execution role resolves at launch; the values +# never appear in the task definition or in `aws ecs describe-tasks`. + +resource "aws_secretsmanager_secret" "api_key" { + name_prefix = "${local.prefix}/${local.ecs_name}/api-key-" + description = "API key the Elastio database monitoring agent presents to the Elastio server" + tags = var.tags +} + +resource "aws_secretsmanager_secret_version" "api_key" { + secret_id = aws_secretsmanager_secret.api_key.id + secret_string = var.api_key +} + +resource "aws_secretsmanager_secret" "database_url" { + name_prefix = "${local.prefix}/${local.ecs_name}/database-url-" + description = "Connection URL of the database the Elastio database monitoring agent watches" + tags = var.tags +} + +resource "aws_secretsmanager_secret_version" "database_url" { + secret_id = aws_secretsmanager_secret.database_url.id + secret_string = var.database_url +} + +# The agent's hashing key, used to minimise evidence before it leaves the +# database's network. Without it the agent generates one and keeps it beside +# the ledger, so a lost ledger took the key with it. Here the key is generated +# once, by Terraform, and outlives every task. It must stay stable for the +# life of the ledger: evidence minimised under a different key cannot be +# compared with what the ledger holds, and redelivered transactions would be +# refused as conflicting. So never taint or replace it while a ledger exists. +# 32 random bytes, hex-encoded: 64 characters, the same shape the agent +# generates for itself. + +resource "random_id" "hash_secret" { + byte_length = 32 +} + +resource "aws_secretsmanager_secret" "hash_secret" { + name_prefix = "${local.prefix}/${local.ecs_name}/hash-secret-" + description = "Hashing key of the Elastio database monitoring agent. Must stay stable for the life of the agent's ledger" + tags = var.tags +} + +resource "aws_secretsmanager_secret_version" "hash_secret" { + secret_id = aws_secretsmanager_secret.hash_secret.id + secret_string = random_id.hash_secret.hex +} + +# Roles. The execution role is ECS's own: pull the image, read the secrets, +# write logs. The task role is what the process would assume if it called +# AWS, and it never does. With a persistent ledger it is also the identity the +# EFS mount helper presents, and carries exactly the permission to mount and +# write the file system through the module's access point. + +locals { + ecs_tasks_assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = "sts:AssumeRole" + Principal = { Service = "ecs-tasks.amazonaws.com" } + }, + ] + }) +} + +resource "aws_iam_role" "execution" { + name_prefix = "${local.prefix}-${local.short_name}-exec-" + description = "ECS task execution role of the Elastio database monitoring agent" + assume_role_policy = local.ecs_tasks_assume_role_policy + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "execution" { + role = aws_iam_role.execution.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" +} + +resource "aws_iam_role_policy" "read_secrets" { + name = "read-${local.prefix}-secrets" + role = aws_iam_role.execution.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = "secretsmanager:GetSecretValue" + Resource = [ + aws_secretsmanager_secret.api_key.arn, + aws_secretsmanager_secret.database_url.arn, + aws_secretsmanager_secret.hash_secret.arn, + ] + }, + ] + }) +} + +resource "aws_iam_role" "task" { + name_prefix = "${local.prefix}-${local.short_name}-task-" + description = "ECS task role of the Elastio database monitoring agent" + assume_role_policy = local.ecs_tasks_assume_role_policy + tags = var.tags +} + +# The ledger's storage. The agent's review re-reads a standing finding's +# original window from the ledger to decide whether later activity explains +# it; a ledger that starts empty after a task replacement has lost that +# window, and the finding can then never be explained. Task replacements are +# routine (a deploy, a forced new deployment, a crash, Fargate's own platform +# maintenance), so by default the ledger lives on EFS and survives them. +# +# One mount target per distinct availability zone among the subnets: EFS +# admits one per AZ, and two subnets in the same AZ share it. + +data "aws_subnet" "this" { + for_each = var.persistent_ledger ? toset(var.subnet_ids) : toset([]) + id = each.value +} + +locals { + # AZ => the subnets in it, and one subnet per AZ for the mount target. + subnets_by_az = { for id, s in data.aws_subnet.this : s.availability_zone => id... } + mount_target_subnets = { for az, ids in local.subnets_by_az : az => sort(ids)[0] } + vpc_id = var.persistent_ledger ? data.aws_subnet.this[var.subnet_ids[0]].vpc_id : null +} + +resource "aws_efs_file_system" "ledger" { + count = var.persistent_ledger ? 1 : 0 + + # No creation_token: the provider generates a unique one. A token derived + # from `name` would exceed EFS's 64-character limit for long names. + encrypted = true + performance_mode = "generalPurpose" + throughput_mode = "bursting" + + tags = merge(var.tags, { Name = "${local.prefix}-${local.ecs_name}-ledger" }) +} + +resource "aws_security_group" "efs" { + count = var.persistent_ledger ? 1 : 0 + + name_prefix = "${local.prefix}-${local.short_name}-efs-" + description = "NFS to the Elastio database monitoring agent ledger, from the agent security groups only" + vpc_id = local.vpc_id + + tags = var.tags + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_vpc_security_group_ingress_rule" "efs_from_agent" { + # count, not for_each, so security groups created in the same apply work. + count = var.persistent_ledger ? length(var.security_group_ids) : 0 + + security_group_id = aws_security_group.efs[0].id + referenced_security_group_id = var.security_group_ids[count.index] + ip_protocol = "tcp" + from_port = 2049 + to_port = 2049 + description = "NFS from the Elastio database monitoring agent task" + tags = var.tags +} + +resource "aws_efs_mount_target" "ledger" { + for_each = var.persistent_ledger ? local.mount_target_subnets : {} + + file_system_id = aws_efs_file_system.ledger[0].id + subnet_id = each.value + security_groups = [aws_security_group.efs[0].id] +} + +# The access point pins every client to uid/gid 65532 and roots it at +# /elastio-dbmon, created 0700 for that uid, so the agent sees a private +# directory it owns and nothing else on the file system. + +resource "aws_efs_access_point" "ledger" { + count = var.persistent_ledger ? 1 : 0 + + file_system_id = aws_efs_file_system.ledger[0].id + + posix_user { + uid = local.uid + gid = local.gid + } + + root_directory { + path = "/${local.prefix}" + + creation_info { + owner_uid = local.uid + owner_gid = local.gid + permissions = "0700" + } + } + + tags = var.tags +} + +resource "aws_iam_role_policy" "mount_ledger" { + count = var.persistent_ledger ? 1 : 0 + + name = "mount-${local.prefix}-ledger" + role = aws_iam_role.task.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "elasticfilesystem:ClientMount", + "elasticfilesystem:ClientWrite", + ] + Resource = aws_efs_file_system.ledger[0].arn + Condition = { + StringEquals = { + "elasticfilesystem:AccessPointArn" = aws_efs_access_point.ledger[0].arn + } + } + }, + ] + }) +} + +# The task. The smallest Fargate size, on ARM because it is the cheaper of +# the two and the image is published for both. The root filesystem is +# read-only; the ledger directory is the EFS access point above, or, with +# persistent_ledger = false, a bind mount onto the task's ephemeral storage, +# which Fargate creates writable for the container's uid. + +resource "aws_ecs_task_definition" "this" { + family = local.ecs_name + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = 256 + memory = 512 + execution_role_arn = aws_iam_role.execution.arn + task_role_arn = aws_iam_role.task.arn + + runtime_platform { + cpu_architecture = "ARM64" + operating_system_family = "LINUX" + } + + ephemeral_storage { + size_in_gib = 21 # the Fargate minimum + } + + volume { + name = "ledger" + + dynamic "efs_volume_configuration" { + for_each = var.persistent_ledger ? [1] : [] + + content { + file_system_id = aws_efs_file_system.ledger[0].id + transit_encryption = "ENABLED" + + authorization_config { + access_point_id = aws_efs_access_point.ledger[0].id + iam = "ENABLED" + } + } + } + } + + container_definitions = jsonencode([ + { + name = "${local.prefix}-agent" + image = var.image + essential = true + user = "${local.uid}:${local.gid}" + + readonlyRootFilesystem = true + + mountPoints = [ + { + sourceVolume = "ledger" + containerPath = local.ledger_dir + readOnly = false + }, + ] + + # The QUELL_* names are the agent binary's configuration contract. + environment = [ + { name = "QUELL_SERVER_URL", value = var.server_url }, + { name = "QUELL_SLOT", value = var.slot }, + { name = "QUELL_PUBLICATION", value = var.publication }, + { name = "QUELL_AGENT_NAME", value = var.name }, + { name = "QUELL_LEDGER_PATH", value = "${local.ledger_dir}/ledger" }, + ] + + secrets = [ + { name = "QUELL_API_KEY", valueFrom = aws_secretsmanager_secret.api_key.arn }, + { name = "QUELL_DATABASE_URL", valueFrom = aws_secretsmanager_secret.database_url.arn }, + { name = "QUELL_HASH_SECRET", valueFrom = aws_secretsmanager_secret.hash_secret.arn }, + ] + + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.this.name + "awslogs-region" = local.region + "awslogs-stream-prefix" = "agent" + } + } + }, + ]) + + tags = var.tags +} + +# The service. Exactly one task, and never two: a replication slot admits one +# consumer, so a rolling deployment that starts the new task before stopping +# the old one would leave the new task failing to attach until the old one +# exits. Minimum healthy 0 / maximum 100 makes ECS stop the old task first. +# The gap is bounded by the slot: write-ahead log accumulates on the database +# while nobody reads it and is streamed when the new task attaches. +# +# With a persistent ledger this is also what keeps the ledger sound. The +# design is one process per ledger with no lease; two tasks writing the same +# EFS ledger during a rolling deployment would corrupt it. Never raise +# deployment_maximum_percent above 100. + +resource "aws_ecs_service" "this" { + name = local.ecs_name + cluster = aws_ecs_cluster.this.id + task_definition = aws_ecs_task_definition.this.arn + launch_type = "FARGATE" + desired_count = 1 + + deployment_minimum_healthy_percent = 0 + deployment_maximum_percent = 100 + + enable_execute_command = false + + network_configuration { + subnets = var.subnet_ids + security_groups = var.security_group_ids + assign_public_ip = var.assign_public_ip + } + + tags = var.tags + + # A task cannot mount the ledger before a mount target exists in its AZ. + depends_on = [aws_efs_mount_target.ledger] +} diff --git a/database-monitoring/terraform/outputs.tf b/database-monitoring/terraform/outputs.tf new file mode 100644 index 0000000..2f7c889 --- /dev/null +++ b/database-monitoring/terraform/outputs.tf @@ -0,0 +1,34 @@ +output "cluster_arn" { + description = "ARN of the ECS cluster the agent runs in." + value = aws_ecs_cluster.this.arn +} + +output "service_name" { + description = "Name of the ECS service." + value = aws_ecs_service.this.name +} + +output "log_group_name" { + description = "CloudWatch log group the agent writes to." + value = aws_cloudwatch_log_group.this.name +} + +output "execution_role_arn" { + description = "IAM role ECS uses to pull the image, read the three secrets and write logs." + value = aws_iam_role.execution.arn +} + +output "task_role_arn" { + description = "IAM role the running container assumes. The agent has no AWS code; with a persistent ledger the role may mount the ledger's file system, and nothing else." + value = aws_iam_role.task.arn +} + +output "ledger_file_system_id" { + description = "ID of the EFS file system holding the agent's ledger, or null when persistent_ledger is false." + value = var.persistent_ledger ? aws_efs_file_system.ledger[0].id : null +} + +output "hash_secret_arn" { + description = "ARN of the Secrets Manager secret holding the agent's hashing key. It must stay stable for the life of the ledger." + value = aws_secretsmanager_secret.hash_secret.arn +} diff --git a/database-monitoring/terraform/tests/module.tftest.hcl b/database-monitoring/terraform/tests/module.tftest.hcl new file mode 100644 index 0000000..b95baed --- /dev/null +++ b/database-monitoring/terraform/tests/module.tftest.hcl @@ -0,0 +1,129 @@ +# Offline checks of what the module plans, with mocked providers: no AWS +# account or credentials needed. Run from this directory with +# terraform init -backend=false && terraform test + +mock_provider "aws" { + mock_data "aws_subnet" { + defaults = { availability_zone = "us-east-1a", vpc_id = "vpc-1" } + } + mock_resource "aws_cloudwatch_log_group" { + defaults = { arn = "arn:aws:logs:us-east-1:123456789012:log-group:x" } + } + mock_resource "aws_efs_file_system" { + defaults = { arn = "arn:aws:elasticfilesystem:us-east-1:123456789012:file-system/fs-1" } + } + mock_resource "aws_efs_access_point" { + defaults = { arn = "arn:aws:elasticfilesystem:us-east-1:123456789012:access-point/fsap-1" } + } + mock_resource "aws_secretsmanager_secret" { + defaults = { arn = "arn:aws:secretsmanager:us-east-1:123456789012:secret:x" } + } + mock_resource "aws_iam_role" { + defaults = { arn = "arn:aws:iam::123456789012:role/x" } + } +} +mock_provider "random" {} + +variables { + name = "orders db" + server_url = "https://x" + api_key = "k" + database_url = "postgres://x" + subnet_ids = ["subnet-a", "subnet-b"] + security_group_ids = ["sg-1", "sg-2"] +} + +# Two subnets in one AZ share a mount target; a third in another AZ gets its own. +run "one_mount_target_per_az" { + command = apply + + variables { + subnet_ids = ["subnet-a", "subnet-b", "subnet-c"] + } + + override_data { + target = data.aws_subnet.this["subnet-c"] + values = { availability_zone = "us-east-1b", vpc_id = "vpc-1" } + } + + assert { + condition = length(aws_efs_mount_target.ledger) == 2 + error_message = "expected one mount target per distinct AZ" + } +} + +run "same_az_two_subnets" { + command = apply + assert { + condition = length(aws_efs_mount_target.ledger) == 1 + error_message = "two subnets in one AZ must share a mount target" + } + assert { + condition = length(aws_vpc_security_group_ingress_rule.efs_from_agent) == 2 + error_message = "one ingress rule per task SG" + } + assert { + condition = one(aws_ecs_task_definition.this.volume).efs_volume_configuration[0].transit_encryption == "ENABLED" + error_message = "the ledger volume must be EFS with transit encryption" + } + assert { + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "\"awslogs-region\":\"us-east-1\"") + error_message = "the log driver must be given the module's region" + } + assert { + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "QUELL_HASH_SECRET") + error_message = "QUELL_HASH_SECRET must reach the container" + } + assert { + condition = aws_ecs_service.this.deployment_maximum_percent == 100 && aws_ecs_service.this.deployment_minimum_healthy_percent == 0 + error_message = "two tasks must never run against one ledger" + } +} + +run "names_and_image" { + command = plan + + variables { + name = "a-very-long-agent-name-for-the-orders-database" + } + + assert { + condition = startswith(aws_iam_role.execution.name_prefix, "elastio-dbmon-") && length(aws_iam_role.execution.name_prefix) <= 38 + error_message = "the execution role prefix must be elastio-dbmon- and fit IAM's 38-character name_prefix limit" + } + assert { + condition = length(aws_iam_role.task.name_prefix) <= 38 + error_message = "the task role prefix must fit IAM's 38-character name_prefix limit" + } + assert { + condition = aws_cloudwatch_log_group.this.name == "/elastio-dbmon/agent/a-very-long-agent-name-for-the-orders-database" + error_message = "unexpected log group name" + } + assert { + condition = startswith(aws_secretsmanager_secret.api_key.name_prefix, "elastio-dbmon/") + error_message = "secrets must be named elastio-dbmon//..." + } + assert { + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "public.ecr.aws/elastio/elastio-database-monitoring-agent:") + error_message = "the default image must be Elastio's public ECR image" + } +} + +run "ephemeral" { + command = apply + variables { + persistent_ledger = false + } + assert { + condition = length(aws_efs_file_system.ledger) == 0 && length(aws_efs_mount_target.ledger) == 0 && length(aws_security_group.efs) == 0 + error_message = "persistent_ledger = false must create no EFS resources" + } + assert { + condition = length(one(aws_ecs_task_definition.this.volume).efs_volume_configuration) == 0 + error_message = "persistent_ledger = false must keep the ephemeral bind mount" + } + assert { + condition = output.ledger_file_system_id == null + error_message = "ledger_file_system_id must be null without EFS" + } +} diff --git a/database-monitoring/terraform/variables.tf b/database-monitoring/terraform/variables.tf new file mode 100644 index 0000000..4b2d043 --- /dev/null +++ b/database-monitoring/terraform/variables.tf @@ -0,0 +1,76 @@ +# The variable names here are a contract: the product renders a filled-in +# module call from them. Rename one and the rendered block stops applying. + +variable "name" { + description = "What the product shows for this agent. Also the ECS cluster, service and task family name, sanitised to the characters ECS allows." + type = string +} + +variable "server_url" { + description = "Base URL of the server the agent reports to." + type = string +} + +variable "api_key" { + description = "Bearer token the agent presents to the server. Stored in Secrets Manager, never in the task definition." + type = string + sensitive = true +} + +variable "database_url" { + description = "postgres:// URL of the database to watch, for a role with REPLICATION. Stored in Secrets Manager, never in the task definition." + type = string + sensitive = true +} + +variable "slot" { + description = "Name of the existing pgoutput replication slot. The agent never creates or drops one." + type = string + default = "elastio_monitor" +} + +variable "publication" { + description = "Name of the publication whose tables are streamed." + type = string + default = "elastio_monitor" +} + +variable "subnet_ids" { + description = "Subnets the task runs in. They must reach the database and the server URL." + type = list(string) +} + +variable "security_group_ids" { + description = "Security groups attached to the task. The database's own security group must admit them on its port." + type = list(string) +} + +variable "image" { + description = "Agent image. The default is the agent release this module version was tested with. Pin a version; :latest moves." + type = string + default = "public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.4" +} + +variable "assign_public_ip" { + description = "Give the task a public IP. Needed only when the subnets have no NAT gateway and the server URL is on the internet." + type = bool + default = false +} + +variable "log_retention_days" { + description = "How long CloudWatch keeps the agent's log." + type = number + default = 7 +} + +variable "persistent_ledger" { + description = "Keep the agent's ledger on an encrypted EFS file system so it survives task replacement. When false, the ledger is on the task's ephemeral storage and every replacement starts it empty, after which findings standing at the time can never be explained by later activity." + type = bool + default = true +} + +variable "tags" { + description = "Tags applied to every resource the module creates." + type = map(string) + default = {} +} diff --git a/database-monitoring/terraform/versions.tf b/database-monitoring/terraform/versions.tf new file mode 100644 index 0000000..af805df --- /dev/null +++ b/database-monitoring/terraform/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = "~> 1.9" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0" + } + random = { + source = "hashicorp/random" + version = ">= 3.0" + } + } +} From 705c669a55d8a00ec14c5e826ff9e9ccc31ed94c Mon Sep 17 00:00:00 2001 From: Maksim Svetlakov Date: Thu, 24 Sep 2026 10:47:24 -0400 Subject: [PATCH 2/8] Harden the database monitoring agent module - Add an EFS file system policy: only the task role may mount, only through the access point, and only over TLS. Without it, any NFS client that reaches a mount target can mount the file system as root. - Make the service depend on the IAM policies, secret versions, and the file system policy, so the first task does not start before it may read its secrets or mount the ledger, and destroy stops the task first. - Validate that subnet_ids and security_group_ids are not empty, and that log_retention_days is a value CloudWatch Logs accepts. --- database-monitoring/terraform/README.md | 6 +- database-monitoring/terraform/main.tf | 59 ++++++++++++++++++- .../terraform/tests/module.tftest.hcl | 17 +++++- database-monitoring/terraform/variables.tf | 15 +++++ 4 files changed, 92 insertions(+), 5 deletions(-) diff --git a/database-monitoring/terraform/README.md b/database-monitoring/terraform/README.md index 7cb7397..92005bc 100644 --- a/database-monitoring/terraform/README.md +++ b/database-monitoring/terraform/README.md @@ -61,6 +61,7 @@ A security group with the usual allow-all egress already allows all three. The d - a security group `elastio-dbmon--efs-*` on the mount targets, which allows TCP 2049 only from `security_group_ids`; - an access point that maps every client to uid/gid 65532 and roots it at `/elastio-dbmon`, created with mode 0700; - a task-role policy that allows `elasticfilesystem:ClientMount` and `ClientWrite` on the file system, only through that access point. + - a file system policy that admits only the task role, only through that access point, and denies every client that does not use TLS. - A Fargate task definition: 0.25 vCPU, 0.5 GB, ARM64, with a read-only root filesystem. The task runs as the image's non-root user. - A service with `desired_count = 1`. Its deployment policy (minimum healthy 0%, maximum 100%) stops the old task before it starts the new one. A replication slot allows only one consumer at a time, and the ledger allows only one writer. Don't raise the maximum. @@ -112,8 +113,8 @@ terraform init -backend=false && terraform test | Name | Version | | --------------------------------------------------------- | ------- | -| [aws](#provider_aws) | >= 5.0 | -| [random](#provider_random) | >= 3.0 | +| [aws](#provider_aws) | 6.66.0 | +| [random](#provider_random) | 3.9.1 | ## Modules @@ -129,6 +130,7 @@ No modules. | [aws_ecs_task_definition.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource | | [aws_efs_access_point.ledger](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_access_point) | resource | | [aws_efs_file_system.ledger](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_file_system) | resource | +| [aws_efs_file_system_policy.ledger](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_file_system_policy) | resource | | [aws_efs_mount_target.ledger](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_mount_target) | resource | | [aws_iam_role.execution](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role.task](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | diff --git a/database-monitoring/terraform/main.tf b/database-monitoring/terraform/main.tf index 09ba3a3..9090cea 100644 --- a/database-monitoring/terraform/main.tf +++ b/database-monitoring/terraform/main.tf @@ -282,6 +282,48 @@ resource "aws_iam_role_policy" "mount_ledger" { }) } +# Without a file system policy, EFS lets any NFS client that reaches a mount +# target mount it as root. The security groups in `security_group_ids` may be +# shared with other workloads, so the file system itself admits only the task +# role, only through the access point, and only over TLS. + +resource "aws_efs_file_system_policy" "ledger" { + count = var.persistent_ledger ? 1 : 0 + + file_system_id = aws_efs_file_system.ledger[0].id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AgentThroughAccessPoint" + Effect = "Allow" + Principal = { AWS = aws_iam_role.task.arn } + Action = [ + "elasticfilesystem:ClientMount", + "elasticfilesystem:ClientWrite", + ] + Resource = aws_efs_file_system.ledger[0].arn + Condition = { + StringEquals = { + "elasticfilesystem:AccessPointArn" = aws_efs_access_point.ledger[0].arn + } + } + }, + { + Sid = "DenyInsecureTransport" + Effect = "Deny" + Principal = { AWS = "*" } + Action = "*" + Resource = aws_efs_file_system.ledger[0].arn + Condition = { + Bool = { "aws:SecureTransport" = "false" } + } + }, + ] + }) +} + # The task. The smallest Fargate size, on ARM because it is the cheaper of # the two and the image is published for both. The root filesystem is # read-only; the ledger directory is the EFS access point above, or, with @@ -402,6 +444,19 @@ resource "aws_ecs_service" "this" { tags = var.tags - # A task cannot mount the ledger before a mount target exists in its AZ. - depends_on = [aws_efs_mount_target.ledger] + # The task definition references the roles and secrets, but not their + # policies or values. Without these, the first task can start before it may + # read the secrets or mount the ledger, and destroy can remove the + # permissions while the task still runs. A task also cannot mount the + # ledger before a mount target exists in its AZ. + depends_on = [ + aws_iam_role_policy_attachment.execution, + aws_iam_role_policy.read_secrets, + aws_iam_role_policy.mount_ledger, + aws_secretsmanager_secret_version.api_key, + aws_secretsmanager_secret_version.database_url, + aws_secretsmanager_secret_version.hash_secret, + aws_efs_mount_target.ledger, + aws_efs_file_system_policy.ledger, + ] } diff --git a/database-monitoring/terraform/tests/module.tftest.hcl b/database-monitoring/terraform/tests/module.tftest.hcl index b95baed..0f5de79 100644 --- a/database-monitoring/terraform/tests/module.tftest.hcl +++ b/database-monitoring/terraform/tests/module.tftest.hcl @@ -78,6 +78,21 @@ run "same_az_two_subnets" { condition = aws_ecs_service.this.deployment_maximum_percent == 100 && aws_ecs_service.this.deployment_minimum_healthy_percent == 0 error_message = "two tasks must never run against one ledger" } + assert { + condition = jsondecode(aws_efs_file_system_policy.ledger[0].policy).Statement[0].Condition.StringEquals["elasticfilesystem:AccessPointArn"] == aws_efs_access_point.ledger[0].arn + error_message = "the file system policy must admit the task only through the access point" + } +} + +run "rejects_empty_network" { + command = plan + + variables { + subnet_ids = [] + security_group_ids = [] + } + + expect_failures = [var.subnet_ids, var.security_group_ids] } run "names_and_image" { @@ -115,7 +130,7 @@ run "ephemeral" { persistent_ledger = false } assert { - condition = length(aws_efs_file_system.ledger) == 0 && length(aws_efs_mount_target.ledger) == 0 && length(aws_security_group.efs) == 0 + condition = length(aws_efs_file_system.ledger) == 0 && length(aws_efs_mount_target.ledger) == 0 && length(aws_security_group.efs) == 0 && length(aws_efs_file_system_policy.ledger) == 0 error_message = "persistent_ledger = false must create no EFS resources" } assert { diff --git a/database-monitoring/terraform/variables.tf b/database-monitoring/terraform/variables.tf index 4b2d043..af065e1 100644 --- a/database-monitoring/terraform/variables.tf +++ b/database-monitoring/terraform/variables.tf @@ -38,11 +38,21 @@ variable "publication" { variable "subnet_ids" { description = "Subnets the task runs in. They must reach the database and the server URL." type = list(string) + + validation { + condition = length(var.subnet_ids) > 0 + error_message = "subnet_ids must contain at least one subnet." + } } variable "security_group_ids" { description = "Security groups attached to the task. The database's own security group must admit them on its port." type = list(string) + + validation { + condition = length(var.security_group_ids) > 0 + error_message = "security_group_ids must contain at least one security group. The EFS mount targets admit NFS only from these groups." + } } variable "image" { @@ -61,6 +71,11 @@ variable "log_retention_days" { description = "How long CloudWatch keeps the agent's log." type = number default = 7 + + validation { + condition = contains([0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1096, 1827, 2192, 2557, 2922, 3288, 3653], var.log_retention_days) + error_message = "log_retention_days must be a value CloudWatch Logs accepts: 0 (never expire), 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1096, 1827, 2192, 2557, 2922, 3288 or 3653." + } } variable "persistent_ledger" { From edf7d09ea7daf267e141af73e88303fa2e6f2f1a Mon Sep 17 00:00:00 2001 From: Maksim Svetlakov Date: Thu, 24 Sep 2026 10:48:56 -0400 Subject: [PATCH 3/8] Regenerate the module README --- database-monitoring/terraform/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database-monitoring/terraform/README.md b/database-monitoring/terraform/README.md index 92005bc..a4c6056 100644 --- a/database-monitoring/terraform/README.md +++ b/database-monitoring/terraform/README.md @@ -113,8 +113,8 @@ terraform init -backend=false && terraform test | Name | Version | | --------------------------------------------------------- | ------- | -| [aws](#provider_aws) | 6.66.0 | -| [random](#provider_random) | 3.9.1 | +| [aws](#provider_aws) | >= 5.0 | +| [random](#provider_random) | >= 3.0 | ## Modules From 98276af94d299b268a1673049218b431b89148cb Mon Sep 17 00:00:00 2001 From: Anshuman Biswas Date: Thu, 24 Sep 2026 11:34:12 -0400 Subject: [PATCH 4/8] Configure the agent by its ELASTIO_DBMON_* names and run 0.1.5 Agent 0.1.5 reads ELASTIO_DBMON_* variables (QUELL_* still as a fallback) and keeps its ledger under /var/lib/elastio-dbmon. The module sets the new names, mounts the ledger there and names the path explicitly, so the EFS access point's ledger is the same file as before. It also sets GOMEMLIMIT to 80% of the task's memory: Go sets no memory limit from the container on its own. The default image is 0.1.5, which this needs. --- database-monitoring/terraform/README.md | 2 +- database-monitoring/terraform/main.tf | 49 +++++++++++++------ .../terraform/tests/module.tftest.hcl | 16 +++++- database-monitoring/terraform/variables.tf | 4 +- 4 files changed, 52 insertions(+), 19 deletions(-) diff --git a/database-monitoring/terraform/README.md b/database-monitoring/terraform/README.md index a4c6056..f5cf504 100644 --- a/database-monitoring/terraform/README.md +++ b/database-monitoring/terraform/README.md @@ -155,7 +155,7 @@ No modules. | [api_key](#input_api_key) | Bearer token the agent presents to the server. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | | [assign_public_ip](#input_assign_public_ip) | Give the task a public IP. Needed only when the subnets have no NAT gateway and the server URL is on the internet. | `bool` | `false` | no | | [database_url](#input_database_url) | postgres:// URL of the database to watch, for a role with REPLICATION. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | -| [image](#input_image) | Agent image. The default is the agent release this module version was tested with. Pin a version; :latest moves. | `string` | `"public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.4"` | no | +| [image](#input_image) | Agent image. The default is the agent release this module version was tested with. Pin a version; :latest moves. Needs 0.1.5 or later: older agents read only the QUELL\_\* variable names. | `string` | `"public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.5"` | no | | [log_retention_days](#input_log_retention_days) | How long CloudWatch keeps the agent's log. | `number` | `7` | no | | [name](#input_name) | What the product shows for this agent. Also the ECS cluster, service and task family name, sanitised to the characters ECS allows. | `string` | n/a | yes | | [persistent_ledger](#input_persistent_ledger) | Keep the agent's ledger on an encrypted EFS file system so it survives task replacement. When false, the ledger is on the task's ephemeral storage and every replacement starts it empty, after which findings standing at the time can never be explained by later activity. | `bool` | `true` | no | diff --git a/database-monitoring/terraform/main.tf b/database-monitoring/terraform/main.tf index 9090cea..b97c658 100644 --- a/database-monitoring/terraform/main.tf +++ b/database-monitoring/terraform/main.tf @@ -7,7 +7,8 @@ locals { # ECS names allow letters, digits, hyphens and underscores, up to 255. The - # product-facing name is passed to the agent untouched as QUELL_AGENT_NAME. + # product-facing name is passed to the agent untouched as + # ELASTIO_DBMON_AGENT_NAME. ecs_name = substr(replace(var.name, "/[^A-Za-z0-9_-]/", "-"), 0, 255) # A shortened form for name prefixes with tight length limits. An IAM role @@ -23,8 +24,24 @@ locals { gid = 65532 # The ledger directory inside the container. It is the agent image's own - # writable volume (owned by the nonroot user), so it must not change. - ledger_dir = "/var/lib/quell" + # writable volume (owned by the nonroot user) from 0.1.5, and the binary's + # default ledger path is beneath it. Only the mount point moves with it: + # the EFS access point's contents, the ledger and nothing else, are the + # same file under either path, because the path is also passed explicitly. + ledger_dir = "/var/lib/elastio-dbmon" + + # Task size. The Fargate floor: one agent reads one database's stream and + # needs no more (measured in elastio/database-monitoring-agent bench/). + task_cpu = 256 + task_memory = 512 + + # The Go runtime's soft memory limit, 80% of the task. Go does not derive + # one from the container on its own, and without it a heap whose live size + # is half the task is allowed to double before it is collected. The agent + # derives the same number from its cgroup when this is unset; it is set + # here too because what Fargate exposes inside the container is not + # something to rely on. + gomemlimit = "${floor(local.task_memory * 0.8)}MiB" } resource "aws_ecs_cluster" "this" { @@ -334,8 +351,8 @@ resource "aws_ecs_task_definition" "this" { family = local.ecs_name requires_compatibilities = ["FARGATE"] network_mode = "awsvpc" - cpu = 256 - memory = 512 + cpu = local.task_cpu + memory = local.task_memory execution_role_arn = aws_iam_role.execution.arn task_role_arn = aws_iam_role.task.arn @@ -383,19 +400,23 @@ resource "aws_ecs_task_definition" "this" { }, ] - # The QUELL_* names are the agent binary's configuration contract. + # The ELASTIO_DBMON_* names are the agent binary's configuration + # contract from 0.1.5. The agent still reads the QUELL_* names older + # deployments set, but an image older than 0.1.5 reads only those, so + # this module needs agent 0.1.5 or later. environment = [ - { name = "QUELL_SERVER_URL", value = var.server_url }, - { name = "QUELL_SLOT", value = var.slot }, - { name = "QUELL_PUBLICATION", value = var.publication }, - { name = "QUELL_AGENT_NAME", value = var.name }, - { name = "QUELL_LEDGER_PATH", value = "${local.ledger_dir}/ledger" }, + { name = "ELASTIO_DBMON_SERVER_URL", value = var.server_url }, + { name = "ELASTIO_DBMON_SLOT", value = var.slot }, + { name = "ELASTIO_DBMON_PUBLICATION", value = var.publication }, + { name = "ELASTIO_DBMON_AGENT_NAME", value = var.name }, + { name = "ELASTIO_DBMON_LEDGER_PATH", value = "${local.ledger_dir}/ledger" }, + { name = "GOMEMLIMIT", value = local.gomemlimit }, ] secrets = [ - { name = "QUELL_API_KEY", valueFrom = aws_secretsmanager_secret.api_key.arn }, - { name = "QUELL_DATABASE_URL", valueFrom = aws_secretsmanager_secret.database_url.arn }, - { name = "QUELL_HASH_SECRET", valueFrom = aws_secretsmanager_secret.hash_secret.arn }, + { name = "ELASTIO_DBMON_API_KEY", valueFrom = aws_secretsmanager_secret.api_key.arn }, + { name = "ELASTIO_DBMON_DATABASE_URL", valueFrom = aws_secretsmanager_secret.database_url.arn }, + { name = "ELASTIO_DBMON_HASH_SECRET", valueFrom = aws_secretsmanager_secret.hash_secret.arn }, ] logConfiguration = { diff --git a/database-monitoring/terraform/tests/module.tftest.hcl b/database-monitoring/terraform/tests/module.tftest.hcl index 0f5de79..87b62b4 100644 --- a/database-monitoring/terraform/tests/module.tftest.hcl +++ b/database-monitoring/terraform/tests/module.tftest.hcl @@ -71,8 +71,20 @@ run "same_az_two_subnets" { error_message = "the log driver must be given the module's region" } assert { - condition = strcontains(aws_ecs_task_definition.this.container_definitions, "QUELL_HASH_SECRET") - error_message = "QUELL_HASH_SECRET must reach the container" + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "ELASTIO_DBMON_HASH_SECRET") + error_message = "ELASTIO_DBMON_HASH_SECRET must reach the container" + } + assert { + condition = !strcontains(aws_ecs_task_definition.this.container_definitions, "QUELL_") + error_message = "the agent is configured by the ELASTIO_DBMON_* names, not the old QUELL_* ones" + } + assert { + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "\"name\":\"GOMEMLIMIT\",\"value\":\"409MiB\"") + error_message = "the Go memory limit must be 80% of the 512 MiB task" + } + assert { + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "\"containerPath\":\"/var/lib/elastio-dbmon\"") && strcontains(aws_ecs_task_definition.this.container_definitions, "\"value\":\"/var/lib/elastio-dbmon/ledger\"") + error_message = "the ledger must be mounted where the agent image keeps it, and named explicitly" } assert { condition = aws_ecs_service.this.deployment_maximum_percent == 100 && aws_ecs_service.this.deployment_minimum_healthy_percent == 0 diff --git a/database-monitoring/terraform/variables.tf b/database-monitoring/terraform/variables.tf index af065e1..df6826d 100644 --- a/database-monitoring/terraform/variables.tf +++ b/database-monitoring/terraform/variables.tf @@ -56,9 +56,9 @@ variable "security_group_ids" { } variable "image" { - description = "Agent image. The default is the agent release this module version was tested with. Pin a version; :latest moves." + description = "Agent image. The default is the agent release this module version was tested with. Pin a version; :latest moves. Needs 0.1.5 or later: older agents read only the QUELL_* variable names." type = string - default = "public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.4" + default = "public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.5" } variable "assign_public_ip" { From eb70e6f70fc3e431c93f83f10cf0a1a23cdc662c Mon Sep 17 00:00:00 2001 From: Maksim Svetlakov Date: Thu, 24 Sep 2026 12:02:45 -0400 Subject: [PATCH 5/8] Restrict the EFS ledger to the task role through the access point EFS admits a client when either its identity policy or the file system policy allows it, so an Allow alone does not keep out other roles with EFS permissions. Add explicit Denies for every principal except the task role and for every mount that does not use the access point. --- database-monitoring/terraform/README.md | 2 +- database-monitoring/terraform/main.tf | 28 +++++++++++++++++-- .../terraform/tests/module.tftest.hcl | 4 +++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/database-monitoring/terraform/README.md b/database-monitoring/terraform/README.md index f5cf504..a2e9d87 100644 --- a/database-monitoring/terraform/README.md +++ b/database-monitoring/terraform/README.md @@ -61,7 +61,7 @@ A security group with the usual allow-all egress already allows all three. The d - a security group `elastio-dbmon--efs-*` on the mount targets, which allows TCP 2049 only from `security_group_ids`; - an access point that maps every client to uid/gid 65532 and roots it at `/elastio-dbmon`, created with mode 0700; - a task-role policy that allows `elasticfilesystem:ClientMount` and `ClientWrite` on the file system, only through that access point. - - a file system policy that admits only the task role, only through that access point, and denies every client that does not use TLS. + - a file system policy that denies every principal except the task role, every mount that does not use that access point, and every client that does not use TLS. - A Fargate task definition: 0.25 vCPU, 0.5 GB, ARM64, with a read-only root filesystem. The task runs as the image's non-root user. - A service with `desired_count = 1`. Its deployment policy (minimum healthy 0%, maximum 100%) stops the old task before it starts the new one. A replication slot allows only one consumer at a time, and the ledger allows only one writer. Don't raise the maximum. diff --git a/database-monitoring/terraform/main.tf b/database-monitoring/terraform/main.tf index b97c658..ef3db8d 100644 --- a/database-monitoring/terraform/main.tf +++ b/database-monitoring/terraform/main.tf @@ -301,8 +301,10 @@ resource "aws_iam_role_policy" "mount_ledger" { # Without a file system policy, EFS lets any NFS client that reaches a mount # target mount it as root. The security groups in `security_group_ids` may be -# shared with other workloads, so the file system itself admits only the task -# role, only through the access point, and only over TLS. +# shared with other workloads. EFS admits a client that either its own IAM +# policy or this policy allows, so an Allow alone does not keep out other +# roles with EFS permissions. The explicit Denies do: only the task role, only +# through the access point, and only over TLS. resource "aws_efs_file_system_policy" "ledger" { count = var.persistent_ledger ? 1 : 0 @@ -327,6 +329,28 @@ resource "aws_efs_file_system_policy" "ledger" { } } }, + { + Sid = "DenyOtherPrincipals" + Effect = "Deny" + Principal = { AWS = "*" } + Action = "elasticfilesystem:Client*" + Resource = aws_efs_file_system.ledger[0].arn + Condition = { + ArnNotEquals = { "aws:PrincipalArn" = aws_iam_role.task.arn } + } + }, + { + Sid = "DenyWithoutAccessPoint" + Effect = "Deny" + Principal = { AWS = "*" } + Action = "elasticfilesystem:Client*" + Resource = aws_efs_file_system.ledger[0].arn + Condition = { + StringNotEquals = { + "elasticfilesystem:AccessPointArn" = aws_efs_access_point.ledger[0].arn + } + } + }, { Sid = "DenyInsecureTransport" Effect = "Deny" diff --git a/database-monitoring/terraform/tests/module.tftest.hcl b/database-monitoring/terraform/tests/module.tftest.hcl index 87b62b4..f27191c 100644 --- a/database-monitoring/terraform/tests/module.tftest.hcl +++ b/database-monitoring/terraform/tests/module.tftest.hcl @@ -94,6 +94,10 @@ run "same_az_two_subnets" { condition = jsondecode(aws_efs_file_system_policy.ledger[0].policy).Statement[0].Condition.StringEquals["elasticfilesystem:AccessPointArn"] == aws_efs_access_point.ledger[0].arn error_message = "the file system policy must admit the task only through the access point" } + assert { + condition = toset([for st in jsondecode(aws_efs_file_system_policy.ledger[0].policy).Statement : st.Sid if st.Effect == "Deny"]) == toset(["DenyOtherPrincipals", "DenyWithoutAccessPoint", "DenyInsecureTransport"]) + error_message = "the file system policy must deny other principals, access without the access point, and access without TLS" + } } run "rejects_empty_network" { From 47357e0c3cae1d0f458710963a1421cc694c7ba9 Mon Sep 17 00:00:00 2001 From: Maksim Svetlakov Date: Thu, 24 Sep 2026 12:24:47 -0400 Subject: [PATCH 6/8] Drop the EFS deny on aws:PrincipalArn EFS does not enforce aws:PrincipalArn for NFS clients, so the deny did not restrict other principals and could refuse the agent itself. Keep only the denies on the access point and TLS, which use keys EFS enforces, and state that the EFS security group is the boundary against other roles with their own EFS permissions. --- database-monitoring/terraform/README.md | 2 +- database-monitoring/terraform/main.tf | 21 ++++++------------- .../terraform/tests/module.tftest.hcl | 4 ++-- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/database-monitoring/terraform/README.md b/database-monitoring/terraform/README.md index a2e9d87..88e63ec 100644 --- a/database-monitoring/terraform/README.md +++ b/database-monitoring/terraform/README.md @@ -61,7 +61,7 @@ A security group with the usual allow-all egress already allows all three. The d - a security group `elastio-dbmon--efs-*` on the mount targets, which allows TCP 2049 only from `security_group_ids`; - an access point that maps every client to uid/gid 65532 and roots it at `/elastio-dbmon`, created with mode 0700; - a task-role policy that allows `elasticfilesystem:ClientMount` and `ClientWrite` on the file system, only through that access point. - - a file system policy that denies every principal except the task role, every mount that does not use that access point, and every client that does not use TLS. + - a file system policy that refuses anonymous clients and denies every mount that does not use that access point or TLS. Other roles in the account with their own EFS permissions can still mount through the access point if the network admits them, so keep `security_group_ids` for the agent only. - A Fargate task definition: 0.25 vCPU, 0.5 GB, ARM64, with a read-only root filesystem. The task runs as the image's non-root user. - A service with `desired_count = 1`. Its deployment policy (minimum healthy 0%, maximum 100%) stops the old task before it starts the new one. A replication slot allows only one consumer at a time, and the ledger allows only one writer. Don't raise the maximum. diff --git a/database-monitoring/terraform/main.tf b/database-monitoring/terraform/main.tf index ef3db8d..5f8d9e1 100644 --- a/database-monitoring/terraform/main.tf +++ b/database-monitoring/terraform/main.tf @@ -300,11 +300,12 @@ resource "aws_iam_role_policy" "mount_ledger" { } # Without a file system policy, EFS lets any NFS client that reaches a mount -# target mount it as root. The security groups in `security_group_ids` may be -# shared with other workloads. EFS admits a client that either its own IAM -# policy or this policy allows, so an Allow alone does not keep out other -# roles with EFS permissions. The explicit Denies do: only the task role, only -# through the access point, and only over TLS. +# target mount it as root. With this policy, anonymous clients are refused, +# and every client must use TLS and the module's access point. EFS enforces +# only a few condition keys for NFS clients, and none identifies the caller, +# so another role in the account with its own EFS client permissions can +# still mount through the access point if the network admits it. The EFS +# security group, which admits only `security_group_ids`, is that boundary. resource "aws_efs_file_system_policy" "ledger" { count = var.persistent_ledger ? 1 : 0 @@ -329,16 +330,6 @@ resource "aws_efs_file_system_policy" "ledger" { } } }, - { - Sid = "DenyOtherPrincipals" - Effect = "Deny" - Principal = { AWS = "*" } - Action = "elasticfilesystem:Client*" - Resource = aws_efs_file_system.ledger[0].arn - Condition = { - ArnNotEquals = { "aws:PrincipalArn" = aws_iam_role.task.arn } - } - }, { Sid = "DenyWithoutAccessPoint" Effect = "Deny" diff --git a/database-monitoring/terraform/tests/module.tftest.hcl b/database-monitoring/terraform/tests/module.tftest.hcl index f27191c..83b62eb 100644 --- a/database-monitoring/terraform/tests/module.tftest.hcl +++ b/database-monitoring/terraform/tests/module.tftest.hcl @@ -95,8 +95,8 @@ run "same_az_two_subnets" { error_message = "the file system policy must admit the task only through the access point" } assert { - condition = toset([for st in jsondecode(aws_efs_file_system_policy.ledger[0].policy).Statement : st.Sid if st.Effect == "Deny"]) == toset(["DenyOtherPrincipals", "DenyWithoutAccessPoint", "DenyInsecureTransport"]) - error_message = "the file system policy must deny other principals, access without the access point, and access without TLS" + condition = toset([for st in jsondecode(aws_efs_file_system_policy.ledger[0].policy).Statement : st.Sid if st.Effect == "Deny"]) == toset(["DenyWithoutAccessPoint", "DenyInsecureTransport"]) + error_message = "the file system policy must deny access without the access point and access without TLS" } } From 387bdde7aacf0c47a69830886fa5c003b3a32187 Mon Sep 17 00:00:00 2001 From: Anshuman Biswas Date: Thu, 24 Sep 2026 12:36:58 -0400 Subject: [PATCH 7/8] Size the agent's task with task_cpu and task_memory, and run 0.1.6 The task was fixed at 0.25 vCPU and 512 MiB. What a larger task buys is the largest transaction the agent holds in memory whole: agent 0.1.6 judges anything bigger from its row counts and reports an oversized transaction with the smallest task memory that would have held it. So the size is now two inputs, task_cpu (default 256) and task_memory (default 512, MiB). They are validated as a pair Fargate runs on ARM64: 256 with 512, 1024 or 2048; 512 with 1024 to 4096; 1024 with 2048 to 8192; 2048 with 4096 to 16384; memory above 512 in steps of 1024. GOMEMLIMIT follows task_memory at floor(task_memory * 0.8) MiB, so the agent's transaction budget follows the size too. The default image is 0.1.6. The README has a Sizing section with the four sizes the product offers, what each holds and costs in us-east-1, and how the Elastio UI says when to size up. The tests plan each size and refuse pairs Fargate does not run. --- database-monitoring/terraform/README.md | 64 +++++++--- .../terraform/examples/basic/main.tf | 6 + database-monitoring/terraform/main.tf | 28 +++-- .../terraform/tests/module.tftest.hcl | 116 +++++++++++++++++- database-monitoring/terraform/variables.tf | 29 ++++- 5 files changed, 211 insertions(+), 32 deletions(-) diff --git a/database-monitoring/terraform/README.md b/database-monitoring/terraform/README.md index 88e63ec..0479645 100644 --- a/database-monitoring/terraform/README.md +++ b/database-monitoring/terraform/README.md @@ -62,12 +62,38 @@ A security group with the usual allow-all egress already allows all three. The d - an access point that maps every client to uid/gid 65532 and roots it at `/elastio-dbmon`, created with mode 0700; - a task-role policy that allows `elasticfilesystem:ClientMount` and `ClientWrite` on the file system, only through that access point. - a file system policy that refuses anonymous clients and denies every mount that does not use that access point or TLS. Other roles in the account with their own EFS permissions can still mount through the access point if the network admits them, so keep `security_group_ids` for the agent only. -- A Fargate task definition: 0.25 vCPU, 0.5 GB, ARM64, with a read-only root filesystem. The task runs as the image's non-root user. +- A Fargate task definition of `task_cpu` and `task_memory` (by default 0.25 vCPU and 0.5 GB), ARM64, with a read-only root filesystem. The task runs as the image's non-root user, with `GOMEMLIMIT` set to 80% of `task_memory`. - A service with `desired_count = 1`. Its deployment policy (minimum healthy 0%, maximum 100%) stops the old task before it starts the new one. A replication slot allows only one consumer at a time, and the ledger allows only one writer. Don't raise the maximum. +## Sizing + +The default size, 0.25 vCPU and 512 MiB, is enough for one database's ordinary traffic. What a larger task buys is the size of the largest single transaction the agent can hold in memory whole. + +A transaction larger than that doesn't stop the agent. The agent judges it from its row counts: how many rows each table lost, and what share of the table that is. A bulk deletion still raises a finding. Checks that need to know which rows changed can't run on it, such as whether the deleted rows were loaded back. The finding says which checks didn't run and why. + +**The Elastio UI tells you when to size up.** When a transaction was too large to hold, the database shows an "oversized transaction" observation. It names the transaction, the rows it changed in each table, and the smallest task memory that would have held it. Set `task_memory` to that value or higher, with a `task_cpu` that allows it, and apply. + +| Size | `task_cpu` | `task_memory` | `GOMEMLIMIT` | Largest transaction held whole, about | Per month, us-east-1 | +| --------------- | ---------- | ------------- | ------------ | -------------------------------------------- | -------------------- | +| small (default) | 256 | 512 | 409 MiB | 275,000 rows deleted or 170,000 inserted | $7.21 | +| medium | 512 | 1024 | 819 MiB | 550,000 deleted or 340,000 inserted | $14.42 | +| large | 1024 | 2048 | 1638 MiB | 1.1 million deleted or 680,000 inserted | $28.84 | +| xlarge | 2048 | 4096 | 3276 MiB | 2.2 million deleted or 1.36 million inserted | $57.67 | + +The row counts assume tables with a single integer primary key. A wider key, or `REPLICA IDENTITY FULL`, means fewer rows. Fargate also accepts other pairs, and the module accepts any pair that Fargate runs on ARM64: + +| `task_cpu` | `task_memory` (MiB) | +| --------------- | ------------------------------- | +| 256 (0.25 vCPU) | 512, 1024, 2048 | +| 512 (0.5 vCPU) | 1024 to 4096, in steps of 1024 | +| 1024 (1 vCPU) | 2048 to 8192, in steps of 1024 | +| 2048 (2 vCPU) | 4096 to 16384, in steps of 1024 | + +A different size replaces the task. The ledger is on EFS, so nothing is lost, and the replication slot keeps the write-ahead log for the minute or so that no task is running. + ## Cost -In us-east-1 at on-demand rates, the smallest ARM Fargate size costs: +In us-east-1 at on-demand rates, ARM Fargate costs $0.03238 per vCPU-hour and $0.00356 per GB-hour. The default size costs: | | rate | per month (730 h) | | --------- | ---------------------- | ----------------- | @@ -75,7 +101,7 @@ In us-east-1 at on-demand rates, the smallest ARM Fargate size costs: | 0.5 GB | $0.00356 per GB-hour | $1.30 | | | | **$7.21** | -On top of that: +The other sizes are in the Sizing table above. On top of the task: - **Secrets Manager:** three secrets at $0.40 each, so $1.20 a month. - **EFS:** the ledger holds only minimised evidence (keys and hashes, never row contents) and is pruned as it goes. It stays at tens of MB, which is cents a month. Mount targets and access points are free. @@ -150,21 +176,23 @@ No modules. ## Inputs -| Name | Description | Type | Default | Required | -| --------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------------------ | :------: | -| [api_key](#input_api_key) | Bearer token the agent presents to the server. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | -| [assign_public_ip](#input_assign_public_ip) | Give the task a public IP. Needed only when the subnets have no NAT gateway and the server URL is on the internet. | `bool` | `false` | no | -| [database_url](#input_database_url) | postgres:// URL of the database to watch, for a role with REPLICATION. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | -| [image](#input_image) | Agent image. The default is the agent release this module version was tested with. Pin a version; :latest moves. Needs 0.1.5 or later: older agents read only the QUELL\_\* variable names. | `string` | `"public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.5"` | no | -| [log_retention_days](#input_log_retention_days) | How long CloudWatch keeps the agent's log. | `number` | `7` | no | -| [name](#input_name) | What the product shows for this agent. Also the ECS cluster, service and task family name, sanitised to the characters ECS allows. | `string` | n/a | yes | -| [persistent_ledger](#input_persistent_ledger) | Keep the agent's ledger on an encrypted EFS file system so it survives task replacement. When false, the ledger is on the task's ephemeral storage and every replacement starts it empty, after which findings standing at the time can never be explained by later activity. | `bool` | `true` | no | -| [publication](#input_publication) | Name of the publication whose tables are streamed. | `string` | `"elastio_monitor"` | no | -| [security_group_ids](#input_security_group_ids) | Security groups attached to the task. The database's own security group must admit them on its port. | `list(string)` | n/a | yes | -| [server_url](#input_server_url) | Base URL of the server the agent reports to. | `string` | n/a | yes | -| [slot](#input_slot) | Name of the existing pgoutput replication slot. The agent never creates or drops one. | `string` | `"elastio_monitor"` | no | -| [subnet_ids](#input_subnet_ids) | Subnets the task runs in. They must reach the database and the server URL. | `list(string)` | n/a | yes | -| [tags](#input_tags) | Tags applied to every resource the module creates. | `map(string)` | `{}` | no | +| Name | Description | Type | Default | Required | +| --------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------------------ | :------: | +| [api_key](#input_api_key) | Bearer token the agent presents to the server. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | +| [assign_public_ip](#input_assign_public_ip) | Give the task a public IP. Needed only when the subnets have no NAT gateway and the server URL is on the internet. | `bool` | `false` | no | +| [database_url](#input_database_url) | postgres:// URL of the database to watch, for a role with REPLICATION. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | +| [image](#input_image) | Agent image. The default is the agent release this module version was tested with. Pin a version; :latest moves. Needs 0.1.5 or later: older agents read only the QUELL\_\* variable names. | `string` | `"public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.6"` | no | +| [log_retention_days](#input_log_retention_days) | How long CloudWatch keeps the agent's log. | `number` | `7` | no | +| [name](#input_name) | What the product shows for this agent. Also the ECS cluster, service and task family name, sanitised to the characters ECS allows. | `string` | n/a | yes | +| [persistent_ledger](#input_persistent_ledger) | Keep the agent's ledger on an encrypted EFS file system so it survives task replacement. When false, the ledger is on the task's ephemeral storage and every replacement starts it empty, after which findings standing at the time can never be explained by later activity. | `bool` | `true` | no | +| [publication](#input_publication) | Name of the publication whose tables are streamed. | `string` | `"elastio_monitor"` | no | +| [security_group_ids](#input_security_group_ids) | Security groups attached to the task. The database's own security group must admit them on its port. | `list(string)` | n/a | yes | +| [server_url](#input_server_url) | Base URL of the server the agent reports to. | `string` | n/a | yes | +| [slot](#input_slot) | Name of the existing pgoutput replication slot. The agent never creates or drops one. | `string` | `"elastio_monitor"` | no | +| [subnet_ids](#input_subnet_ids) | Subnets the task runs in. They must reach the database and the server URL. | `list(string)` | n/a | yes | +| [tags](#input_tags) | Tags applied to every resource the module creates. | `map(string)` | `{}` | no | +| [task_cpu](#input_task_cpu) | CPU units for the Fargate task: 256 (0.25 vCPU), 512, 1024 or 2048. Together with task_memory it must be a size Fargate runs on ARM64; see Sizing in the README. | `number` | `256` | no | +| [task_memory](#input_task_memory) | Memory for the Fargate task, in MiB. With task_cpu 256: 512, 1024 or 2048. With 512: 1024 to 4096. With 1024: 2048 to 8192. With 2048: 4096 to 16384. Above 512, in steps of 1024. The agent's Go memory limit, and the largest transaction it holds whole, follow it. Raise it when the Elastio UI says a transaction was too large for the agent to hold. | `number` | `512` | no | ## Outputs diff --git a/database-monitoring/terraform/examples/basic/main.tf b/database-monitoring/terraform/examples/basic/main.tf index f8df436..b848df0 100644 --- a/database-monitoring/terraform/examples/basic/main.tf +++ b/database-monitoring/terraform/examples/basic/main.tf @@ -8,4 +8,10 @@ module "elastio_database_monitoring_agent" { subnet_ids = var.subnet_ids security_group_ids = var.security_group_ids + + # The smallest size, and the default. Raise both when the Elastio UI says a + # transaction was too large for the agent to hold; see Sizing in the + # module's README. + task_cpu = 256 + task_memory = 512 } diff --git a/database-monitoring/terraform/main.tf b/database-monitoring/terraform/main.tf index 5f8d9e1..6ca7199 100644 --- a/database-monitoring/terraform/main.tf +++ b/database-monitoring/terraform/main.tf @@ -30,17 +30,23 @@ locals { # same file under either path, because the path is also passed explicitly. ledger_dir = "/var/lib/elastio-dbmon" - # Task size. The Fargate floor: one agent reads one database's stream and - # needs no more (measured in elastio/database-monitoring-agent bench/). - task_cpu = 256 - task_memory = 512 - - # The Go runtime's soft memory limit, 80% of the task. Go does not derive - # one from the container on its own, and without it a heap whose live size - # is half the task is allowed to double before it is collected. The agent - # derives the same number from its cgroup when this is unset; it is set - # here too because what Fargate exposes inside the container is not - # something to rely on. + # Task size. The default is the Fargate floor: one agent reads one + # database's stream and needs no more for ordinary traffic (measured in + # elastio/database-monitoring-agent bench/). What a larger task buys is the + # size of the largest transaction the agent holds whole; past that it + # judges the transaction from its row counts and says so. See Sizing in the + # README. + task_cpu = var.task_cpu + task_memory = var.task_memory + + # The Go runtime's soft memory limit, 80% of the task, and so the budget + # for one open transaction (30% of this). Go does not derive one from the + # container on its own, and without it a heap whose live size is half the + # task is allowed to double before it is collected. The agent derives the + # same number itself when this is unset (from 0.1.6 through the ECS task + # metadata, because a Fargate task's limit is on the task and the + # container's cgroup reads "max"); it is set here so the number in force + # is visible in the task definition. gomemlimit = "${floor(local.task_memory * 0.8)}MiB" } diff --git a/database-monitoring/terraform/tests/module.tftest.hcl b/database-monitoring/terraform/tests/module.tftest.hcl index 83b62eb..487b608 100644 --- a/database-monitoring/terraform/tests/module.tftest.hcl +++ b/database-monitoring/terraform/tests/module.tftest.hcl @@ -135,11 +135,123 @@ run "names_and_image" { error_message = "secrets must be named elastio-dbmon//..." } assert { - condition = strcontains(aws_ecs_task_definition.this.container_definitions, "public.ecr.aws/elastio/elastio-database-monitoring-agent:") - error_message = "the default image must be Elastio's public ECR image" + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "\"image\":\"public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.6\"") + error_message = "the default image must be Elastio's public ECR image of agent 0.1.6, the first to report oversized transactions and read its limit on Fargate" } } +# The default is the smallest size, and the Go memory limit follows it. +run "default_size" { + command = plan + assert { + condition = aws_ecs_task_definition.this.cpu == "256" && aws_ecs_task_definition.this.memory == "512" + error_message = "the default task is 0.25 vCPU and 512 MiB" + } + assert { + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "\"name\":\"GOMEMLIMIT\",\"value\":\"409MiB\"") + error_message = "GOMEMLIMIT must be 80% of the 512 MiB task" + } +} + +# Each size the product offers, and one between them: the task takes the +# size, and GOMEMLIMIT is floor(task_memory * 0.8) MiB. +run "size_medium" { + command = plan + variables { + task_cpu = 512 + task_memory = 1024 + } + assert { + condition = aws_ecs_task_definition.this.cpu == "512" && aws_ecs_task_definition.this.memory == "1024" + error_message = "medium is 512 CPU units and 1024 MiB" + } + assert { + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "\"name\":\"GOMEMLIMIT\",\"value\":\"819MiB\"") + error_message = "GOMEMLIMIT must be 819MiB for a 1024 MiB task" + } +} + +run "size_large" { + command = plan + variables { + task_cpu = 1024 + task_memory = 2048 + } + assert { + condition = aws_ecs_task_definition.this.cpu == "1024" && aws_ecs_task_definition.this.memory == "2048" + error_message = "large is 1024 CPU units and 2048 MiB" + } + assert { + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "\"name\":\"GOMEMLIMIT\",\"value\":\"1638MiB\"") + error_message = "GOMEMLIMIT must be 1638MiB for a 2048 MiB task" + } +} + +run "size_xlarge" { + command = plan + variables { + task_cpu = 2048 + task_memory = 4096 + } + assert { + condition = aws_ecs_task_definition.this.cpu == "2048" && aws_ecs_task_definition.this.memory == "4096" + error_message = "xlarge is 2048 CPU units and 4096 MiB" + } + assert { + condition = strcontains(aws_ecs_task_definition.this.container_definitions, "\"name\":\"GOMEMLIMIT\",\"value\":\"3276MiB\"") + error_message = "GOMEMLIMIT must be 3276MiB for a 4096 MiB task" + } +} + +run "size_between_presets" { + command = plan + variables { + task_cpu = 512 + task_memory = 3072 + } + assert { + condition = aws_ecs_task_definition.this.memory == "3072" && strcontains(aws_ecs_task_definition.this.container_definitions, "\"name\":\"GOMEMLIMIT\",\"value\":\"2457MiB\"") + error_message = "any pair Fargate runs is accepted, and GOMEMLIMIT follows it" + } +} + +# Pairs Fargate does not run are refused at plan, not by ECS at apply. +run "rejects_memory_too_large_for_cpu" { + command = plan + variables { + task_cpu = 256 + task_memory = 4096 + } + expect_failures = [var.task_memory] +} + +run "rejects_memory_too_small_for_cpu" { + command = plan + variables { + task_cpu = 2048 + task_memory = 2048 + } + expect_failures = [var.task_memory] +} + +run "rejects_memory_off_the_1024_step" { + command = plan + variables { + task_cpu = 512 + task_memory = 1536 + } + expect_failures = [var.task_memory] +} + +run "rejects_unknown_cpu" { + command = plan + variables { + task_cpu = 4096 + task_memory = 8192 + } + expect_failures = [var.task_cpu] +} + run "ephemeral" { command = apply variables { diff --git a/database-monitoring/terraform/variables.tf b/database-monitoring/terraform/variables.tf index df6826d..6832fcf 100644 --- a/database-monitoring/terraform/variables.tf +++ b/database-monitoring/terraform/variables.tf @@ -58,7 +58,34 @@ variable "security_group_ids" { variable "image" { description = "Agent image. The default is the agent release this module version was tested with. Pin a version; :latest moves. Needs 0.1.5 or later: older agents read only the QUELL_* variable names." type = string - default = "public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.5" + default = "public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.6" +} + +variable "task_cpu" { + description = "CPU units for the Fargate task: 256 (0.25 vCPU), 512, 1024 or 2048. Together with task_memory it must be a size Fargate runs on ARM64; see Sizing in the README." + type = number + default = 256 + + validation { + condition = contains([256, 512, 1024, 2048], var.task_cpu) + error_message = "task_cpu must be 256, 512, 1024 or 2048." + } +} + +variable "task_memory" { + description = "Memory for the Fargate task, in MiB. With task_cpu 256: 512, 1024 or 2048. With 512: 1024 to 4096. With 1024: 2048 to 8192. With 2048: 4096 to 16384. Above 512, in steps of 1024. The agent's Go memory limit, and the largest transaction it holds whole, follow it. Raise it when the Elastio UI says a transaction was too large for the agent to hold." + type = number + default = 512 + + validation { + condition = ( + (var.task_cpu == 256 && contains([512, 1024, 2048], var.task_memory)) || + (var.task_cpu == 512 && var.task_memory >= 1024 && var.task_memory <= 4096 && var.task_memory % 1024 == 0) || + (var.task_cpu == 1024 && var.task_memory >= 2048 && var.task_memory <= 8192 && var.task_memory % 1024 == 0) || + (var.task_cpu == 2048 && var.task_memory >= 4096 && var.task_memory <= 16384 && var.task_memory % 1024 == 0) + ) + error_message = "task_cpu and task_memory must be a pair Fargate supports on ARM64: 256 with 512, 1024 or 2048; 512 with 1024 to 4096; 1024 with 2048 to 8192; 2048 with 4096 to 16384. Above 512 MiB, memory goes in steps of 1024." + } } variable "assign_public_ip" { From ebd11aefc306881d8b6e536860f7a9c14f7dcb65 Mon Sep 17 00:00:00 2001 From: Maksim Svetlakov Date: Thu, 24 Sep 2026 12:50:04 -0400 Subject: [PATCH 8/8] Describe the image property the ephemeral ledger mount needs The comment said Fargate makes the bind mount writable for the container's uid. It is writable when the image has a VOLUME at the ledger directory owned by uid 65532, because ECS copies that VOLUME into the bind mount. Also note that the task size is now configurable. --- database-monitoring/terraform/main.tf | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/database-monitoring/terraform/main.tf b/database-monitoring/terraform/main.tf index 6ca7199..9a9d0a2 100644 --- a/database-monitoring/terraform/main.tf +++ b/database-monitoring/terraform/main.tf @@ -362,11 +362,13 @@ resource "aws_efs_file_system_policy" "ledger" { }) } -# The task. The smallest Fargate size, on ARM because it is the cheaper of -# the two and the image is published for both. The root filesystem is -# read-only; the ledger directory is the EFS access point above, or, with -# persistent_ledger = false, a bind mount onto the task's ephemeral storage, -# which Fargate creates writable for the container's uid. +# The task. By default the smallest Fargate size (see task_cpu and +# task_memory), on ARM because it is the cheaper of the two and the image is +# published for both. The root filesystem is read-only; the ledger directory +# is the EFS access point above, or, with persistent_ledger = false, a bind +# mount onto the task's ephemeral storage. That mount is writable when the +# image has a VOLUME at the ledger directory owned by uid 65532: ECS copies a +# VOLUME at the container path into the bind mount, owner included. resource "aws_ecs_task_definition" "this" { family = local.ecs_name