From 2261392075d49ff94ccfd08e886c3be81c8bfd1a Mon Sep 17 00:00:00 2001 From: Anshuman Biswas Date: Thu, 24 Sep 2026 20:47:55 -0400 Subject: [PATCH] database-monitoring: keep the module README to usage --- database-monitoring/terraform/README.md | 86 +++++++++++----------- database-monitoring/terraform/main.tf | 52 +++++-------- database-monitoring/terraform/outputs.tf | 6 +- database-monitoring/terraform/variables.tf | 4 +- 4 files changed, 70 insertions(+), 78 deletions(-) diff --git a/database-monitoring/terraform/README.md b/database-monitoring/terraform/README.md index 0479645..2b36884 100644 --- a/database-monitoring/terraform/README.md +++ b/database-monitoring/terraform/README.md @@ -33,7 +33,7 @@ You can find the full version of this example in [`examples/basic`](./examples/b The `image` input defaults to the agent release this module version was tested with. Leave it unset unless Elastio support asks you to pin a different one. -Pass `api_key` and `database_url` as sensitive variables or from your own secret store. They are stored in two Secrets Manager secrets that the module creates, and ECS reads them when the task starts. They never appear in the task definition. The module also generates a third secret, the agent's hashing key. +Pass `api_key` and `database_url` as sensitive variables or from your own secret store. They are stored in two Secrets Manager secrets that the module creates, and ECS reads them when the task starts. They never appear in the task definition. The module also generates a third secret, a random key for the agent. ## Prerequisites @@ -45,7 +45,7 @@ Pass `api_key` and `database_url` as sensitive variables or from your own secret - the database port, to the database; - TCP 443, to your Elastio tenant; -- TCP 2049, to the VPC, for NFS to the ledger's EFS file system (only needed with the default `persistent_ledger = true`). +- TCP 2049, to the VPC, for NFS to the agent's EFS file system (only needed with the default `persistent_ledger = true`). A security group with the usual allow-all egress already allows all three. The database's own security group must allow inbound traffic from these security groups on the database port. @@ -53,9 +53,9 @@ A security group with the usual allow-all egress already allows all three. The d - An ECS cluster, service and task family named after `name` (with any characters ECS doesn't allow replaced), with Container Insights turned off. - A CloudWatch log group named `/elastio-dbmon/agent/`. -- Three Secrets Manager secrets, `elastio-dbmon//api-key-*`, `.../database-url-*` and `.../hash-secret-*`. The hashing key is 32 random bytes, hex-encoded. The agent minimises evidence under this key, so the key must stay the same for the life of the ledger. Never taint or replace it. +- Three Secrets Manager secrets, `elastio-dbmon//api-key-*`, `.../database-url-*` and `.../hash-secret-*`. The third is a random key, 32 bytes hex-encoded, that must stay the same for the life of the agent's state. Never taint or replace it. - IAM roles `elastio-dbmon--exec-*` and `elastio-dbmon--task-*`. The execution role has `AmazonECSTaskExecutionRolePolicy` and permission to read exactly those three secrets. The agent makes no AWS calls, so the task role carries only the EFS mount permission below. -- With `persistent_ledger = true` (the default), storage for the ledger: +- With `persistent_ledger = true` (the default), storage so the agent's state persists across task replacement: - an encrypted EFS file system tagged `elastio-dbmon--ledger`, using General Purpose performance mode and bursting throughput; - one mount target per distinct availability zone among `subnet_ids`; - a security group `elastio-dbmon--efs-*` on the mount targets, which allows TCP 2049 only from `security_group_ids`; @@ -63,24 +63,22 @@ A security group with the usual allow-all egress already allows all three. The d - a task-role policy that allows `elasticfilesystem:ClientMount` and `ClientWrite` on the file system, only through that access point. - a file system policy that refuses anonymous clients and denies every mount that does not use that access point or TLS. Other roles in the account with their own EFS permissions can still mount through the access point if the network admits them, so keep `security_group_ids` for the agent only. - A Fargate task definition of `task_cpu` and `task_memory` (by default 0.25 vCPU and 0.5 GB), ARM64, with a read-only root filesystem. The task runs as the image's non-root user, with `GOMEMLIMIT` set to 80% of `task_memory`. -- A service with `desired_count = 1`. Its deployment policy (minimum healthy 0%, maximum 100%) stops the old task before it starts the new one. A replication slot allows only one consumer at a time, and the ledger allows only one writer. Don't raise the maximum. +- A service with `desired_count = 1`. Its deployment policy (minimum healthy 0%, maximum 100%) stops the old task before it starts the new one. A replication slot allows only one consumer at a time, and the agent's state allows only one writer. Don't raise the maximum. ## Sizing -The default size, 0.25 vCPU and 512 MiB, is enough for one database's ordinary traffic. What a larger task buys is the size of the largest single transaction the agent can hold in memory whole. +The default size, 0.25 vCPU and 512 MiB, is enough for one database's ordinary traffic. A larger task handles larger single transactions. A transaction larger than the task can handle doesn't stop the agent. -A transaction larger than that doesn't stop the agent. The agent judges it from its row counts: how many rows each table lost, and what share of the table that is. A bulk deletion still raises a finding. Checks that need to know which rows changed can't run on it, such as whether the deleted rows were loaded back. The finding says which checks didn't run and why. +**The Elastio UI tells you when to size up.** When a transaction was too large for the task, the database shows an "oversized transaction" observation with the smallest task memory that would have handled it. Set `task_memory` to that value or higher, with a `task_cpu` that allows it, and apply. -**The Elastio UI tells you when to size up.** When a transaction was too large to hold, the database shows an "oversized transaction" observation. It names the transaction, the rows it changed in each table, and the smallest task memory that would have held it. Set `task_memory` to that value or higher, with a `task_cpu` that allows it, and apply. - -| Size | `task_cpu` | `task_memory` | `GOMEMLIMIT` | Largest transaction held whole, about | Per month, us-east-1 | +| Size | `task_cpu` | `task_memory` | `GOMEMLIMIT` | Largest transaction, about | Per month, us-east-1 | | --------------- | ---------- | ------------- | ------------ | -------------------------------------------- | -------------------- | | small (default) | 256 | 512 | 409 MiB | 275,000 rows deleted or 170,000 inserted | $7.21 | | medium | 512 | 1024 | 819 MiB | 550,000 deleted or 340,000 inserted | $14.42 | | large | 1024 | 2048 | 1638 MiB | 1.1 million deleted or 680,000 inserted | $28.84 | | xlarge | 2048 | 4096 | 3276 MiB | 2.2 million deleted or 1.36 million inserted | $57.67 | -The row counts assume tables with a single integer primary key. A wider key, or `REPLICA IDENTITY FULL`, means fewer rows. Fargate also accepts other pairs, and the module accepts any pair that Fargate runs on ARM64: +The transaction sizes are approximate and vary with table shape. Fargate also accepts other pairs, and the module accepts any pair that Fargate runs on ARM64: | `task_cpu` | `task_memory` (MiB) | | --------------- | ------------------------------- | @@ -89,7 +87,7 @@ The row counts assume tables with a single integer primary key. A wider key, or | 1024 (1 vCPU) | 2048 to 8192, in steps of 1024 | | 2048 (2 vCPU) | 4096 to 16384, in steps of 1024 | -A different size replaces the task. The ledger is on EFS, so nothing is lost, and the replication slot keeps the write-ahead log for the minute or so that no task is running. +A different size replaces the task. The agent's state is on EFS, so nothing is lost, and the replication slot keeps the write-ahead log for the minute or so that no task is running. ## Cost @@ -104,7 +102,7 @@ In us-east-1 at on-demand rates, ARM Fargate costs $0.03238 per vCPU-hour and $0 The other sizes are in the Sizing table above. On top of the task: - **Secrets Manager:** three secrets at $0.40 each, so $1.20 a month. -- **EFS:** the ledger holds only minimised evidence (keys and hashes, never row contents) and is pruned as it goes. It stays at tens of MB, which is cents a month. Mount targets and access points are free. +- **EFS:** the agent's state stays at tens of MB, which is cents a month. Mount targets and access points are free. - **Ephemeral storage and CloudWatch:** the extra GB of ephemeral storage above the free 20 GB, and CloudWatch log ingestion, each cost a few cents. If the subnets use a NAT gateway, its hourly charge will be your largest cost by far. You pay that charge whether or not you deploy this module. @@ -113,9 +111,15 @@ If the subnets use a NAT gateway, its hourly charge will be your largest cost by The agent writes its log to the log group above. `enable_execute_command` is off. -The ledger is on EFS, so it survives task replacement. That matters for findings that are still standing when the task is replaced, because the agent re-reads a finding's original window from the ledger to decide whether later activity explains it. With `persistent_ledger = false`, the ledger is on the task's ephemeral storage and every replacement starts it empty. Nothing is lost on the Elastio side, but findings that were standing at the time can no longer be explained. +The agent's state is on EFS, so it persists across task replacement. With `persistent_ledger = false`, the state is on the task's ephemeral storage and every replacement starts it empty. Nothing is lost on the Elastio side. + +## Upgrading + +To upgrade the agent, bump the module `version` and apply. A new version replaces the task; the agent's state is on EFS, so nothing is lost. + +## Removing -To upgrade the agent, bump the module `version` and apply. To remove the agent, run `terraform destroy`, then **drop the replication slot** in the database. An abandoned slot keeps write-ahead log until the disk fills. +To remove the agent, run `terraform destroy`, then **drop the replication slot** in the database. An abandoned slot keeps write-ahead log until the disk fills. ## Testing the module @@ -176,34 +180,34 @@ No modules. ## Inputs -| Name | Description | Type | Default | Required | -| --------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------------------ | :------: | -| [api_key](#input_api_key) | Bearer token the agent presents to the server. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | -| [assign_public_ip](#input_assign_public_ip) | Give the task a public IP. Needed only when the subnets have no NAT gateway and the server URL is on the internet. | `bool` | `false` | no | -| [database_url](#input_database_url) | postgres:// URL of the database to watch, for a role with REPLICATION. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | -| [image](#input_image) | Agent image. The default is the agent release this module version was tested with. Pin a version; :latest moves. Needs 0.1.5 or later: older agents read only the QUELL\_\* variable names. | `string` | `"public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.6"` | no | -| [log_retention_days](#input_log_retention_days) | How long CloudWatch keeps the agent's log. | `number` | `7` | no | -| [name](#input_name) | What the product shows for this agent. Also the ECS cluster, service and task family name, sanitised to the characters ECS allows. | `string` | n/a | yes | -| [persistent_ledger](#input_persistent_ledger) | Keep the agent's ledger on an encrypted EFS file system so it survives task replacement. When false, the ledger is on the task's ephemeral storage and every replacement starts it empty, after which findings standing at the time can never be explained by later activity. | `bool` | `true` | no | -| [publication](#input_publication) | Name of the publication whose tables are streamed. | `string` | `"elastio_monitor"` | no | -| [security_group_ids](#input_security_group_ids) | Security groups attached to the task. The database's own security group must admit them on its port. | `list(string)` | n/a | yes | -| [server_url](#input_server_url) | Base URL of the server the agent reports to. | `string` | n/a | yes | -| [slot](#input_slot) | Name of the existing pgoutput replication slot. The agent never creates or drops one. | `string` | `"elastio_monitor"` | no | -| [subnet_ids](#input_subnet_ids) | Subnets the task runs in. They must reach the database and the server URL. | `list(string)` | n/a | yes | -| [tags](#input_tags) | Tags applied to every resource the module creates. | `map(string)` | `{}` | no | -| [task_cpu](#input_task_cpu) | CPU units for the Fargate task: 256 (0.25 vCPU), 512, 1024 or 2048. Together with task_memory it must be a size Fargate runs on ARM64; see Sizing in the README. | `number` | `256` | no | -| [task_memory](#input_task_memory) | Memory for the Fargate task, in MiB. With task_cpu 256: 512, 1024 or 2048. With 512: 1024 to 4096. With 1024: 2048 to 8192. With 2048: 4096 to 16384. Above 512, in steps of 1024. The agent's Go memory limit, and the largest transaction it holds whole, follow it. Raise it when the Elastio UI says a transaction was too large for the agent to hold. | `number` | `512` | no | +| Name | Description | Type | Default | Required | +| --------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------------------ | :------: | +| [api_key](#input_api_key) | Bearer token the agent presents to the server. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | +| [assign_public_ip](#input_assign_public_ip) | Give the task a public IP. Needed only when the subnets have no NAT gateway and the server URL is on the internet. | `bool` | `false` | no | +| [database_url](#input_database_url) | postgres:// URL of the database to watch, for a role with REPLICATION. Stored in Secrets Manager, never in the task definition. | `string` | n/a | yes | +| [image](#input_image) | Agent image. The default is the agent release this module version was tested with. Pin a version; :latest moves. Needs 0.1.5 or later: older agents read only the QUELL\_\* variable names. | `string` | `"public.ecr.aws/elastio/elastio-database-monitoring-agent:0.1.6"` | no | +| [log_retention_days](#input_log_retention_days) | How long CloudWatch keeps the agent's log. | `number` | `7` | no | +| [name](#input_name) | What the product shows for this agent. Also the ECS cluster, service and task family name, sanitised to the characters ECS allows. | `string` | n/a | yes | +| [persistent_ledger](#input_persistent_ledger) | Keep the agent's state on an encrypted EFS file system so it persists across task replacement. When false, the state is on the task's ephemeral storage and every replacement starts it empty. | `bool` | `true` | no | +| [publication](#input_publication) | Name of the publication whose tables are streamed. | `string` | `"elastio_monitor"` | no | +| [security_group_ids](#input_security_group_ids) | Security groups attached to the task. The database's own security group must admit them on its port. | `list(string)` | n/a | yes | +| [server_url](#input_server_url) | Base URL of the server the agent reports to. | `string` | n/a | yes | +| [slot](#input_slot) | Name of the existing pgoutput replication slot. The agent never creates or drops one. | `string` | `"elastio_monitor"` | no | +| [subnet_ids](#input_subnet_ids) | Subnets the task runs in. They must reach the database and the server URL. | `list(string)` | n/a | yes | +| [tags](#input_tags) | Tags applied to every resource the module creates. | `map(string)` | `{}` | no | +| [task_cpu](#input_task_cpu) | CPU units for the Fargate task: 256 (0.25 vCPU), 512, 1024 or 2048. Together with task_memory it must be a size Fargate runs on ARM64; see Sizing in the README. | `number` | `256` | no | +| [task_memory](#input_task_memory) | Memory for the Fargate task, in MiB. With task_cpu 256: 512, 1024 or 2048. With 512: 1024 to 4096. With 1024: 2048 to 8192. With 2048: 4096 to 16384. Above 512, in steps of 1024. The agent's Go memory limit follows it. Raise it when the Elastio UI recommends a larger size. | `number` | `512` | no | ## Outputs -| Name | Description | -| -------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [cluster_arn](#output_cluster_arn) | ARN of the ECS cluster the agent runs in. | -| [execution_role_arn](#output_execution_role_arn) | IAM role ECS uses to pull the image, read the three secrets and write logs. | -| [hash_secret_arn](#output_hash_secret_arn) | ARN of the Secrets Manager secret holding the agent's hashing key. It must stay stable for the life of the ledger. | -| [ledger_file_system_id](#output_ledger_file_system_id) | ID of the EFS file system holding the agent's ledger, or null when persistent_ledger is false. | -| [log_group_name](#output_log_group_name) | CloudWatch log group the agent writes to. | -| [service_name](#output_service_name) | Name of the ECS service. | -| [task_role_arn](#output_task_role_arn) | IAM role the running container assumes. The agent has no AWS code; with a persistent ledger the role may mount the ledger's file system, and nothing else. | +| Name | Description | +| -------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| [cluster_arn](#output_cluster_arn) | ARN of the ECS cluster the agent runs in. | +| [execution_role_arn](#output_execution_role_arn) | IAM role ECS uses to pull the image, read the three secrets and write logs. | +| [hash_secret_arn](#output_hash_secret_arn) | ARN of the Secrets Manager secret holding the agent's key. It must stay stable for the life of the agent's state. | +| [ledger_file_system_id](#output_ledger_file_system_id) | ID of the EFS file system holding the agent's state, or null when persistent_ledger is false. | +| [log_group_name](#output_log_group_name) | CloudWatch log group the agent writes to. | +| [service_name](#output_service_name) | Name of the ECS service. | +| [task_role_arn](#output_task_role_arn) | IAM role the running container assumes. The agent has no AWS code; with persistent state the role may mount the state file system, and nothing else. | diff --git a/database-monitoring/terraform/main.tf b/database-monitoring/terraform/main.tf index 9a9d0a2..cabd3fc 100644 --- a/database-monitoring/terraform/main.tf +++ b/database-monitoring/terraform/main.tf @@ -2,7 +2,7 @@ # # The agent is a single static binary that needs a network path to the # database and to the server URL, three secrets, and a small writable directory -# for its ledger that outlives the task. Everything below is the least that +# for its state that outlives the task. Everything below is the least that # gives it those. locals { @@ -23,24 +23,19 @@ locals { uid = 65532 gid = 65532 - # The ledger directory inside the container. It is the agent image's own - # writable volume (owned by the nonroot user) from 0.1.5, and the binary's - # default ledger path is beneath it. Only the mount point moves with it: - # the EFS access point's contents, the ledger and nothing else, are the - # same file under either path, because the path is also passed explicitly. + # The agent's state directory inside the container. It is the agent image's + # own writable volume (owned by the nonroot user) from 0.1.5, and the + # binary's default state path is beneath it. The path is also passed + # explicitly, so the EFS access point's contents are the same file under + # either path. ledger_dir = "/var/lib/elastio-dbmon" - # Task size. The default is the Fargate floor: one agent reads one - # database's stream and needs no more for ordinary traffic (measured in - # elastio/database-monitoring-agent bench/). What a larger task buys is the - # size of the largest transaction the agent holds whole; past that it - # judges the transaction from its row counts and says so. See Sizing in the - # README. + # Task size. The default is the Fargate floor, which is enough for one + # database's ordinary traffic. See Sizing in the README. task_cpu = var.task_cpu task_memory = var.task_memory - # The Go runtime's soft memory limit, 80% of the task, and so the budget - # for one open transaction (30% of this). Go does not derive one from the + # The Go runtime's soft memory limit, 80% of the task. Go does not derive one from the # container on its own, and without it a heap whose live size is half the # task is allowed to double before it is collected. The agent derives the # same number itself when this is unset (from 0.1.6 through the ECS task @@ -100,15 +95,10 @@ resource "aws_secretsmanager_secret_version" "database_url" { secret_string = var.database_url } -# The agent's hashing key, used to minimise evidence before it leaves the -# database's network. Without it the agent generates one and keeps it beside -# the ledger, so a lost ledger took the key with it. Here the key is generated -# once, by Terraform, and outlives every task. It must stay stable for the -# life of the ledger: evidence minimised under a different key cannot be -# compared with what the ledger holds, and redelivered transactions would be -# refused as conflicting. So never taint or replace it while a ledger exists. -# 32 random bytes, hex-encoded: 64 characters, the same shape the agent -# generates for itself. +# A random key for the agent, generated once by Terraform so that it outlives +# every task. It must stay the same for the life of the agent's state, so +# never taint or replace it while that state exists. 32 random bytes, +# hex-encoded. resource "random_id" "hash_secret" { byte_length = 32 @@ -127,7 +117,7 @@ resource "aws_secretsmanager_secret_version" "hash_secret" { # Roles. The execution role is ECS's own: pull the image, read the secrets, # write logs. The task role is what the process would assume if it called -# AWS, and it never does. With a persistent ledger it is also the identity the +# AWS, and it never does. With persistent state it is also the identity the # EFS mount helper presents, and carries exactly the permission to mount and # write the file system through the module's access point. @@ -183,12 +173,10 @@ resource "aws_iam_role" "task" { tags = var.tags } -# The ledger's storage. The agent's review re-reads a standing finding's -# original window from the ledger to decide whether later activity explains -# it; a ledger that starts empty after a task replacement has lost that -# window, and the finding can then never be explained. Task replacements are -# routine (a deploy, a forced new deployment, a crash, Fargate's own platform -# maintenance), so by default the ledger lives on EFS and survives them. +# Storage for the agent's state, so it persists across task replacement. +# Task replacements are routine (a deploy, a forced new deployment, a crash, +# Fargate's own platform maintenance), so by default the state lives on EFS +# and survives them. # # One mount target per distinct availability zone among the subnets: EFS # admits one per AZ, and two subnets in the same AZ share it. @@ -364,10 +352,10 @@ resource "aws_efs_file_system_policy" "ledger" { # The task. By default the smallest Fargate size (see task_cpu and # task_memory), on ARM because it is the cheaper of the two and the image is -# published for both. The root filesystem is read-only; the ledger directory +# published for both. The root filesystem is read-only; the state directory # is the EFS access point above, or, with persistent_ledger = false, a bind # mount onto the task's ephemeral storage. That mount is writable when the -# image has a VOLUME at the ledger directory owned by uid 65532: ECS copies a +# image has a VOLUME at the state directory owned by uid 65532: ECS copies a # VOLUME at the container path into the bind mount, owner included. resource "aws_ecs_task_definition" "this" { diff --git a/database-monitoring/terraform/outputs.tf b/database-monitoring/terraform/outputs.tf index 2f7c889..b4da320 100644 --- a/database-monitoring/terraform/outputs.tf +++ b/database-monitoring/terraform/outputs.tf @@ -19,16 +19,16 @@ output "execution_role_arn" { } output "task_role_arn" { - description = "IAM role the running container assumes. The agent has no AWS code; with a persistent ledger the role may mount the ledger's file system, and nothing else." + description = "IAM role the running container assumes. The agent has no AWS code; with persistent state the role may mount the state file system, and nothing else." value = aws_iam_role.task.arn } output "ledger_file_system_id" { - description = "ID of the EFS file system holding the agent's ledger, or null when persistent_ledger is false." + description = "ID of the EFS file system holding the agent's state, or null when persistent_ledger is false." value = var.persistent_ledger ? aws_efs_file_system.ledger[0].id : null } output "hash_secret_arn" { - description = "ARN of the Secrets Manager secret holding the agent's hashing key. It must stay stable for the life of the ledger." + description = "ARN of the Secrets Manager secret holding the agent's key. It must stay stable for the life of the agent's state." value = aws_secretsmanager_secret.hash_secret.arn } diff --git a/database-monitoring/terraform/variables.tf b/database-monitoring/terraform/variables.tf index 6832fcf..8723f3a 100644 --- a/database-monitoring/terraform/variables.tf +++ b/database-monitoring/terraform/variables.tf @@ -73,7 +73,7 @@ variable "task_cpu" { } variable "task_memory" { - description = "Memory for the Fargate task, in MiB. With task_cpu 256: 512, 1024 or 2048. With 512: 1024 to 4096. With 1024: 2048 to 8192. With 2048: 4096 to 16384. Above 512, in steps of 1024. The agent's Go memory limit, and the largest transaction it holds whole, follow it. Raise it when the Elastio UI says a transaction was too large for the agent to hold." + description = "Memory for the Fargate task, in MiB. With task_cpu 256: 512, 1024 or 2048. With 512: 1024 to 4096. With 1024: 2048 to 8192. With 2048: 4096 to 16384. Above 512, in steps of 1024. The agent's Go memory limit follows it. Raise it when the Elastio UI recommends a larger size." type = number default = 512 @@ -106,7 +106,7 @@ variable "log_retention_days" { } variable "persistent_ledger" { - description = "Keep the agent's ledger on an encrypted EFS file system so it survives task replacement. When false, the ledger is on the task's ephemeral storage and every replacement starts it empty, after which findings standing at the time can never be explained by later activity." + description = "Keep the agent's state on an encrypted EFS file system so it persists across task replacement. When false, the state is on the task's ephemeral storage and every replacement starts it empty." type = bool default = true }