EBS volumes count management improvement

Author: DmitryFomin1

README.md

@@ -19,7 +19,7 @@ Your source PostgreSQL database can be located anywhere, but DLE with other comp
     * Read/Write permissions on Cloudwatch
 
 ## How to use
-- :construction: Currently, it is supposed that you run `terraform` commands on a Linux machine. MacOS and Windows support is not yet implemented (but planned).
+- :construction: Currently, it is supposed that you run `terraform` commands on a Linux machine or MacOS. Windows support is not yet implemented (but planned).
 - It is recommended to clone this Git repository and adjust for your needs. Below we provide the detailed step-by-step instructions for quick start (see "Quick start") for a PoC setup
 - To configure parameters used by Terraform (and the Database Lab Engine itself), you will need to modify `terraform.tfvars` and create a file with secrets (`secret.tfvars`)
 - This Terraform module can be run independently or combined with any other standard Terraform module. You can learn more about using Terraform and the Terraform CLI [here](https://www.terraform.io/docs/cli/commands/index.html)
@@ -58,6 +58,7 @@ The following steps were tested on Ubuntu 20.04 but supposed to be valid for oth
     aws_deploy_ec2_instance_type = "c5.large"
     aws_deploy_ec2_instance_tag_name = "DBLABserver-ec2instance"
     aws_deploy_ebs_size = "10"
+    aws_deploy_ec2_volumes_count = "2"
     aws_deploy_ebs_type = "gp2"
     aws_deploy_allow_ssh_from_cidrs = ["0.0.0.0/0"]
     aws_deploy_dns_api_subdomain = "tf-test" # subdomain in aws.postgres.ai, fqdn will be ${dns_api_subdomain}-engine.aws.postgres
@@ -73,7 +74,12 @@ The following steps were tested on Ubuntu 20.04 but supposed to be valid for oth
 
     platform_project_name = "aws_test_tf"
 
+    # list of ssh public keys stored in files
     ssh_public_keys_files_list = ["~/.ssh/id_rsa.pub"]
+    # or provided inline
+    ssh_public_keys_list = [
+      "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDhbblazDXCFEc21DtFzprWC8DiqidnVRROzp6J6BeJR9+XydPUtl0Rt2mcNvxL5ro5bI9u5JRW8aDd6s+Orpr66hEDdwQTbT1wp5nyduFQcT3rR   +aeDSilQvAHjr4/z/GZ6IgZ5MICSIh5hJJagHoxAVqeS9dCA27tv/n2T2XrxIUeBhywH1EmfwrnEw97tHM8F+yegayFDI1nVOUWUIxFMaygMygix8uKbQ2fl4rkkxG2oEx7uyAFMXHt4bewNbZuAp8b/b5ODL6tGHuHhcwfbWGriCO+l7UOf1K9maTx00o4wkzAPyd+qs70y/1iMX2YOOLYaYYdptEnFal2DVoD example@example.com"
+      ]
     ```
 1. Create `secret.tfvars` containing `source_postgres_password`, `platform_access_token`, and `vcs_github_secret_token`. An example:
     ```config

dle-logical-init.sh.tpl

@@ -181,7 +181,7 @@ case "${source_type}" in
     .retrieval.spec.logicalRestore.options.dumpLocation="${source_pgdump_s3_mount_point}/${source_pgdump_path_on_s3_bucket}"
   ' $dle_config_path/server.yml
 
-  nProcessors = $(getconf _NPROCESSORS_ONLN)
+  nProcessors=$(getconf _NPROCESSORS_ONLN)
   yq e -i '
     .retrieval.spec.logicalDump.options.parallelJobs=${postgres_dump_parallel_jobs} |
     .retrieval.spec.logicalRestore.options.parallelJobs=$nProcessors

instance.tf

@@ -49,7 +49,6 @@ resource "aws_instance" "aws_ec2" {
     dle_verification_token = "${random_string.dle_verification_token.result}"
     dle_debug_mode = "${var.dle_debug_mode}"
     dle_retrieval_refresh_timetable = "${var.dle_retrieval_refresh_timetable}"
-    dle_disks = "${join(" ",var.aws_deploy_ec2_volumes_names)}"
     dle_version = "${var.dle_version}"
     joe_version = "${var.joe_version}"
     aws_deploy_dns_zone_name = "${var.aws_deploy_dns_zone_name}"

terraform.tfvars

@@ -9,7 +9,7 @@ aws_deploy_ec2_instance_type = "c5.large"
 aws_deploy_ec2_instance_tag_name = "DBLABserver-ec2instance"
 aws_deploy_ebs_size = "10"
 aws_deploy_ebs_type = "gp2"
-aws_deploy_ec2_volumes_names = ["/dev/xvdf", "/dev/xvdg",]
+aws_deploy_ec2_volumes_count = "2"
 aws_deploy_allow_ssh_from_cidrs = ["0.0.0.0/0"]
 aws_deploy_dns_api_subdomain = "tf-test" # subdomain in aws.postgres.ai, fqdn will be ${dns_api_subdomain}.aws.postgres.ai
 
@@ -29,3 +29,6 @@ platform_project_name = "aws_test_tf"
 # have them placed to authorized_keys. Instead of ssh_public_keys_files_list,
 # it is possible to use ssh_public_keys_list containing public keys as text values.
 ssh_public_keys_files_list = ["~/.ssh/id_rsa.pub"]
+ssh_public_keys_list = [
+"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDhbblazDXCFEc21DtFzprWC8DiqidnVRROzp6J6BeJR9+XydPUtl0Rt2mcNvxL5ro5bI9u5JRW8aDd6s+Orpr66hEDdwQTbT1wp5nyduFQcT3rR+aeDSilQvAHjr4/z/GZ6IgZ5MICSIh5hJJagHoxAVqeS9dCA27tv/n2T2XrxIUeBhywH1EmfwrnEw97tHM8F+yegayFDI1nVOUWUIxFMaygMygix8uKbQ2fl4rkkxG2oEx7uyAFMXHt4bewNbZuAp8b/b5ODL6tGHuHhcwfbWGriCO+l7UOf1K9maTx00o4wkzAPyd+qs70y/1iMX2YOOLYaYYdptEnFal2DVoD example@example.com"
+]

variables.tf

@@ -77,18 +77,9 @@ variable "aws_deploy_ebs_type" {
    default = "gp2"
 }
 
-# If we need to have more data disks, this array has to be extended.
-# TODO: change logic – user sets the number of disks only, not thinking about names
-variable "aws_deploy_ec2_volumes_names" {
-  description = "List of paths for EBS volumes mounts"
-  # This list is of "non-nitro" instances. For "nitro" ones,
-  # the real disk names will be different and in fact these names 
-  # will be ignored. However, we still need to pass something here
-  # to proceed with the disk attachment.
-  default = [
-    "/dev/xvdf",
-    "/dev/xvdg",
-  ]
+variable "aws_deploy_ec2_volumes_count" {
+  description = "Number (from 1 to 22) of EBS volumes attached to EC2 to create ZFS pools"
+  default = "2"
 }
 
 variable "source_postgres_dbname" {
@@ -192,3 +183,35 @@ variable "ssh_public_keys_list"{
  description = "List of ssh public keys to copy to the provisioned instance with DLE"
  default = []
 }
+
+variable "aws_deploy_ec2_ebs_volumes_names" {
+  description = "List of paths for EBS volumes mounts"
+  # This list is of "non-nitro" instances. For "nitro" ones,
+  # the real disk names will be different and in fact these names
+  # will be ignored. However, we still need to pass something here
+  # to proceed with the disk attachment.
+  default = [
+    "/dev/xvde",
+    "/dev/xvdf",
+    "/dev/xvdg",
+    "/dev/xvdh",
+    "/dev/xvdi",
+    "/dev/xvdj",
+    "/dev/xvdk",
+    "/dev/xvdl",
+    "/dev/xvdm",
+    "/dev/xvdn",
+    "/dev/xvdo",
+    "/dev/xvdp",
+    "/dev/xvdq",
+    "/dev/xvdr",
+    "/dev/xvds",
+    "/dev/xvdt",
+    "/dev/xvdu",
+    "/dev/xvdv",
+    "/dev/xvdw",
+    "/dev/xvdx",
+    "/dev/xvdy",
+    "/dev/xvdz",
+  ]
+}

volumes.tf

@@ -1,13 +1,13 @@
 resource "aws_volume_attachment" "ebs_att" {
-  count = "${length(tolist(var.aws_deploy_ec2_volumes_names))}"
-  device_name = "${element(var.aws_deploy_ec2_volumes_names, count.index)}"
+  count = "${var.aws_deploy_ec2_volumes_count}"
+  device_name = "${element(var.aws_deploy_ec2_ebs_volumes_names, count.index)}"
   volume_id = "${element(aws_ebs_volume.DLEVolume.*.id, count.index)}"
   instance_id = "${aws_instance.aws_ec2.id}"
   force_detach = true
 }
 
 resource "aws_ebs_volume" "DLEVolume" {
-  count = "${length(tolist(var.aws_deploy_ec2_volumes_names))}"
+  count = "${var.aws_deploy_ec2_volumes_count}"
   availability_zone = "${var.aws_deploy_ebs_availability_zone}"
   encrypted = "${var.aws_deploy_ebs_encrypted}"
   size  = "${var.aws_deploy_ebs_size}"