Skip to content

Commit fd03be6

Browse files
agneumagneum
committed
Merge branch '278-config-structure' into 'master'
refactor: redefine configuration structure for all Database Lab products (database-lab#278) See merge request postgres-ai/terraform-postgres-ai-database-lab!32
2 parents feb7a71 + 4fd5a48 commit fd03be6

File tree

4 files changed

+89
-62
lines changed

4 files changed

+89
-62
lines changed

‎dle-logical-init.sh.tpl‎

Lines changed: 77 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -100,44 +100,50 @@ for i in $${!disks[@]}; do
100100
done
101101

102102
# Adjust DLE config
103-
mkdir -p ~/.dblab/postgres_conf/
104-
105-
curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version_full}/configs/config.example.logical_generic.yml --output ~/.dblab/server.yml
106-
curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version_full}/configs/postgres/pg_hba.conf \
107-
--output ~/.dblab/postgres_conf/pg_hba.conf
108-
curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version_full}/configs/postgres/postgresql.conf --output ~/.dblab/postgres_conf/postgresql.conf
109-
cat /tmp/postgresql_clones_custom.conf >> ~/.dblab/postgres_conf/postgresql.conf
110-
111-
sed -ri "s/^(\s*)(debug:.*$)/\1debug: ${dle_debug_mode}/" ~/.dblab/server.yml
112-
sed -ri "s/^(\s*)(verificationToken:.*$)/\1verificationToken: ${dle_verification_token}/" ~/.dblab/server.yml
113-
sed -ri "s/^(\s*)(timetable:.*$)/\1timetable: \"${dle_retrieval_refresh_timetable}\"/" ~/.dblab/server.yml
114-
sed -ri "s/^(\s*)(forceInit:.*$)/\1forceInit: true/" ~/.dblab/server.yml
115-
sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${source_postgres_dbname}/" ~/.dblab/server.yml
103+
dle_config_path="/home/ubuntu/.dblab/engine/configs"
104+
dle_meta_path="/home/ubuntu/.dblab/engine/meta"
105+
postgres_conf_path="/home/ubuntu/.dblab/postgres_conf"
106+
107+
mkdir -p $dle_config_path
108+
mkdir -p $dle_meta_path
109+
mkdir -p $postgres_conf_path
110+
111+
curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/configs/config.example.logical_generic.yml --output $dle_config_path/server.yml
112+
curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/configs/standard/postgres/control/pg_hba.conf \
113+
--output $postgres_conf_path/pg_hba.conf
114+
curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/configs/standard/postgres/control/postgresql.conf --output $postgres_conf_path/postgresql.conf
115+
cat /tmp/postgresql_clones_custom.conf >> $postgres_conf_path/postgresql.conf
116+
117+
sed -ri "s/^(\s*)(debug:.*$)/\1debug: ${dle_debug_mode}/" $dle_config_path/server.yml
118+
sed -ri "s/^(\s*)(verificationToken:.*$)/\1verificationToken: ${dle_verification_token}/" $dle_config_path/server.yml
119+
sed -ri "s/^(\s*)(timetable:.*$)/\1timetable: \"${dle_retrieval_refresh_timetable}\"/" $dle_config_path/server.yml
120+
sed -ri "s/^(\s*)(forceInit:.*$)/\1forceInit: true/" $dle_config_path/server.yml
121+
sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${source_postgres_dbname}/" $dle_config_path/server.yml
116122
# Enable Platform
117-
sed -ri "s/^(\s*)(#platform:$)/\1platform: /" ~/.dblab/server.yml
118-
sed -ri "s/^(\s*)(# url: \"https\\:\\/\\/postgres.ai\\/api\\/general\"$)/\1 url: \"https\\:\\/\\/postgres.ai\\/api\\/general\" /" ~/.dblab/server.yml
119-
sed -ri "s/^(\s*)(# accessToken: \"platform_access_token\"$)/\1 accessToken: \"${platform_access_token}\"/" ~/.dblab/server.yml
120-
sed -ri "s/^(\s*)(# enablePersonalTokens: true$)/\1 enablePersonalTokens: true/" ~/.dblab/server.yml
121-
sed -ri "s/:13/:${source_postgres_version}/g" ~/.dblab/server.yml
123+
sed -ri "s/^(\s*)(#platform:$)/\1platform: /" $dle_config_path/server.yml
124+
sed -ri "s/^(\s*)(# url: \"https\\:\\/\\/postgres.ai\\/api\\/general\"$)/\1 url: \"https\\:\\/\\/postgres.ai\\/api\\/general\" /" $dle_config_path/server.yml
125+
sed -ri "s/^(\s*)(# accessToken: \"platform_access_token\"$)/\1 accessToken: \"${platform_access_token}\"/" $dle_config_path/server.yml
126+
sed -ri "s/^(\s*)(# enablePersonalTokens: true$)/\1 enablePersonalTokens: true/" $dle_config_path/server.yml
127+
sed -ri "s/:13/:${source_postgres_version}/g" $dle_config_path/server.yml
122128

123129
case "${source_type}" in
124130

125131
postgres)
126132
# Mount directory to store dump files.
127133
extra_mount="--volume /var/lib/dblab/dblab_pool_00/dump:/var/lib/dblab/dblab_pool/dump"
128134

129-
sed -ri "s/^(\s*)(host: 34.56.78.90$)/\1host: ${source_postgres_host}/" ~/.dblab/server.yml
130-
sed -ri "s/^(\s*)(port: 5432$)/\1port: ${source_postgres_port}/" ~/.dblab/server.yml
131-
sed -ri "s/^(\s*)( username: postgres$)/\1 username: ${source_postgres_username}/" ~/.dblab/server.yml
132-
sed -ri "s/^(\s*)(password:.*$)/\1password: ${source_postgres_password}/" ~/.dblab/server.yml
135+
sed -ri "s/^(\s*)(host: 34.56.78.90$)/\1host: ${source_postgres_host}/" $dle_config_path/server.yml
136+
sed -ri "s/^(\s*)(port: 5432$)/\1port: ${source_postgres_port}/" $dle_config_path/server.yml
137+
sed -ri "s/^(\s*)( username: postgres$)/\1 username: ${source_postgres_username}/" $dle_config_path/server.yml
138+
sed -ri "s/^(\s*)(password:.*$)/\1password: ${source_postgres_password}/" $dle_config_path/server.yml
133139
# restore pg_dump via pipe - without saving it on the disk
134-
sed -ri "s/^(\s*)(parallelJobs:.*$)/\1parallelJobs: 1/" ~/.dblab/server.yml
135-
sed -ri "s/^(\s*)(# immediateRestore:.*$)/\1immediateRestore: /" ~/.dblab/server.yml
136-
sed -ri "s/^(\s*)(# forceInit: false.*$)/\1 forceInit: true /" ~/.dblab/server.yml
137-
sed -ri "s/^(\s*)( # configs:$)/\1 configs: /" ~/.dblab/server.yml
138-
sed -ri "s/^(\s*)( # shared_preload_libraries: .*$)/\1 shared_preload_libraries: '${postgres_config_shared_preload_libraries}'/" ~/.dblab/server.yml
139-
sed -ri "s/^(\s*)( shared_preload_libraries:.*$)/\1 shared_preload_libraries: '${postgres_config_shared_preload_libraries}'/" ~/.dblab/server.yml
140-
sed -ri "s/^(\s*)(- logicalRestore.*$)/\1#- logicalRestore /" ~/.dblab/server.yml
140+
sed -ri "s/^(\s*)(parallelJobs:.*$)/\1parallelJobs: 1/" $dle_config_path/server.yml
141+
sed -ri "s/^(\s*)(# immediateRestore:.*$)/\1immediateRestore: /" $dle_config_path/server.yml
142+
sed -ri "s/^(\s*)(# forceInit: false.*$)/\1 forceInit: true /" $dle_config_path/server.yml
143+
sed -ri "s/^(\s*)( # configs:$)/\1 configs: /" $dle_config_path/server.yml
144+
sed -ri "s/^(\s*)( # shared_preload_libraries: .*$)/\1 shared_preload_libraries: '${postgres_config_shared_preload_libraries}'/" $dle_config_path/server.yml
145+
sed -ri "s/^(\s*)( shared_preload_libraries:.*$)/\1 shared_preload_libraries: '${postgres_config_shared_preload_libraries}'/" $dle_config_path/server.yml
146+
sed -ri "s/^(\s*)(- logicalRestore.*$)/\1#- logicalRestore /" $dle_config_path/server.yml
141147
;;
142148

143149
s3)
@@ -146,11 +152,11 @@ case "${source_type}" in
146152
s3fs ${source_pgdump_s3_bucket} ${source_pgdump_s3_mount_point} -o iam_role -o allow_other
147153

148154
extra_mount="--volume ${source_pgdump_s3_mount_point}:${source_pgdump_s3_mount_point}"
149-
150-
sed -ri "s/^(\s*)(- logicalDump.*$)/\1#- logicalDump /" ~/.dblab/server.yml
151-
sed -ri "s|^(\s*)( dumpLocation:.*$)|\1 dumpLocation: ${source_pgdump_s3_mount_point}/${source_pgdump_path_on_s3_bucket}|" ~/.dblab/server.yml
152-
sed -ri '/is always single-threaded./{n;s/.*/ parallelJobs: '${postgres_dump_parallel_jobs}'/}' ~/.dblab/server.yml
153-
sed -ri '/jobs to restore faster./{n;s/.*/ parallelJobs: '$(getconf _NPROCESSORS_ONLN)'/}' ~/.dblab/server.yml
155+
156+
sed -ri "s/^(\s*)(- logicalDump.*$)/\1#- logicalDump /" $dle_config_path/server.yml
157+
sed -ri "s|^(\s*)( dumpLocation:.*$)|\1 dumpLocation: ${source_pgdump_s3_mount_point}/${source_pgdump_path_on_s3_bucket}|" $dle_config_path/server.yml
158+
sed -ri '/is always single-threaded./{n;s/.*/ parallelJobs: '${postgres_dump_parallel_jobs}'/}' $dle_config_path/server.yml
159+
sed -ri '/jobs to restore faster./{n;s/.*/ parallelJobs: '$(getconf _NPROCESSORS_ONLN)'/}' $dle_config_path/server.yml
154160
;;
155161

156162
esac
@@ -162,21 +168,22 @@ sudo docker run \
162168
--publish 2345:2345 \
163169
--volume /var/run/docker.sock:/var/run/docker.sock \
164170
--volume /var/lib/dblab:/var/lib/dblab/:rshared \
165-
--volume ~/.dblab/server.yml:/home/dblab/configs/config.yml \
166-
--volume /root/.dblab/postgres_conf:/home/dblab/configs/postgres \
171+
--volume $dle_config_path:/home/dblab/configs:ro \
172+
--volume $dle_meta_path:/home/dblab/meta \
173+
--volume $postgres_conf_path:/home/dblab/standard/postgres/control \
167174
$extra_mount \
168175
--env DOCKER_API_VERSION=1.39 \
169176
--detach \
170177
--restart on-failure \
171-
registry.gitlab.com/postgres-ai/database-lab/dblab-server:${dle_version_full}
178+
registry.gitlab.com/postgres-ai/database-lab/dblab-server:${dle_version}
172179

173180
### Waiting for the Database Lab Engine initialization.
174181
for i in {1..30000}; do
175182
curl http://localhost:2345 > /dev/null 2>&1 && break || echo "dblab is not ready yet"
176183
sleep 10
177184
done
178185

179-
curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version_full}/scripts/cli_install.sh | bash
186+
curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/scripts/cli_install.sh | bash
180187
sudo mv ~/.dblab/dblab /usr/local/bin/dblab
181188
dblab init \
182189
--environment-id=tutorial \
@@ -185,31 +192,44 @@ dblab init \
185192
--insecure
186193

187194
# Configure and run Joe Bot container.
188-
cp /home/ubuntu/joe.yml ~/.dblab/joe.yml
189-
sed -ri "s/^(\s*)(debug:.*$)/\1debug: ${dle_debug_mode}/" ~/.dblab/joe.yml
190-
sed -ri "s/^(\s*)( token:.*$)/\1 token: ${platform_access_token}/" ~/.dblab/joe.yml
191-
sed -ri "s/^(\s*)( token:.*$)/\1 token: ${dle_verification_token}/" ~/.dblab/joe.yml
192-
sed -ri "s/^(\s*)( url:.*$)/\1 url: \"http\\:\\/\\/localhost\\:2345\"/" ~/.dblab/joe.yml
193-
sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${source_postgres_dbname}/" ~/.dblab/joe.yml
194-
sed -ri "s/^(\s*)(signingSecret:.*$)/\1signingSecret: ${platform_joe_signing_secret}/" ~/.dblab/joe.yml
195-
sed -ri "s/^(\s*)(project:.*$)/\1project: ${platform_project_name}/" ~/.dblab/joe.yml
195+
joe_config_path="/home/ubuntu/.dblab/joe/configs"
196+
joe_meta_path="/home/ubuntu/.dblab/joe/meta"
197+
198+
mkdir -p $joe_config_path
199+
mkdir -p $joe_meta_path
200+
201+
# Copy configuration file from Packer-baked image.
202+
cp /home/ubuntu/joe.yml $joe_config_path/joe.yml
203+
204+
sed -ri "s/^(\s*)(debug:.*$)/\1debug: ${dle_debug_mode}/" $joe_config_path/joe.yml
205+
sed -ri "s/^(\s*)( token:.*$)/\1 token: ${platform_access_token}/" $joe_config_path/joe.yml
206+
sed -ri "s/^(\s*)( token:.*$)/\1 token: ${dle_verification_token}/" $joe_config_path/joe.yml
207+
sed -ri "s/^(\s*)( url:.*$)/\1 url: \"http\\:\\/\\/localhost\\:2345\"/" $joe_config_path/joe.yml
208+
sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${source_postgres_dbname}/" $joe_config_path/joe.yml
209+
sed -ri "s/^(\s*)(signingSecret:.*$)/\1signingSecret: ${platform_joe_signing_secret}/" $joe_config_path/joe.yml
210+
sed -ri "s/^(\s*)(project:.*$)/\1project: ${platform_project_name}/" $joe_config_path/joe.yml
196211

197212
sudo docker run \
198213
--name joe_bot \
199214
--network=host \
200215
--restart=on-failure \
201-
--volume ~/.dblab/joe.yml:/home/config/config.yml \
216+
--volume $joe_config_path:/home/configs:ro \
217+
--volume $joe_meta_path:/home/meta \
202218
--detach \
203-
postgresai/joe:latest
219+
registry.gitlab.com/postgres-ai/joe:${joe_version}
204220

205221
# Configure and run DB Migration Checker.
206-
curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version_full}/configs/config.example.run_ci.yaml --output ~/.dblab/run_ci.yaml
207-
sed -ri "s/^(\s*)(debug:.*$)/\1debug: ${dle_debug_mode}/" ~/.dblab/run_ci.yaml
208-
sed -ri "s/^(\s*)( verificationToken: \"secret_token\".*$)/\1 verificationToken: ${vcs_db_migration_checker_verification_token}/" ~/.dblab/run_ci.yaml
209-
sed -ri "s/^(\s*)( url: \"https\\:\\/\\/dblab.domain.com\"$)/\1 url: \"http\\:\\/\\/dblab_server\\:2345\"/" ~/.dblab/run_ci.yaml
210-
sed -ri "s/^(\s*)( verificationToken: \"checker_secret_token\".*$)/\1 verificationToken: ${dle_verification_token}/" ~/.dblab/run_ci.yaml
211-
sed -ri "s/^(\s*)( accessToken:.*$)/\1 accessToken: ${platform_access_token}/" ~/.dblab/run_ci.yaml
212-
sed -ri "s/^(\s*)( token:.*$)/\1 token: ${vcs_github_secret_token}/" ~/.dblab/run_ci.yaml
222+
ci_checker_config_path="/home/ubuntu/.dblab/ci_checker/configs"
223+
mkdir -p $ci_checker_config_path
224+
225+
curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/configs/config.example.ci_checker.yml --output $ci_checker_config_path/ci_checker.yml
226+
227+
sed -ri "s/^(\s*)(debug:.*$)/\1debug: ${dle_debug_mode}/" $ci_checker_config_path/ci_checker.yml
228+
sed -ri "s/^(\s*)( verificationToken: \"secret_token\".*$)/\1 verificationToken: ${vcs_db_migration_checker_verification_token}/" $ci_checker_config_path/ci_checker.yml
229+
sed -ri "s/^(\s*)( url: \"https\\:\\/\\/dblab.domain.com\"$)/\1 url: \"http\\:\\/\\/dblab_server\\:2345\"/" $ci_checker_config_path/ci_checker.yml
230+
sed -ri "s/^(\s*)( verificationToken: \"checker_secret_token\".*$)/\1 verificationToken: ${dle_verification_token}/" $ci_checker_config_path/ci_checker.yml
231+
sed -ri "s/^(\s*)( accessToken:.*$)/\1 accessToken: ${platform_access_token}/" $ci_checker_config_path/ci_checker.yml
232+
sed -ri "s/^(\s*)( token:.*$)/\1 token: ${vcs_github_secret_token}/" $ci_checker_config_path/ci_checker.yml
213233

214234
sudo docker run \
215235
--name dblab_ci_checker \
@@ -219,5 +239,5 @@ sudo docker run \
219239
--publish 2500:2500 \
220240
--volume /var/run/docker.sock:/var/run/docker.sock \
221241
--volume /tmp/ci_checker:/tmp/ci_checker \
222-
--volume ~/.dblab/run_ci.yaml:/home/dblab/configs/run_ci.yaml \
223-
registry.gitlab.com/postgres-ai/database-lab/dblab-ci-checker:${dle_version_full}
242+
--volume $ci_checker_config_path:/home/dblab/configs:ro \
243+
registry.gitlab.com/postgres-ai/database-lab/dblab-ci-checker:${dle_version}

‎instance.tf‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ data "template_file" "init" {
2929
dle_debug_mode = "${var.dle_debug_mode}"
3030
dle_retrieval_refresh_timetable = "${var.dle_retrieval_refresh_timetable}"
3131
dle_disks = "${join(" ",var.aws_deploy_ec2_volumes_names)}"
32-
dle_version_full = "${var.dle_version_full}"
32+
dle_version = "${var.dle_version}"
33+
joe_version = "${var.joe_version}"
3334
aws_deploy_dns_zone_name = "${var.aws_deploy_dns_zone_name}"
3435
aws_deploy_dns_api_subdomain = "${var.aws_deploy_dns_api_subdomain}"
3536
aws_deploy_certificate_email = "${var.aws_deploy_certificate_email}"

‎terraform.tfvars‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
dle_version_full = "2.4.1"
1+
dle_version = "2.5.0"
2+
joe_version = "0.10.0"
23

34
aws_ami_name = "DBLABserver*"
45
aws_keypair = "YOUR_AWS_KEYPAIR"

‎variables.tf‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,14 @@ variable "aws_ami_name" {
33
default = "DBLABserver"
44
}
55

6-
variable "dle_version_full" {
7-
description = "3-digit DLE version (2-digit major + minor)"
8-
default = "2.4.1"
6+
variable "dle_version" {
7+
description = "Semantic DLE version (3-digits: major, minor, patch)"
8+
default = "2.5.0"
9+
}
10+
11+
variable "joe_version" {
12+
description = "Semantic Joe Bot version (3-digits: major, minor, patch)"
13+
default = "0.10.0"
914
}
1015

1116
variable "aws_deploy_region" {

0 commit comments

Comments
 (0)