Skip to content

Commit f0ea59f

Browse files
agneumagneum
committed
Merge branch 'configuration-processing' into 'master'
feat: replace sed instructions with the yq processor commands to make configuration adjustment more reliable in the TF-template See merge request postgres-ai/terraform-postgres-ai-database-lab!41
2 parents 4c85841 + d3f01cc commit f0ea59f

File tree

1 file changed

+61
-41
lines changed

1 file changed

+61
-41
lines changed

‎dle-logical-init.sh.tpl‎

Lines changed: 61 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -114,37 +114,45 @@ curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/configs/st
114114
curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/configs/standard/postgres/control/postgresql.conf --output $postgres_conf_path/postgresql.conf
115115
cat /tmp/postgresql_clones_custom.conf >> $postgres_conf_path/postgresql.conf
116116

117-
sed -ri "s/^(\s*)(debug:.*$)/\1debug: ${dle_debug_mode}/" $dle_config_path/server.yml
118-
sed -ri "s/^(\s*)(verificationToken:.*$)/\1verificationToken: ${dle_verification_token}/" $dle_config_path/server.yml
119-
sed -ri "s/^(\s*)(timetable:.*$)/\1timetable: \"${dle_retrieval_refresh_timetable}\"/" $dle_config_path/server.yml
120-
sed -ri "s/^(\s*)(forceInit:.*$)/\1forceInit: true/" $dle_config_path/server.yml
121-
sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${source_postgres_dbname}/" $dle_config_path/server.yml
117+
yq e -i '
118+
.global.debug=${dle_debug_mode} |
119+
.server.verificationToken="${dle_verification_token}" |
120+
.retrieval.refresh.timetable="${dle_retrieval_refresh_timetable}" |
121+
.retrieval.spec.logicalDump.options.source.connection.dbname="${source_postgres_dbname}" |
122+
.retrieval.spec.logicalRestore.options.forceInit=true |
123+
.databaseContainer.dockerImage="postgresai/extended-postgres:${source_postgres_version}"
124+
' $dle_config_path/server.yml
125+
122126
# Enable Platform
123-
sed -ri "s/^(\s*)(#platform:$)/\1platform: /" $dle_config_path/server.yml
124-
sed -ri "s/^(\s*)(# url: \"https\\:\\/\\/postgres.ai\\/api\\/general\"$)/\1 url: \"https\\:\\/\\/postgres.ai\\/api\\/general\" /" $dle_config_path/server.yml
125-
sed -ri "s/^(\s*)(# accessToken: \"platform_access_token\"$)/\1 accessToken: \"${platform_access_token}\"/" $dle_config_path/server.yml
126-
sed -ri "s/^(\s*)(# enablePersonalTokens: true$)/\1 enablePersonalTokens: true/" $dle_config_path/server.yml
127-
sed -ri "s/:13/:${source_postgres_version}/g" $dle_config_path/server.yml
127+
yq e -i '
128+
.platform.url = "https://postgres.ai/api/general" |
129+
.platform.accessToken = "${platform_access_token}" |
130+
.platform.enablePersonalTokens = true |
131+
' $dle_config_path/server.yml
128132

129133
case "${source_type}" in
130134

131135
postgres)
132136
# Mount directory to store dump files.
133137
extra_mount="--volume /var/lib/dblab/dblab_pool_00/dump:/var/lib/dblab/dblab_pool/dump"
134138

135-
sed -ri "s/^(\s*)(host: 34.56.78.90$)/\1host: ${source_postgres_host}/" $dle_config_path/server.yml
136-
sed -ri "s/^(\s*)(port: 5432$)/\1port: ${source_postgres_port}/" $dle_config_path/server.yml
137-
sed -ri "s/^(\s*)( username: postgres$)/\1 username: ${source_postgres_username}/" $dle_config_path/server.yml
138-
sed -ri "s/^(\s*)(password:.*$)/\1password: ${source_postgres_password}/" $dle_config_path/server.yml
139+
yq e -i '
140+
.retrieval.spec.logicalDump.options.source.connection.host = ${source_postgres_host}" |
141+
.retrieval.spec.logicalDump.options.source.connection.port = ${source_postgres_port}" |
142+
.retrieval.spec.logicalDump.options.source.connection.username = ${source_postgres_username}" |
143+
.retrieval.spec.logicalDump.options.source.connection.password = ${source_postgres_password}" |
144+
.retrieval.spec.logicalDump.options.parallelJobs = 1"
145+
' $dle_config_path/server.yml
146+
139147
# restore pg_dump via pipe - without saving it on the disk
140-
sed -ri "s/^(\s*)(parallelJobs:.*$)/\1parallelJobs: 1/" $dle_config_path/server.yml
141-
sed -ri "s/^(\s*)(# immediateRestore:.*$)/\1immediateRestore: /" $dle_config_path/server.yml
142-
sed -ri "s/^(\s*)(# enabled: true.*$)/\1 enabled: true /" $dle_config_path/server.yml
143-
sed -ri "s/^(\s*)(# forceInit: false.*$)/\1 forceInit: true /" $dle_config_path/server.yml
144-
sed -ri "s/^(\s*)( # configs:$)/\1 configs: /" $dle_config_path/server.yml
145-
sed -ri "s/^(\s*)( # shared_preload_libraries: .*$)/\1 shared_preload_libraries: '${postgres_config_shared_preload_libraries}'/" $dle_config_path/server.yml
146-
sed -ri "s/^(\s*)( shared_preload_libraries:.*$)/\1 shared_preload_libraries: '${postgres_config_shared_preload_libraries}'/" $dle_config_path/server.yml
147-
sed -ri "s/^(\s*)(- logicalRestore.*$)/\1#- logicalRestore /" $dle_config_path/server.yml
148+
yq e -i '
149+
.databaseContainer.dockerImage="postgresai/extended-postgres:${source_postgres_version}" |
150+
.retrieval.spec.logicalDump.options.immediateRestore.enabled=true |
151+
.retrieval.spec.logicalDump.options.immediateRestore.forceInit=true |
152+
.retrieval.spec.logicalDump.options.immediateRestore.configs alias = .databaseConfig |
153+
del(.retrieval.jobs[] | select(. == "logicalRestore")) |
154+
.databaseConfig.configs.shared_preload_libraries = ${postgres_config_shared_preload_libraries}
155+
' $dle_config_path/server.yml
148156
;;
149157

150158
s3)
@@ -154,10 +162,16 @@ case "${source_type}" in
154162

155163
extra_mount="--volume ${source_pgdump_s3_mount_point}:${source_pgdump_s3_mount_point}"
156164

157-
sed -ri "s/^(\s*)(- logicalDump.*$)/\1#- logicalDump /" $dle_config_path/server.yml
158-
sed -ri "s|^(\s*)( dumpLocation:.*$)|\1 dumpLocation: ${source_pgdump_s3_mount_point}/${source_pgdump_path_on_s3_bucket}|" $dle_config_path/server.yml
159-
sed -ri '/is always single-threaded./{n;s/.*/ parallelJobs: '${postgres_dump_parallel_jobs}'/}' $dle_config_path/server.yml
160-
sed -ri '/jobs to restore faster./{n;s/.*/ parallelJobs: '$(getconf _NPROCESSORS_ONLN)'/}' $dle_config_path/server.yml
165+
yq e -i '
166+
del(.retrieval.jobs[] | select(. == "logicalDump")) |
167+
.retrieval.spec.logicalRestore.options.dumpLocation="${source_pgdump_s3_mount_point}/${source_pgdump_path_on_s3_bucket}"
168+
' $dle_config_path/server.yml
169+
170+
nProcessors = $(getconf _NPROCESSORS_ONLN)
171+
yq e -i '
172+
.retrieval.spec.logicalDump.options.parallelJobs=${postgres_dump_parallel_jobs} |
173+
.retrieval.spec.logicalRestore.options.parallelJobs=$nProcessors
174+
' $dle_config_path/server.yml
161175
;;
162176

163177
esac
@@ -199,16 +213,20 @@ joe_meta_path="/home/ubuntu/.dblab/joe/meta"
199213
mkdir -p $joe_config_path
200214
mkdir -p $joe_meta_path
201215

202-
# Copy configuration file from Packer-baked image.
203-
cp /home/ubuntu/joe.yml $joe_config_path/joe.yml
216+
curl https://gitlab.com/postgres-ai/joe/-/raw/${joe_version}/configs/config.example.yml --output $joe_config_path/joe.yml
204217

205-
sed -ri "s/^(\s*)(debug:.*$)/\1debug: ${dle_debug_mode}/" $joe_config_path/joe.yml
206-
sed -ri "s/^(\s*)( token:.*$)/\1 token: ${platform_access_token}/" $joe_config_path/joe.yml
207-
sed -ri "s/^(\s*)( token:.*$)/\1 token: ${dle_verification_token}/" $joe_config_path/joe.yml
208-
sed -ri "s/^(\s*)( url:.*$)/\1 url: \"http\\:\\/\\/localhost\\:2345\"/" $joe_config_path/joe.yml
209-
sed -ri "s/^(\s*)(dbname:.*$)/\1dbname: ${source_postgres_dbname}/" $joe_config_path/joe.yml
210-
sed -ri "s/^(\s*)(signingSecret:.*$)/\1signingSecret: ${platform_joe_signing_secret}/" $joe_config_path/joe.yml
211-
sed -ri "s/^(\s*)(project:.*$)/\1project: ${platform_project_name}/" $joe_config_path/joe.yml
218+
yq e -i '
219+
.app.debug = ${dle_debug_mode} |
220+
.platform.token = "${platform_access_token}" |
221+
.channelMapping.dblabServers.prod1.token = "${dle_verification_token}" |
222+
.channelMapping.dblabServers.prod1.url = "http://localhost:2345" |
223+
.channelMapping.communicationTypes.webui[0].credentials.signingSecret = "${platform_joe_signing_secret}" |
224+
.channelMapping.communicationTypes.webui[0].channels[0].project = "${platform_project_name}" |
225+
.channelMapping.communicationTypes.webui[0].channels[0].dblabParams.dbname = "${source_postgres_dbname}" |
226+
del(.channelMapping.communicationTypes.slack) |
227+
del(.channelMapping.communicationTypes.slackrtm) |
228+
del(.channelMapping.communicationTypes.slacksm)
229+
' $joe_config_path/joe.yml
212230

213231
sudo docker run \
214232
--name joe_bot \
@@ -225,12 +243,14 @@ mkdir -p $ci_checker_config_path
225243

226244
curl https://gitlab.com/postgres-ai/database-lab/-/raw/${dle_version}/configs/config.example.ci_checker.yml --output $ci_checker_config_path/ci_checker.yml
227245

228-
sed -ri "s/^(\s*)(debug:.*$)/\1debug: ${dle_debug_mode}/" $ci_checker_config_path/ci_checker.yml
229-
sed -ri "s/^(\s*)( verificationToken: \"secret_token\".*$)/\1 verificationToken: ${vcs_db_migration_checker_verification_token}/" $ci_checker_config_path/ci_checker.yml
230-
sed -ri "s/^(\s*)( url: \"https\\:\\/\\/dblab.domain.com\"$)/\1 url: \"http\\:\\/\\/dblab_server\\:2345\"/" $ci_checker_config_path/ci_checker.yml
231-
sed -ri "s/^(\s*)( verificationToken: \"checker_secret_token\".*$)/\1 verificationToken: ${dle_verification_token}/" $ci_checker_config_path/ci_checker.yml
232-
sed -ri "s/^(\s*)( accessToken:.*$)/\1 accessToken: ${platform_access_token}/" $ci_checker_config_path/ci_checker.yml
233-
sed -ri "s/^(\s*)( token:.*$)/\1 token: ${vcs_github_secret_token}/" $ci_checker_config_path/ci_checker.yml
246+
yq e -i '
247+
.app.debug = ${dle_debug_mode} |
248+
.app.verificationToken = "${vcs_db_migration_checker_verification_token}" |
249+
.dle.url = "http://dblab_server:2345" |
250+
.dle.verificationToken = "${dle_verification_token}" |
251+
.platform.accessToken = "${platform_access_token}" |
252+
.source.token = "${vcs_github_secret_token}"
253+
' $ci_checker_config_path/ci_checker.yml
234254

235255
sudo docker run \
236256
--name dblab_ci_checker \

0 commit comments

Comments
 (0)