diff --git a/.gitignore b/.gitignore index 61532b6..c0c16a4 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,7 @@ docker-compose/nginx/conf.d/default.conf docker-compose/keycloak/providers/ docker-compose/backups/ + + +#Ignore possible real environnement file +docker-compose/.env \ No newline at end of file diff --git a/README.md b/README.md index aaea25a..dcf40d1 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Docker Compose profiles control which services start: | `frontend` | reverse-proxy, crm-frontend | Web serving layer | | `init` | keycloak-config, swagger-doc-gen, generation-doc-gen, simulation-doc-gen, news-board-doc-gen, billing-doc-gen, krakend-config, crm-frontend-config | One-shot config generators | | `migration` | optimce-migrator | One-shot CRM schema migrations | -| `backup` | crm-database-backup, keycloak-db-backup, allocation-key-db-backup, simulation-key-db-backup, news-board-db-backup, billing-db-backup | Database backup services | +| `backup` | database backup services, backup-upload | Database dumps and S3-compatible backup upload | Default startup uses `backend` + `frontend` profiles. @@ -70,7 +70,9 @@ docker compose --profile backend --profile migration run --rm optimce-migrator ## Automatic Backups -Backups run automatically before `stop` or `restart`, one dump per database: +Backups run automatically before `stop` or `restart`. Each database is dumped locally, +uploaded to the configured external S3-compatible bucket, verified, and then removed +from the VPS: - CRM → `backups/crm_db_YYYYMMDD_HHMMSS.sql` - Keycloak → `backups/keycloak_YYYYMMDD_HHMMSS.sql` - Allocation-key → `backups/allocation_key_YYYYMMDD_HHMMSS.sql` @@ -78,8 +80,13 @@ Backups run automatically before `stop` or `restart`, one dump per database: - News-board → `backups/news_board_YYYYMMDD_HHMMSS.sql` - Billing → `backups/billing_YYYYMMDD_HHMMSS.sql` -A failing dump is reported but does not abort the others or block the shutdown. -Retention cleanup is not currently implemented; old backup files must be removed manually. +A failing dump or upload prevents shutdown and retains the local files for retry. +The uploader uses `rclone copy`, never `sync`, so local cleanup cannot delete remote +backups. Remote retention must be configured through the S3 provider lifecycle rules. + +Configure `BACKUP_S3_*` in `docker-compose/.env`. The endpoint is provider-independent +and supports AWS S3 and other S3-compatible services. Use a dedicated bucket or prefix +and credentials restricted to that bucket/prefix. Manual backup: ```bash @@ -89,9 +96,11 @@ docker compose -f docker-compose/docker-compose.yml --profile backup run --rm al docker compose -f docker-compose/docker-compose.yml --profile backup run --rm simulation-key-db-backup docker compose -f docker-compose/docker-compose.yml --profile backup run --rm news-board-db-backup docker compose -f docker-compose/docker-compose.yml --profile backup run --rm billing-db-backup +docker compose -f docker-compose/docker-compose.yml --profile backup --env-file docker-compose/.env run --rm backup-upload ``` -Backups are stored in `docker-compose/backups/`. +Manual database dumps must be followed by `backup-upload` to upload, verify, and clear +`docker-compose/backups/`. Failed uploads leave the local files untouched. ## Configuration diff --git a/docker-compose/.env b/docker-compose/.env.example similarity index 84% rename from docker-compose/.env rename to docker-compose/.env.example index a4f8ade..d093bf9 100644 --- a/docker-compose/.env +++ b/docker-compose/.env.example @@ -74,29 +74,20 @@ MINIO_PUBLIC_ENDPOINT=http://s3.localhost TEMPLATES_BUCKET=optimce-templates OUTPUT_BUCKET=optimce-documents +# ============================================ +# BACKUP PROFILE - External S3-compatible storage +# ============================================ +# Use an S3-compatible endpoint. Keep this separate from application storage. +BACKUP_S3_ENDPOINT=https://s3.example.com +BACKUP_S3_REGION=us-east-1 +BACKUP_S3_BUCKET=optimce-database-backups +BACKUP_S3_ACCESS_KEY=changeme_backup_access_key +BACKUP_S3_SECRET_KEY=changeme_backup_secret_key +BACKUP_S3_PREFIX=production +BACKUP_S3_FORCE_PATH_STYLE=false + # ============================================ # BACKEND PROFILE - OpenTelemetry # ============================================ REMOTE_LOGGING=false OTEL_EXPORTER_OTLP_ENDPOINT=none - -# ============================================ -# FRONTEND PROFILE - Nginx upstream hosts -# ============================================ -CRM_FRONTEND_UPSTREAM=crm-frontend:80 -KRAKEND_UPSTREAM=krakend:8080 -KEYCLOAK_UPSTREAM=keycloak:8080 -MINIO_UPSTREAM=minio:9000 - -# Protocols for upstream proxies -CRM_FRONTEND_PROTOCOL=http -KRAKEND_PROTOCOL=http -KEYCLOAK_PROTOCOL=http -MINIO_PROTOCOL=http - -# ============================================ -# FRONTEND PROFILE - SSL/HTTPS -# ============================================ -USE_HTTPS=false -SSL_CERT_PATH=/etc/nginx/certs/live/${DOMAIN_HOST}/fullchain.pem -SSL_CERT_KEY_PATH=/etc/nginx/certs/live/${DOMAIN_HOST}/privkey.pem diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 731dedc..fb7fd6e 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -96,25 +96,6 @@ services: command: - < /config/prod-config.template.json > /config/prod-config.json - nginx-config: - profiles: ["init"] - image: ghcr.io/ourouk/alpine-envsubst:main - env_file: - - .env - volumes: - - ./nginx/conf-template.d:/config - - ./nginx/conf.d:/output - entrypoint: /bin/sh - command: - - "-c" - - | - if [ "${USE_HTTPS}" = "true" ]; then - cp /config/default-https.template.conf /tmp/template.conf; - else - cp /config/default-http.template.conf /tmp/template.conf; - fi - envsubst < /tmp/template.conf | sed 's/§/$/g' > /output/default.conf - crm-frontend-config: profiles: ["init"] image: ghcr.io/ourouk/alpine-envsubst:main @@ -369,6 +350,13 @@ services: - keycloak - api-gateway - reverse-proxy + labels: + traefik.enable: "true" + traefik.http.routers.keycloak.rule: "Host(`${DOMAIN_HOST}`) && PathPrefix(`/keycloak`)" + traefik.http.routers.keycloak.entrypoints: "websecure" + traefik.http.routers.keycloak.tls.certresolver: "letsencrypt" + traefik.http.routers.keycloak.service: "keycloak" + traefik.http.services.keycloak.loadbalancer.server.port: "8080" depends_on: keycloak-db: condition: service_healthy @@ -409,6 +397,13 @@ services: networks: - reverse-proxy - minio + labels: + traefik.enable: "true" + traefik.http.routers.minio.rule: "Host(`s3.${DOMAIN_HOST}`)" + traefik.http.routers.minio.entrypoints: "websecure" + traefik.http.routers.minio.tls.certresolver: "letsencrypt" + traefik.http.routers.minio.service: "minio" + traefik.http.services.minio.loadbalancer.server.port: "9000" healthcheck: test: ["CMD", "mc", "ready", "local"] interval: 10s @@ -451,6 +446,15 @@ services: networks: - api-gateway - reverse-proxy + labels: + traefik.enable: "true" + traefik.http.routers.krakend.rule: "Host(`${DOMAIN_HOST}`) && PathPrefix(`/api`)" + traefik.http.routers.krakend.entrypoints: "websecure" + traefik.http.routers.krakend.tls.certresolver: "letsencrypt" + traefik.http.routers.krakend.service: "krakend" + traefik.http.routers.krakend.middlewares: "krakend-stripprefix" + traefik.http.services.krakend.loadbalancer.server.port: "8080" + traefik.http.middlewares.krakend-stripprefix.stripprefix.prefixes: "/api" depends_on: crm-backend: condition: service_started @@ -832,32 +836,35 @@ services: # ============================================ # FRONTEND PROFILE - Web serving # ============================================ - reverse-proxy: - profiles: ["frontend"] - image: nginx:latest + traefik: + profiles: ["frontend", "backend"] + image: traefik:v3 + restart: unless-stopped ports: - "80:80" - "443:443" + command: + - "--providers.docker=true" + - "--providers.docker.exposedByDefault=false" + - "--providers.docker.network=reverse-proxy" + - "--providers.file.directory=/etc/traefik/dynamic" + - "--providers.file.watch=true" + - "--entryPoints.web.address=:80" + - "--entryPoints.websecure.address=:443" + - "--entryPoints.websecure.http.middlewares=security-headers@file" + - "--entryPoints.web.http.redirections.entryPoint.to=websecure" + - "--entryPoints.web.http.redirections.entryPoint.scheme=https" + - "--certificatesResolvers.letsencrypt.acme.httpChallenge=true" + - "--certificatesResolvers.letsencrypt.acme.httpChallenge.entryPoint=web" + - "--certificatesResolvers.letsencrypt.acme.email=${SSL_EMAIL}" + - "--certificatesResolvers.letsencrypt.acme.storage=/letsencrypt/acme.json" + - "--accesslog=true" volumes: - - ./nginx/conf.d:/etc/nginx/conf.d - - ./nginx/certs:/etc/nginx/certs + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./traefik/letsencrypt:/letsencrypt + - ./traefik/dynamic:/etc/traefik/dynamic:ro networks: - reverse-proxy - - minio - - api-gateway - - keycloak - - crm - - certbot: - profiles: ["frontend"] - image: certbot/certbot:latest - volumes: - - ./nginx/certs:/etc/letsencrypt - - ./nginx/certbot-webroot:/webroot - command: certonly --webroot -w /webroot -d ${DOMAIN_HOST} --agree-tos -m ${SSL_EMAIL} --non-interactive - environment: - - DOMAIN_HOST=${DOMAIN_HOST} - - SSL_EMAIL=${SSL_EMAIL} crm-frontend: profiles: ["frontend"] @@ -866,6 +873,17 @@ services: - ./crm-frontend-config/config.json:/usr/share/nginx/html/assets/config/config.json networks: - reverse-proxy + labels: + traefik.enable: "true" + traefik.http.routers.crm-frontend.rule: "Host(`${DOMAIN_HOST}`)" + traefik.http.routers.crm-frontend.entrypoints: "websecure" + traefik.http.routers.crm-frontend.tls.certresolver: "letsencrypt" + traefik.http.routers.crm-frontend.service: "crm-frontend" + traefik.http.routers.crm-frontend.middlewares: "spa-fallback" + traefik.http.services.crm-frontend.loadbalancer.server.port: "80" + traefik.http.middlewares.spa-fallback.errors.status: "404" + traefik.http.middlewares.spa-fallback.errors.service: "crm-frontend" + traefik.http.middlewares.spa-fallback.errors.query: "/index.html" # ============================================ # BACKUP PROFILE - Database backups @@ -879,7 +897,12 @@ services: - PGPASSWORD=${CRM_DB_PASSWORD} command: > sh -c ' - pg_dump -h crm-database -U postgres -d crm_db -f /backups/crm_db_$(date +%Y%m%d_%H%M%S).sql ; + timestamp=$$(date +%Y%m%d_%H%M%S); + tmp=/backups/crm_db_$${timestamp}.sql.tmp; + output=/backups/crm_db_$${timestamp}.sql; + trap "rm -f \"$$tmp\"" EXIT; + pg_dump -h crm-database -U postgres -d crm_db -f "$$tmp"; + mv "$$tmp" "$$output"; ' networks: - crm @@ -893,7 +916,12 @@ services: - PGPASSWORD=${KEYCLOAK_DB_PASSWORD} command: > sh -c ' - pg_dump -h keycloak-db -U postgres -d keycloak -f /backups/keycloak_$(date +%Y%m%d_%H%M%S).sql ; + timestamp=$$(date +%Y%m%d_%H%M%S); + tmp=/backups/keycloak_$${timestamp}.sql.tmp; + output=/backups/keycloak_$${timestamp}.sql; + trap "rm -f \"$$tmp\"" EXIT; + pg_dump -h keycloak-db -U postgres -d keycloak -f "$$tmp"; + mv "$$tmp" "$$output"; ' networks: - keycloak @@ -907,7 +935,12 @@ services: - PGPASSWORD=${ALLOCATION_KEY_DB_PASSWORD} command: > sh -c ' - pg_dump -h allocation-key-db -U postgres -d allocation_key_local -f /backups/allocation_key_$(date +%Y%m%d_%H%M%S).sql ; + timestamp=$$(date +%Y%m%d_%H%M%S); + tmp=/backups/allocation_key_$${timestamp}.sql.tmp; + output=/backups/allocation_key_$${timestamp}.sql; + trap "rm -f \"$$tmp\"" EXIT; + pg_dump -h allocation-key-db -U postgres -d allocation_key_local -f "$$tmp"; + mv "$$tmp" "$$output"; ' networks: - allocation-key @@ -921,7 +954,12 @@ services: - PGPASSWORD=${SIMULATION_KEY_DB_PASSWORD} command: > sh -c ' - pg_dump -h simulation-key-db -U postgres -d simulation_key_local -f /backups/simulation_key_$(date +%Y%m%d_%H%M%S).sql ; + timestamp=$$(date +%Y%m%d_%H%M%S); + tmp=/backups/simulation_key_$${timestamp}.sql.tmp; + output=/backups/simulation_key_$${timestamp}.sql; + trap "rm -f \"$$tmp\"" EXIT; + pg_dump -h simulation-key-db -U postgres -d simulation_key_local -f "$$tmp"; + mv "$$tmp" "$$output"; ' networks: - simulation-key @@ -935,7 +973,12 @@ services: - PGPASSWORD=${NEWS_BOARD_DB_PASSWORD} command: > sh -c ' - pg_dump -h news-board-db -U postgres -d news_board_local -f /backups/news_board_$(date +%Y%m%d_%H%M%S).sql ; + timestamp=$$(date +%Y%m%d_%H%M%S); + tmp=/backups/news_board_$${timestamp}.sql.tmp; + output=/backups/news_board_$${timestamp}.sql; + trap "rm -f \"$$tmp\"" EXIT; + pg_dump -h news-board-db -U postgres -d news_board_local -f "$$tmp"; + mv "$$tmp" "$$output"; ' networks: - news-board @@ -949,11 +992,42 @@ services: - PGPASSWORD=${BILLING_DB_PASSWORD} command: > sh -c ' - pg_dump -h billing-db -U postgres -d billing_local -f /backups/billing_$(date +%Y%m%d_%H%M%S).sql ; + timestamp=$$(date +%Y%m%d_%H%M%S); + tmp=/backups/billing_$${timestamp}.sql.tmp; + output=/backups/billing_$${timestamp}.sql; + trap "rm -f \"$$tmp\"" EXIT; + pg_dump -h billing-db -U postgres -d billing_local -f "$$tmp"; + mv "$$tmp" "$$output"; ' networks: - billing + backup-upload: + profiles: ["backup"] + image: rclone/rclone:latest + volumes: + - ./backups:/backups + environment: + - RCLONE_CONFIG_BACKUP_TYPE=s3 + - RCLONE_CONFIG_BACKUP_PROVIDER=Other + - RCLONE_CONFIG_BACKUP_ENDPOINT=${BACKUP_S3_ENDPOINT} + - RCLONE_CONFIG_BACKUP_REGION=${BACKUP_S3_REGION} + - RCLONE_CONFIG_BACKUP_ACCESS_KEY_ID=${BACKUP_S3_ACCESS_KEY} + - RCLONE_CONFIG_BACKUP_SECRET_ACCESS_KEY=${BACKUP_S3_SECRET_KEY} + - RCLONE_CONFIG_BACKUP_FORCE_PATH_STYLE=${BACKUP_S3_FORCE_PATH_STYLE} + - BACKUP_S3_BUCKET=${BACKUP_S3_BUCKET} + - BACKUP_S3_PREFIX=${BACKUP_S3_PREFIX} + command: > + sh -c ' + set -eu; + rclone copy /backups "backup:$${BACKUP_S3_BUCKET}/$${BACKUP_S3_PREFIX}" --include "*.sql"; + rclone check /backups "backup:$${BACKUP_S3_BUCKET}/$${BACKUP_S3_PREFIX}" --include "*.sql" --one-way; + for file in /backups/*.sql; do + [ -e "$$file" ] || continue; + rm -- "$$file"; + done; + ' + volumes: crm_db_data: keycloak_db_data: @@ -963,8 +1037,6 @@ volumes: billing_db_data: minio_data: nats_data: - krakend_config: - keycloak_data: networks: reverse-proxy: diff --git a/docker-compose/nginx/certbot.ini b/docker-compose/nginx/certbot.ini deleted file mode 100644 index 8e68ad5..0000000 --- a/docker-compose/nginx/certbot.ini +++ /dev/null @@ -1,4 +0,0 @@ -[main] -authenticator = webroot -webroot-path = /webroot -logs-dir = /var/log/letsencrypt \ No newline at end of file diff --git a/docker-compose/nginx/conf-template.d/default-http.template.conf b/docker-compose/nginx/conf-template.d/default-http.template.conf deleted file mode 100644 index a8964c5..0000000 --- a/docker-compose/nginx/conf-template.d/default-http.template.conf +++ /dev/null @@ -1,80 +0,0 @@ -upstream crm-frontend { - server ${CRM_FRONTEND_UPSTREAM}; -} - -upstream krakend { - server ${KRAKEND_UPSTREAM}; -} - -upstream keycloak { - server ${KEYCLOAK_UPSTREAM}; -} -upstream minio{ - server ${MINIO_UPSTREAM}; -} - -map §http_upgrade §connection_upgrade { - default "upgrade"; - '' "close"; -} - -server { - listen 80; - server_name ${DOMAIN_HOST}; - - location / { - proxy_pass ${CRM_FRONTEND_PROTOCOL}://crm-frontend; - include /etc/nginx/conf.d/proxy-common-headers.conf; - proxy_set_header Upgrade §http_upgrade; - proxy_set_header Connection §connection_upgrade; - proxy_intercept_errors on; - error_page 404 = @angular_spa_fallback; - } - - location @angular_spa_fallback { - if (§request_uri ~* "\.[A-Za-z0-9]{2,8}(§|\?)") { - return 404; - } - - rewrite ^ /index.html break; - proxy_pass ${CRM_FRONTEND_PROTOCOL}://crm-frontend; - include /etc/nginx/conf.d/proxy-common-headers.conf; - } - - location = /api { - return 301 /api/; - } - - location /api/ { - proxy_pass ${KRAKEND_PROTOCOL}://krakend/; - include /etc/nginx/conf.d/proxy-common-headers.conf; - proxy_set_header X-Forwarded-Host §host; - proxy_set_header X-Forwarded-Port §server_port; - proxy_set_header X-Forwarded-Prefix /api; - } - - location = /keycloak { - return 301 /keycloak/; - } - - location /keycloak/ { - proxy_pass ${KEYCLOAK_PROTOCOL}://keycloak; - include /etc/nginx/conf.d/proxy-common-headers.conf; - proxy_set_header X-Forwarded-Host §host; - proxy_set_header X-Forwarded-Port §server_port; - proxy_set_header Forwarded "host=§host:§server_port;proto=§scheme"; - } - -} - -server { - listen 80; - server_name ~^s3\.; - - location / { - proxy_pass ${MINIO_PROTOCOL}://minio; - include /etc/nginx/conf.d/proxy-common-headers.conf; - proxy_set_header X-Forwarded-Host §host; - proxy_set_header X-Forwarded-Proto §scheme; - } -} \ No newline at end of file diff --git a/docker-compose/nginx/conf-template.d/default-https.template.conf b/docker-compose/nginx/conf-template.d/default-https.template.conf deleted file mode 100644 index 4bc579a..0000000 --- a/docker-compose/nginx/conf-template.d/default-https.template.conf +++ /dev/null @@ -1,107 +0,0 @@ -upstream crm-frontend { - server ${CRM_FRONTEND_UPSTREAM}; -} - -upstream krakend { - server ${KRAKEND_UPSTREAM}; -} - -upstream keycloak { - server ${KEYCLOAK_UPSTREAM}; -} -upstream minio{ - server ${MINIO_UPSTREAM}; -} - -map §http_upgrade §connection_upgrade { - default "upgrade"; - '' "close"; -} - -server { - listen 80; - server_name ${DOMAIN_HOST}; - location /.well-known/acme-challenge/ { - root /var/www/certbot; - } - location / { - return 301 https://§host§request_uri; - } -} - -server { - listen 80; - server_name s3.${DOMAIN_HOST}; - location /.well-known/acme-challenge/ { - root /var/www/certbot; - } - location / { - return 301 https://§host§request_uri; - } -} - -server { - listen 443 ssl; - server_name ${DOMAIN_HOST}; - - ssl_certificate ${SSL_CERT_PATH}; - ssl_certificate_key ${SSL_CERT_KEY_PATH}; - - location / { - proxy_pass ${CRM_FRONTEND_PROTOCOL}://crm-frontend; - include /etc/nginx/conf.d/proxy-common-headers.conf; - proxy_set_header Upgrade §http_upgrade; - proxy_set_header Connection §connection_upgrade; - proxy_intercept_errors on; - error_page 404 = @angular_spa_fallback; - } - - location @angular_spa_fallback { - if (§request_uri ~* "\.[A-Za-z0-9]{2,8}(§|\?)") { - return 404; - } - - rewrite ^ /index.html break; - proxy_pass ${CRM_FRONTEND_PROTOCOL}://crm-frontend; - include /etc/nginx/conf.d/proxy-common-headers.conf; - } - - location = /api { - return 301 /api/; - } - - location /api/ { - proxy_pass ${KRAKEND_PROTOCOL}://krakend/; - include /etc/nginx/conf.d/proxy-common-headers.conf; - proxy_set_header X-Forwarded-Host §host; - proxy_set_header X-Forwarded-Port §server_port; - proxy_set_header X-Forwarded-Prefix /api; - } - - location = /keycloak { - return 301 /keycloak/; - } - - location /keycloak/ { - proxy_pass ${KEYCLOAK_PROTOCOL}://keycloak; - include /etc/nginx/conf.d/proxy-common-headers.conf; - proxy_set_header X-Forwarded-Host §host; - proxy_set_header X-Forwarded-Port §server_port; - proxy_set_header Forwarded "host=§host:§server_port;proto=§scheme"; - } -} - -server { - listen 443 ssl; - server_name s3.${DOMAIN_HOST}; - - ssl_certificate ${SSL_CERT_PATH}; - ssl_certificate_key ${SSL_CERT_KEY_PATH}; - - location / { - proxy_pass ${MINIO_PROTOCOL}://minio; - include /etc/nginx/conf.d/proxy-common-headers.conf; - proxy_set_header X-Forwarded-Host §host; - proxy_set_header X-Forwarded-Proto §scheme; - } -} \ No newline at end of file diff --git a/docker-compose/nginx/conf.d/proxy-common-headers.conf b/docker-compose/nginx/conf.d/proxy-common-headers.conf deleted file mode 100644 index ac31526..0000000 --- a/docker-compose/nginx/conf.d/proxy-common-headers.conf +++ /dev/null @@ -1,5 +0,0 @@ -proxy_http_version 1.1; -proxy_set_header Host $host; -proxy_set_header X-Real-IP $remote_addr; -proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; -proxy_set_header X-Forwarded-Proto $scheme; diff --git a/docker-compose/traefik/dynamic/conf.yml b/docker-compose/traefik/dynamic/conf.yml new file mode 100644 index 0000000..661cbd8 --- /dev/null +++ b/docker-compose/traefik/dynamic/conf.yml @@ -0,0 +1,11 @@ +http: + middlewares: + security-headers: + headers: + stsSeconds: 31536000 + stsIncludeSubdomains: true + stsPreload: true + frameDeny: true + contentTypeNosniff: true + referrerPolicy: "strict-origin-when-cross-origin" + permissionsPolicy: "camera=(), microphone=(), geolocation=()" diff --git a/docker-compose/traefik/letsencrypt/acme.json b/docker-compose/traefik/letsencrypt/acme.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/docker-compose/traefik/letsencrypt/acme.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/docker-stack.bat b/docker-stack.bat index 423d6dc..7ecca6e 100644 --- a/docker-stack.bat +++ b/docker-stack.bat @@ -121,14 +121,29 @@ call :do_backup exit /b %errorlevel% :do_backup -rem One entry per database in the backup profile. A failure is reported but does not -rem abort the remaining dumps, so one broken annex cannot block the shutdown. +rem Run every dump, but do not upload or delete anything if one dump fails. +set "BACKUP_FAILED=0" for %%J in (crm-database-backup keycloak-db-backup allocation-key-db-backup simulation-key-db-backup news-board-db-backup billing-db-backup) do ( echo Running %%J... %DOCKER_COMPOSE_CMD% -f "%COMPOSE_FILE%" --env-file "%ENV_FILE%" --profile backup run --rm %%J - if errorlevel 1 echo %%J failed + if errorlevel 1 ( + echo %%J failed + set "BACKUP_FAILED=1" + ) ) -echo Backups completed. + +if "%BACKUP_FAILED%"=="1" ( + echo At least one backup failed; local files were retained. + exit /b 1 +) + +echo Uploading and verifying backups... +%DOCKER_COMPOSE_CMD% -f "%COMPOSE_FILE%" --env-file "%ENV_FILE%" --profile backup run --rm backup-upload +if errorlevel 1 ( + echo Backup upload or verification failed; local files were retained. + exit /b 1 +) +echo Backups uploaded, verified, and removed locally. exit /b 0 :parse_start_options @@ -159,4 +174,4 @@ if /i "%~1"=="--wait-backend" ( goto :parse_start_options ) echo Unknown option: %~1 -exit /b 1 \ No newline at end of file +exit /b 1 diff --git a/docker-stack.sh b/docker-stack.sh index 03fe4d5..3b23032 100755 --- a/docker-stack.sh +++ b/docker-stack.sh @@ -80,15 +80,26 @@ stop_stack() { } do_backup() { - # One entry per database in the backup profile. A failure is reported but does - # not abort the remaining dumps, so one broken annex cannot block the shutdown. + # Run every dump, but do not upload or delete anything if one dump fails. local jobs="crm-database-backup keycloak-db-backup allocation-key-db-backup simulation-key-db-backup news-board-db-backup billing-db-backup" + local failed=0 for job in $jobs; do echo "Running ${job}..." - compose -f "$COMPOSE_FILE" --profile backup --env-file "$ENV_FILE" run --rm "$job" || echo "${job} failed" + if ! compose -f "$COMPOSE_FILE" --profile backup --env-file "$ENV_FILE" run --rm "$job"; then + echo "${job} failed" + failed=1 + fi done - echo "Backups completed." + + if [ "$failed" -ne 0 ]; then + echo "At least one backup failed; local files were retained." + return 1 + fi + + echo "Uploading and verifying backups..." + compose -f "$COMPOSE_FILE" --profile backup --env-file "$ENV_FILE" run --rm backup-upload + echo "Backups uploaded, verified, and removed locally." } parse_start_options() { @@ -152,4 +163,4 @@ main() { esac } -main "$@" \ No newline at end of file +main "$@"