Wip traefik switch

.gitignore

@@ -16,3 +16,7 @@ docker-compose/nginx/conf.d/default.conf
 docker-compose/keycloak/providers/
 
 docker-compose/backups/
+
+
+#Ignore possible real environnement file
+docker-compose/.env

README.md

@@ -56,7 +56,7 @@ Docker Compose profiles control which services start:
 | `frontend` | reverse-proxy, crm-frontend | Web serving layer |
 | `init` | keycloak-config, swagger-doc-gen, generation-doc-gen, simulation-doc-gen, news-board-doc-gen, billing-doc-gen, krakend-config, crm-frontend-config | One-shot config generators |
 | `migration` | optimce-migrator | One-shot CRM schema migrations |
-| `backup` | crm-database-backup, keycloak-db-backup, allocation-key-db-backup, simulation-key-db-backup, news-board-db-backup, billing-db-backup | Database backup services |
+| `backup` | database backup services, backup-upload | Database dumps and S3-compatible backup upload |
 
 Default startup uses `backend` + `frontend` profiles.
 
@@ -70,16 +70,23 @@ docker compose --profile backend --profile migration run --rm optimce-migrator
 
 ## Automatic Backups
 
-Backups run automatically before `stop` or `restart`, one dump per database:
+Backups run automatically before `stop` or `restart`. Each database is dumped locally,
+uploaded to the configured external S3-compatible bucket, verified, and then removed
+from the VPS:
 - CRM → `backups/crm_db_YYYYMMDD_HHMMSS.sql`
 - Keycloak → `backups/keycloak_YYYYMMDD_HHMMSS.sql`
 - Allocation-key → `backups/allocation_key_YYYYMMDD_HHMMSS.sql`
 - Simulation-key → `backups/simulation_key_YYYYMMDD_HHMMSS.sql`
 - News-board → `backups/news_board_YYYYMMDD_HHMMSS.sql`
 - Billing → `backups/billing_YYYYMMDD_HHMMSS.sql`
 
-A failing dump is reported but does not abort the others or block the shutdown.
-Retention cleanup is not currently implemented; old backup files must be removed manually.
+A failing dump or upload prevents shutdown and retains the local files for retry.
+The uploader uses `rclone copy`, never `sync`, so local cleanup cannot delete remote
+backups. Remote retention must be configured through the S3 provider lifecycle rules.
+
+Configure `BACKUP_S3_*` in `docker-compose/.env`. The endpoint is provider-independent
+and supports AWS S3 and other S3-compatible services. Use a dedicated bucket or prefix
+and credentials restricted to that bucket/prefix.
 
 Manual backup:
 ```bash
@@ -89,9 +96,11 @@ docker compose -f docker-compose/docker-compose.yml --profile backup run --rm al
 docker compose -f docker-compose/docker-compose.yml --profile backup run --rm simulation-key-db-backup
 docker compose -f docker-compose/docker-compose.yml --profile backup run --rm news-board-db-backup
 docker compose -f docker-compose/docker-compose.yml --profile backup run --rm billing-db-backup
+docker compose -f docker-compose/docker-compose.yml --profile backup --env-file docker-compose/.env run --rm backup-upload
 ```
 
-Backups are stored in `docker-compose/backups/`.
+Manual database dumps must be followed by `backup-upload` to upload, verify, and clear
+`docker-compose/backups/`. Failed uploads leave the local files untouched.
 
 ## Configuration
 

docker-compose/.env → docker-compose/.env.example

@@ -74,29 +74,20 @@ MINIO_PUBLIC_ENDPOINT=http://s3.localhost
 TEMPLATES_BUCKET=optimce-templates
 OUTPUT_BUCKET=optimce-documents
 
+# ============================================
+# BACKUP PROFILE - External S3-compatible storage
+# ============================================
+# Use an S3-compatible endpoint. Keep this separate from application storage.
+BACKUP_S3_ENDPOINT=https://s3.example.com
+BACKUP_S3_REGION=us-east-1
+BACKUP_S3_BUCKET=optimce-database-backups
+BACKUP_S3_ACCESS_KEY=changeme_backup_access_key
+BACKUP_S3_SECRET_KEY=changeme_backup_secret_key
+BACKUP_S3_PREFIX=production
+BACKUP_S3_FORCE_PATH_STYLE=false
+
 # ============================================
 # BACKEND PROFILE - OpenTelemetry
 # ============================================
 REMOTE_LOGGING=false
 OTEL_EXPORTER_OTLP_ENDPOINT=none
-
-# ============================================
-# FRONTEND PROFILE - Nginx upstream hosts
-# ============================================
-CRM_FRONTEND_UPSTREAM=crm-frontend:80
-KRAKEND_UPSTREAM=krakend:8080
-KEYCLOAK_UPSTREAM=keycloak:8080
-MINIO_UPSTREAM=minio:9000
-
-# Protocols for upstream proxies
-CRM_FRONTEND_PROTOCOL=http
-KRAKEND_PROTOCOL=http
-KEYCLOAK_PROTOCOL=http
-MINIO_PROTOCOL=http
-
-# ============================================
-# FRONTEND PROFILE - SSL/HTTPS
-# ============================================
-USE_HTTPS=false
-SSL_CERT_PATH=/etc/nginx/certs/live/${DOMAIN_HOST}/fullchain.pem
-SSL_CERT_KEY_PATH=/etc/nginx/certs/live/${DOMAIN_HOST}/privkey.pem

docker-compose/docker-compose.yml

@@ -96,25 +96,6 @@ services:
     command:
       - < /config/prod-config.template.json > /config/prod-config.json
 
-  nginx-config:
-    profiles: ["init"]
-    image: ghcr.io/ourouk/alpine-envsubst:main
-    env_file:
-      - .env
-    volumes:
-      - ./nginx/conf-template.d:/config
-      - ./nginx/conf.d:/output
-    entrypoint: /bin/sh
-    command:
-      - "-c"
-      - |
-        if [ "${USE_HTTPS}" = "true" ]; then
-          cp /config/default-https.template.conf /tmp/template.conf;
-        else
-          cp /config/default-http.template.conf /tmp/template.conf;
-        fi
-        envsubst < /tmp/template.conf | sed 's/§/$/g' > /output/default.conf
-
   crm-frontend-config:
     profiles: ["init"]
     image: ghcr.io/ourouk/alpine-envsubst:main
@@ -369,6 +350,13 @@ services:
       - keycloak
       - api-gateway
       - reverse-proxy
+    labels:
+      traefik.enable: "true"
+      traefik.http.routers.keycloak.rule: "Host(`${DOMAIN_HOST}`) && PathPrefix(`/keycloak`)"
+      traefik.http.routers.keycloak.entrypoints: "websecure"
+      traefik.http.routers.keycloak.tls.certresolver: "letsencrypt"
+      traefik.http.routers.keycloak.service: "keycloak"
+      traefik.http.services.keycloak.loadbalancer.server.port: "8080"
     depends_on:
       keycloak-db:
         condition: service_healthy
@@ -409,6 +397,13 @@ services:
     networks:
       - reverse-proxy
       - minio
+    labels:
+      traefik.enable: "true"
+      traefik.http.routers.minio.rule: "Host(`s3.${DOMAIN_HOST}`)"
+      traefik.http.routers.minio.entrypoints: "websecure"
+      traefik.http.routers.minio.tls.certresolver: "letsencrypt"
+      traefik.http.routers.minio.service: "minio"
+      traefik.http.services.minio.loadbalancer.server.port: "9000"
     healthcheck:
       test: ["CMD", "mc", "ready", "local"]
       interval: 10s
@@ -451,6 +446,15 @@ services:
     networks:
       - api-gateway
       - reverse-proxy
+    labels:
+      traefik.enable: "true"
+      traefik.http.routers.krakend.rule: "Host(`${DOMAIN_HOST}`) && PathPrefix(`/api`)"
+      traefik.http.routers.krakend.entrypoints: "websecure"
+      traefik.http.routers.krakend.tls.certresolver: "letsencrypt"
+      traefik.http.routers.krakend.service: "krakend"
+      traefik.http.routers.krakend.middlewares: "krakend-stripprefix"
+      traefik.http.services.krakend.loadbalancer.server.port: "8080"
+      traefik.http.middlewares.krakend-stripprefix.stripprefix.prefixes: "/api"
     depends_on:
       crm-backend:
         condition: service_started
@@ -832,32 +836,35 @@ services:
   # ============================================
   # FRONTEND PROFILE - Web serving
   # ============================================
-  reverse-proxy:
-    profiles: ["frontend"]
-    image: nginx:latest
+  traefik:
+    profiles: ["frontend", "backend"]
+    image: traefik:v3
+    restart: unless-stopped
     ports:
       - "80:80"
       - "443:443"
+    command:
+      - "--providers.docker=true"
+      - "--providers.docker.exposedByDefault=false"
+      - "--providers.docker.network=reverse-proxy"
+      - "--providers.file.directory=/etc/traefik/dynamic"
+      - "--providers.file.watch=true"
+      - "--entryPoints.web.address=:80"
+      - "--entryPoints.websecure.address=:443"
+      - "--entryPoints.websecure.http.middlewares=security-headers@file"
+      - "--entryPoints.web.http.redirections.entryPoint.to=websecure"
+      - "--entryPoints.web.http.redirections.entryPoint.scheme=https"
+      - "--certificatesResolvers.letsencrypt.acme.httpChallenge=true"
+      - "--certificatesResolvers.letsencrypt.acme.httpChallenge.entryPoint=web"
+      - "--certificatesResolvers.letsencrypt.acme.email=${SSL_EMAIL}"
+      - "--certificatesResolvers.letsencrypt.acme.storage=/letsencrypt/acme.json"
+      - "--accesslog=true"
     volumes:
-      - ./nginx/conf.d:/etc/nginx/conf.d
-      - ./nginx/certs:/etc/nginx/certs
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./traefik/letsencrypt:/letsencrypt
+      - ./traefik/dynamic:/etc/traefik/dynamic:ro
     networks:
       - reverse-proxy
-      - minio
-      - api-gateway
-      - keycloak
-      - crm
-
-  certbot:
-    profiles: ["frontend"]
-    image: certbot/certbot:latest
-    volumes:
-      - ./nginx/certs:/etc/letsencrypt
-      - ./nginx/certbot-webroot:/webroot
-    command: certonly --webroot -w /webroot -d ${DOMAIN_HOST} --agree-tos -m ${SSL_EMAIL} --non-interactive
-    environment:
-      - DOMAIN_HOST=${DOMAIN_HOST}
-      - SSL_EMAIL=${SSL_EMAIL}
 
   crm-frontend:
     profiles: ["frontend"]
@@ -866,6 +873,17 @@ services:
       - ./crm-frontend-config/config.json:/usr/share/nginx/html/assets/config/config.json
     networks:
       - reverse-proxy
+    labels:
+      traefik.enable: "true"
+      traefik.http.routers.crm-frontend.rule: "Host(`${DOMAIN_HOST}`)"
+      traefik.http.routers.crm-frontend.entrypoints: "websecure"
+      traefik.http.routers.crm-frontend.tls.certresolver: "letsencrypt"
+      traefik.http.routers.crm-frontend.service: "crm-frontend"
+      traefik.http.routers.crm-frontend.middlewares: "spa-fallback"
+      traefik.http.services.crm-frontend.loadbalancer.server.port: "80"
+      traefik.http.middlewares.spa-fallback.errors.status: "404"
+      traefik.http.middlewares.spa-fallback.errors.service: "crm-frontend"
+      traefik.http.middlewares.spa-fallback.errors.query: "/index.html"
 
   # ============================================
   # BACKUP PROFILE - Database backups
@@ -879,7 +897,12 @@ services:
       - PGPASSWORD=${CRM_DB_PASSWORD}
     command: >
       sh -c '
-        pg_dump -h crm-database -U postgres -d crm_db -f /backups/crm_db_$(date +%Y%m%d_%H%M%S).sql ;
+        timestamp=$$(date +%Y%m%d_%H%M%S);
+        tmp=/backups/crm_db_$${timestamp}.sql.tmp;
+        output=/backups/crm_db_$${timestamp}.sql;
+        trap "rm -f \"$$tmp\"" EXIT;
+        pg_dump -h crm-database -U postgres -d crm_db -f "$$tmp";
+        mv "$$tmp" "$$output";
       '
     networks:
       - crm
@@ -893,7 +916,12 @@ services:
       - PGPASSWORD=${KEYCLOAK_DB_PASSWORD}
     command: >
       sh -c '
-        pg_dump -h keycloak-db -U postgres -d keycloak -f /backups/keycloak_$(date +%Y%m%d_%H%M%S).sql ;
+        timestamp=$$(date +%Y%m%d_%H%M%S);
+        tmp=/backups/keycloak_$${timestamp}.sql.tmp;
+        output=/backups/keycloak_$${timestamp}.sql;
+        trap "rm -f \"$$tmp\"" EXIT;
+        pg_dump -h keycloak-db -U postgres -d keycloak -f "$$tmp";
+        mv "$$tmp" "$$output";
       '
     networks:
       - keycloak
@@ -907,7 +935,12 @@ services:
       - PGPASSWORD=${ALLOCATION_KEY_DB_PASSWORD}
     command: >
       sh -c '
-        pg_dump -h allocation-key-db -U postgres -d allocation_key_local -f /backups/allocation_key_$(date +%Y%m%d_%H%M%S).sql ;
+        timestamp=$$(date +%Y%m%d_%H%M%S);
+        tmp=/backups/allocation_key_$${timestamp}.sql.tmp;
+        output=/backups/allocation_key_$${timestamp}.sql;
+        trap "rm -f \"$$tmp\"" EXIT;
+        pg_dump -h allocation-key-db -U postgres -d allocation_key_local -f "$$tmp";
+        mv "$$tmp" "$$output";
       '
     networks:
       - allocation-key
@@ -921,7 +954,12 @@ services:
       - PGPASSWORD=${SIMULATION_KEY_DB_PASSWORD}
     command: >
       sh -c '
-        pg_dump -h simulation-key-db -U postgres -d simulation_key_local -f /backups/simulation_key_$(date +%Y%m%d_%H%M%S).sql ;
+        timestamp=$$(date +%Y%m%d_%H%M%S);
+        tmp=/backups/simulation_key_$${timestamp}.sql.tmp;
+        output=/backups/simulation_key_$${timestamp}.sql;
+        trap "rm -f \"$$tmp\"" EXIT;
+        pg_dump -h simulation-key-db -U postgres -d simulation_key_local -f "$$tmp";
+        mv "$$tmp" "$$output";
       '
     networks:
       - simulation-key
@@ -935,7 +973,12 @@ services:
       - PGPASSWORD=${NEWS_BOARD_DB_PASSWORD}
     command: >
       sh -c '
-        pg_dump -h news-board-db -U postgres -d news_board_local -f /backups/news_board_$(date +%Y%m%d_%H%M%S).sql ;
+        timestamp=$$(date +%Y%m%d_%H%M%S);
+        tmp=/backups/news_board_$${timestamp}.sql.tmp;
+        output=/backups/news_board_$${timestamp}.sql;
+        trap "rm -f \"$$tmp\"" EXIT;
+        pg_dump -h news-board-db -U postgres -d news_board_local -f "$$tmp";
+        mv "$$tmp" "$$output";
       '
     networks:
       - news-board
@@ -949,11 +992,42 @@ services:
       - PGPASSWORD=${BILLING_DB_PASSWORD}
     command: >
       sh -c '
-        pg_dump -h billing-db -U postgres -d billing_local -f /backups/billing_$(date +%Y%m%d_%H%M%S).sql ;
+        timestamp=$$(date +%Y%m%d_%H%M%S);
+        tmp=/backups/billing_$${timestamp}.sql.tmp;
+        output=/backups/billing_$${timestamp}.sql;
+        trap "rm -f \"$$tmp\"" EXIT;
+        pg_dump -h billing-db -U postgres -d billing_local -f "$$tmp";
+        mv "$$tmp" "$$output";
       '
     networks:
       - billing
 
+  backup-upload:
+    profiles: ["backup"]
+    image: rclone/rclone:latest
+    volumes:
+      - ./backups:/backups
+    environment:
+      - RCLONE_CONFIG_BACKUP_TYPE=s3
+      - RCLONE_CONFIG_BACKUP_PROVIDER=Other
+      - RCLONE_CONFIG_BACKUP_ENDPOINT=${BACKUP_S3_ENDPOINT}
+      - RCLONE_CONFIG_BACKUP_REGION=${BACKUP_S3_REGION}
+      - RCLONE_CONFIG_BACKUP_ACCESS_KEY_ID=${BACKUP_S3_ACCESS_KEY}
+      - RCLONE_CONFIG_BACKUP_SECRET_ACCESS_KEY=${BACKUP_S3_SECRET_KEY}
+      - RCLONE_CONFIG_BACKUP_FORCE_PATH_STYLE=${BACKUP_S3_FORCE_PATH_STYLE}
+      - BACKUP_S3_BUCKET=${BACKUP_S3_BUCKET}
+      - BACKUP_S3_PREFIX=${BACKUP_S3_PREFIX}
+    command: >
+      sh -c '
+        set -eu;
+        rclone copy /backups "backup:$${BACKUP_S3_BUCKET}/$${BACKUP_S3_PREFIX}" --include "*.sql";
+        rclone check /backups "backup:$${BACKUP_S3_BUCKET}/$${BACKUP_S3_PREFIX}" --include "*.sql" --one-way;
+        for file in /backups/*.sql; do
+          [ -e "$$file" ] || continue;
+          rm -- "$$file";
+        done;
+      '
+
 volumes:
   crm_db_data:
   keycloak_db_data:
@@ -963,8 +1037,6 @@ volumes:
   billing_db_data:
   minio_data:
   nats_data:
-  krakend_config:
-  keycloak_data:
 
 networks:
   reverse-proxy: