Enhance compliance rules, integrations, and CI/CD pipeline

.dockerignore

@@ -0,0 +1,19 @@
+.git
+.github
+.pytest_cache
+.ruff_cache
+.venv
+venv
+**/.venv
+**/venv
+**/__pycache__
+**/node_modules
+*.py[cod]
+*.log
+.env
+.env.*
+ai/vectorstore
+coverage.xml
+.coverage
+htmlcov
+sbom.cyclonedx.json

.github/workflows/ci.yml

@@ -475,6 +475,7 @@ jobs:
         with:
           image-ref: openshield-ci-scan:${{ github.sha }}
           severity: CRITICAL,HIGH
+          ignore-unfixed: true
           exit-code: "1"
 
   # ── Backend test suite with coverage ─────────────────────────────────────
@@ -512,6 +513,11 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
 
+      - name: Apply database migrations
+        env:
+          DATABASE_URL: "postgresql://ci:ci@localhost:5432/ci_db"
+        run: alembic upgrade head
+
       - name: Run test suite with coverage
         env:
           DATABASE_URL: "postgresql://ci:ci@localhost:5432/ci_db"

.github/workflows/deploy.yml

@@ -1,12 +1,37 @@
-name: Deploy API to Render
+name: Deploy API and worker to Render
 
 on:
-  workflow_dispatch:   # allows manual trigger from GitHub UI
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Target Render environment
+        required: true
+        type: choice
+        options:
+          - staging
+          - production
+      run_smoke_tests:
+        description: Run smoke tests after deploy
+        required: true
+        default: true
+        type: boolean
+
+concurrency:
+  group: deploy-${{ inputs.environment }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+permissions:
+  id-token: write
+  contents: read
 
 jobs:
   deploy:
-    name: Deploy to Render
+    name: Deploy to Render (${{ inputs.environment }})
     runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
 
     steps:
       - name: Checkout repository
@@ -17,7 +42,6 @@ jobs:
         with:
           python-version: "3.11"
 
-      # ── Dependency caching ─────────────────────────────────────────────
       - name: Cache pip dependencies
         uses: actions/cache@v4
         with:
@@ -31,30 +55,86 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
 
-      # ── Secret check (Determines if smoke tests should run) ───────────
-      - name: Check for JWT_SECRET
-        id: check_config
-        run: |
-          if [ -n "${{ secrets.JWT_SECRET }}" ]; then
-            echo "is_configured=true" >> $GITHUB_OUTPUT
-          else
-            echo "is_configured=false" >> $GITHUB_OUTPUT
-          fi
+      # This must remain before either create step: invalid branch/environment
+      # combinations and missing configuration must result in zero Render POSTs.
+      - name: Validate deployment preflight
+        env:
+          DEPLOY_ENVIRONMENT: ${{ inputs.environment }}
+          RUN_SMOKE_TESTS: ${{ inputs.run_smoke_tests }}
+          GITHUB_REF_NAME: ${{ github.ref_name }}
+          GITHUB_SHA: ${{ github.sha }}
+          RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }}
+          RENDER_STAGING_SERVICE_ID: ${{ secrets.RENDER_STAGING_SERVICE_ID }}
+          RENDER_STAGING_WORKER_SERVICE_ID: ${{ secrets.RENDER_STAGING_WORKER_SERVICE_ID }}
+          RENDER_PRODUCTION_SERVICE_ID: ${{ secrets.RENDER_PRODUCTION_SERVICE_ID }}
+          RENDER_PRODUCTION_WORKER_SERVICE_ID: ${{ secrets.RENDER_PRODUCTION_WORKER_SERVICE_ID }}
+          STAGING_API_URL: ${{ secrets.STAGING_API_URL }}
+          PRODUCTION_API_URL: ${{ secrets.PRODUCTION_API_URL }}
+          JWT_SECRET: ${{ secrets.JWT_SECRET }}
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
+          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+        run: python scripts/render_deploy_preflight.py
+
+      # Creation and polling are separate so both exact deployment IDs are
+      # retained and independently monitored. POST creation is never retried.
+      - name: Create API deployment
+        id: create_api
+        env:
+          RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }}
+          RENDER_SERVICE_ID: ${{ env.RENDER_API_SERVICE_ID }}
+          RENDER_SERVICE_NAME: API
+          GITHUB_SHA: ${{ github.sha }}
+        run: python scripts/render_deploy.py create
+
+      - name: Create worker deployment
+        id: create_worker
+        env:
+          RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }}
+          RENDER_SERVICE_ID: ${{ env.RENDER_WORKER_SERVICE_ID }}
+          RENDER_SERVICE_NAME: worker
+          GITHUB_SHA: ${{ github.sha }}
+        run: python scripts/render_deploy.py create
 
-      # ── Wait for Render auto-deployment ────────────────────────────────
-      # Render handles the actual physical deployment when you push.
-      # We just pause the Action to let Render's servers finish building.
-      - name: Wait for app to initialise
+      - name: Wait for API deployment
+        id: wait_api
+        continue-on-error: true
+        env:
+          RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }}
+          RENDER_SERVICE_ID: ${{ env.RENDER_API_SERVICE_ID }}
+          RENDER_SERVICE_NAME: API
+          RENDER_DEPLOY_ID: ${{ steps.create_api.outputs.deploy_id }}
+          GITHUB_SHA: ${{ github.sha }}
+        run: python scripts/render_deploy.py wait
+
+      - name: Wait for worker deployment
+        id: wait_worker
+        continue-on-error: true
+        env:
+          RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }}
+          RENDER_SERVICE_ID: ${{ env.RENDER_WORKER_SERVICE_ID }}
+          RENDER_SERVICE_NAME: worker
+          RENDER_DEPLOY_ID: ${{ steps.create_worker.outputs.deploy_id }}
+          GITHUB_SHA: ${{ github.sha }}
+        run: python scripts/render_deploy.py wait
+
+      - name: Require both deployments to be live
+        if: always()
+        env:
+          API_RESULT: ${{ steps.wait_api.outcome }}
+          WORKER_RESULT: ${{ steps.wait_worker.outcome }}
+          API_DEPLOY_ID: ${{ steps.create_api.outputs.deploy_id }}
+          WORKER_DEPLOY_ID: ${{ steps.create_worker.outputs.deploy_id }}
+          GITHUB_SHA: ${{ github.sha }}
         run: |
-          echo "Waiting 120 seconds for Render to build and start the app..."
-          sleep 120
+          if [ "$API_RESULT" != "success" ] || [ "$WORKER_RESULT" != "success" ]; then
+            echo "ERROR: coordinated deployment failed for SHA $GITHUB_SHA. API deploy $API_DEPLOY_ID: $API_RESULT; worker deploy $WORKER_DEPLOY_ID: $WORKER_RESULT."
+            exit 1
+          fi
+          echo "API deploy $API_DEPLOY_ID and worker deploy $WORKER_DEPLOY_ID are live at SHA $GITHUB_SHA."
 
-      # ── Health gate ────────────────────────────────────────────────────
       - name: Health gate check
-        id: health_gate
-        env:
-          # Use secret URL if provided, otherwise fallback to default
-          API_URL: ${{ secrets.API_URL || 'https://openshield-api.onrender.com' }}
         run: |
           MAX_RETRIES=5
           RETRY_DELAY=15
@@ -64,42 +144,24 @@ jobs:
           for i in $(seq 1 $MAX_RETRIES); do
             echo "Health check attempt $i of $MAX_RETRIES..."
             HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$URL" --max-time 30) || true
-
             if [ "$HTTP_STATUS" -eq 200 ]; then
               echo "Health check passed (HTTP $HTTP_STATUS)"
               exit 0
             fi
-
-            echo "Got HTTP $HTTP_STATUS — retrying in ${RETRY_DELAY}s..."
+            echo "Got HTTP $HTTP_STATUS; retrying in ${RETRY_DELAY}s..."
             sleep $RETRY_DELAY
           done
+          echo "ERROR: Health gate failed after $MAX_RETRIES attempts on ${{ inputs.environment }}."
+          exit 1
 
-          echo "HEALTH GATE FAILED after $MAX_RETRIES attempts"
-          echo "Note: If you haven't set up Render for this fork, this is expected."
-          # Only allow failure on feature branches; fail on main/dev
-          if [[ "${{ github.ref }}" == "refs/heads/main" || "${{ github.ref }}" == "refs/heads/dev" ]]; then
-            echo "ERROR: Health check failed on protected branch. Deployment verification required."
-            exit 1
-          else
-            echo "Allowing health check failure on feature branch (infra may not be set up)"
-            exit 0
-          fi
-
-      # ── Smoke tests ────────────────────────────────────────────────────
       - name: Run smoke tests against live deployment
-        if: steps.check_config.outputs.is_configured == 'true' || github.event_name == 'workflow_dispatch'
+        if: inputs.run_smoke_tests
         env:
-          API_URL: ${{ secrets.API_URL || 'https://openshield-api.onrender.com' }}
           JWT_SECRET: ${{ secrets.JWT_SECRET }}
           AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
-          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
           RUN_REAL_SCAN: "true"
         run: |
-          if [[ "${{ github.ref }}" == "refs/heads/main" && -z "${{ secrets.JWT_SECRET }}" ]]; then
-            echo "ERROR: Cannot run smoke tests on main branch without JWT_SECRET configured"
-            exit 1
-          fi
           echo "Running smoke tests against: $API_URL"
-          python tests/smoke_test.py
+          python tests/smoke_test.py

.github/workflows/terraform-plan.yml

@@ -0,0 +1,62 @@
+name: Terraform Plan
+
+on:
+  pull_request:
+    paths:
+      - "infra/terraform/**"
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  plan:
+    name: Terraform fmt / validate / plan
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: infra/terraform
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+
+      - name: Set up Terraform
+        uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
+        with:
+          terraform_version: "~1.6"
+          terraform_wrapper: false
+
+      - name: Terraform fmt
+        run: terraform fmt -check -recursive
+
+      - name: Check for Terraform Cloud token
+        id: check_token
+        run: |
+          if [ -n "${{ secrets.TF_API_TOKEN }}" ]; then
+            echo "has_token=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_token=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Terraform validate (no credentials)
+        if: steps.check_token.outputs.has_token == 'false'
+        run: |
+          echo "TF_API_TOKEN not configured — skipping real plan, running structural validation only."
+          echo 'terraform {' > override.tf
+          echo '  backend "local" {}' >> override.tf
+          echo '}' >> override.tf
+          terraform init -backend=false
+          terraform validate
+          rm override.tf
+
+      - name: Terraform init (real backend)
+        if: steps.check_token.outputs.has_token == 'true'
+        env:
+          TF_TOKEN_app_terraform_io: ${{ secrets.TF_API_TOKEN }}
+        run: terraform init
+
+      - name: Terraform plan
+        if: steps.check_token.outputs.has_token == 'true'
+        env:
+          TF_TOKEN_app_terraform_io: ${{ secrets.TF_API_TOKEN }}
+        run: terraform plan -no-color

CONTRIBUTING.md

@@ -213,22 +213,29 @@ Most list methods return an empty list on failure. Methods that fetch one resour
 ```bash
 # Python 3.10+
 pip install -r requirements.txt
-# Installs Flask, Azure SDK clients, requests, psycopg2, PyJWT, and PyYAML for CI workflow validation.
+# Installs Flask, Alembic, Azure SDK clients, requests, psycopg2, PyJWT, and PyYAML for CI workflow validation.
 
 # Frontend
 # The frontend directory is currently a scaffold. The React dashboard MVP is on the roadmap.
 
-# API
-FLASK_APP=api/app.py flask run --debug
-
 # Database (Docker)
 docker run --name openshield-db \
   -e POSTGRES_USER=openshield \
   -e POSTGRES_PASSWORD=openshield \
   -e POSTGRES_DB=openshield \
   -p 5432:5432 -d postgres
+
+export DATABASE_URL=postgresql://openshield:openshield@localhost:5432/openshield
+alembic upgrade head
+
+# API
+FLASK_APP=api/app.py flask run --debug
 ```
 
+See [Database Migrations](docs/database-migrations.md) before creating or applying
+a schema change. Migration revisions are written explicitly because OpenShield
+does not use ORM metadata.
+
 ---
 
 ## Code Standards

Dockerfile

@@ -1,9 +1,12 @@
-FROM python:3.11-slim-bookworm
+FROM python:3.11-slim-trixie
 
 WORKDIR /app
 
 COPY requirements.txt .
-RUN pip install --no-cache-dir --upgrade pip && \
+RUN pip install --no-cache-dir --upgrade \
+        pip==26.1.2 \
+        setuptools==83.0.0 \
+        wheel==0.46.3 && \
     pip install --no-cache-dir -r requirements.txt
 
 COPY . .

README.md

@@ -162,6 +162,10 @@ export AZURE_CLIENT_ID=your-client-id
 export AZURE_CLIENT_SECRET=your-client-secret
 export AZURE_TENANT_ID=your-tenant-id
 export JWT_SECRET=your-strong-secret   # used to protect write endpoints (scan trigger, AI)
+export DATABASE_URL=postgresql://openshield:openshield@localhost:5432/openshield
+
+# Create or update the database schema
+alembic upgrade head
 
 # Run a scan
 python -c "
@@ -175,6 +179,9 @@ print(json.dumps(result, indent=2))
 FLASK_APP=api/app.py flask run
 ```
 
+See [Database Migrations](docs/database-migrations.md) for schema changes and the
+one-time onboarding step required for existing production databases.
+
 **Frontend (React dashboard)**
 
 ```bash