diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..81599e4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +.git +.github +.pytest_cache +.ruff_cache +.venv +venv +**/.venv +**/venv +**/__pycache__ +**/node_modules +*.py[cod] +*.log +.env +.env.* +ai/vectorstore +coverage.xml +.coverage +htmlcov +sbom.cyclonedx.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de43685..1b4a0b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -475,6 +475,7 @@ jobs: with: image-ref: openshield-ci-scan:${{ github.sha }} severity: CRITICAL,HIGH + ignore-unfixed: true exit-code: "1" # ── Backend test suite with coverage ───────────────────────────────────── @@ -512,6 +513,11 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt + - name: Apply database migrations + env: + DATABASE_URL: "postgresql://ci:ci@localhost:5432/ci_db" + run: alembic upgrade head + - name: Run test suite with coverage env: DATABASE_URL: "postgresql://ci:ci@localhost:5432/ci_db" diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5d63a23..61af1ce 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,12 +1,37 @@ -name: Deploy API to Render +name: Deploy API and worker to Render on: - workflow_dispatch: # allows manual trigger from GitHub UI + workflow_dispatch: + inputs: + environment: + description: Target Render environment + required: true + type: choice + options: + - staging + - production + run_smoke_tests: + description: Run smoke tests after deploy + required: true + default: true + type: boolean + +concurrency: + group: deploy-${{ inputs.environment }} + cancel-in-progress: false + +permissions: + contents: read + +permissions: + id-token: write + contents: read jobs: deploy: - name: Deploy to Render + name: Deploy to Render (${{ inputs.environment }}) runs-on: ubuntu-latest + environment: ${{ inputs.environment }} steps: - name: Checkout repository @@ -17,7 +42,6 @@ jobs: with: python-version: "3.11" - # ── Dependency caching ───────────────────────────────────────────── - name: Cache pip dependencies uses: actions/cache@v4 with: @@ -31,30 +55,86 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt - # ── Secret check (Determines if smoke tests should run) ─────────── - - name: Check for JWT_SECRET - id: check_config - run: | - if [ -n "${{ secrets.JWT_SECRET }}" ]; then - echo "is_configured=true" >> $GITHUB_OUTPUT - else - echo "is_configured=false" >> $GITHUB_OUTPUT - fi + # This must remain before either create step: invalid branch/environment + # combinations and missing configuration must result in zero Render POSTs. + - name: Validate deployment preflight + env: + DEPLOY_ENVIRONMENT: ${{ inputs.environment }} + RUN_SMOKE_TESTS: ${{ inputs.run_smoke_tests }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SHA: ${{ github.sha }} + RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }} + RENDER_STAGING_SERVICE_ID: ${{ secrets.RENDER_STAGING_SERVICE_ID }} + RENDER_STAGING_WORKER_SERVICE_ID: ${{ secrets.RENDER_STAGING_WORKER_SERVICE_ID }} + RENDER_PRODUCTION_SERVICE_ID: ${{ secrets.RENDER_PRODUCTION_SERVICE_ID }} + RENDER_PRODUCTION_WORKER_SERVICE_ID: ${{ secrets.RENDER_PRODUCTION_WORKER_SERVICE_ID }} + STAGING_API_URL: ${{ secrets.STAGING_API_URL }} + PRODUCTION_API_URL: ${{ secrets.PRODUCTION_API_URL }} + JWT_SECRET: ${{ secrets.JWT_SECRET }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + run: python scripts/render_deploy_preflight.py + + # Creation and polling are separate so both exact deployment IDs are + # retained and independently monitored. POST creation is never retried. + - name: Create API deployment + id: create_api + env: + RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }} + RENDER_SERVICE_ID: ${{ env.RENDER_API_SERVICE_ID }} + RENDER_SERVICE_NAME: API + GITHUB_SHA: ${{ github.sha }} + run: python scripts/render_deploy.py create + + - name: Create worker deployment + id: create_worker + env: + RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }} + RENDER_SERVICE_ID: ${{ env.RENDER_WORKER_SERVICE_ID }} + RENDER_SERVICE_NAME: worker + GITHUB_SHA: ${{ github.sha }} + run: python scripts/render_deploy.py create - # ── Wait for Render auto-deployment ──────────────────────────────── - # Render handles the actual physical deployment when you push. - # We just pause the Action to let Render's servers finish building. - - name: Wait for app to initialise + - name: Wait for API deployment + id: wait_api + continue-on-error: true + env: + RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }} + RENDER_SERVICE_ID: ${{ env.RENDER_API_SERVICE_ID }} + RENDER_SERVICE_NAME: API + RENDER_DEPLOY_ID: ${{ steps.create_api.outputs.deploy_id }} + GITHUB_SHA: ${{ github.sha }} + run: python scripts/render_deploy.py wait + + - name: Wait for worker deployment + id: wait_worker + continue-on-error: true + env: + RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }} + RENDER_SERVICE_ID: ${{ env.RENDER_WORKER_SERVICE_ID }} + RENDER_SERVICE_NAME: worker + RENDER_DEPLOY_ID: ${{ steps.create_worker.outputs.deploy_id }} + GITHUB_SHA: ${{ github.sha }} + run: python scripts/render_deploy.py wait + + - name: Require both deployments to be live + if: always() + env: + API_RESULT: ${{ steps.wait_api.outcome }} + WORKER_RESULT: ${{ steps.wait_worker.outcome }} + API_DEPLOY_ID: ${{ steps.create_api.outputs.deploy_id }} + WORKER_DEPLOY_ID: ${{ steps.create_worker.outputs.deploy_id }} + GITHUB_SHA: ${{ github.sha }} run: | - echo "Waiting 120 seconds for Render to build and start the app..." - sleep 120 + if [ "$API_RESULT" != "success" ] || [ "$WORKER_RESULT" != "success" ]; then + echo "ERROR: coordinated deployment failed for SHA $GITHUB_SHA. API deploy $API_DEPLOY_ID: $API_RESULT; worker deploy $WORKER_DEPLOY_ID: $WORKER_RESULT." + exit 1 + fi + echo "API deploy $API_DEPLOY_ID and worker deploy $WORKER_DEPLOY_ID are live at SHA $GITHUB_SHA." - # ── Health gate ──────────────────────────────────────────────────── - name: Health gate check - id: health_gate - env: - # Use secret URL if provided, otherwise fallback to default - API_URL: ${{ secrets.API_URL || 'https://openshield-api.onrender.com' }} run: | MAX_RETRIES=5 RETRY_DELAY=15 @@ -64,42 +144,24 @@ jobs: for i in $(seq 1 $MAX_RETRIES); do echo "Health check attempt $i of $MAX_RETRIES..." HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$URL" --max-time 30) || true - if [ "$HTTP_STATUS" -eq 200 ]; then echo "Health check passed (HTTP $HTTP_STATUS)" exit 0 fi - - echo "Got HTTP $HTTP_STATUS — retrying in ${RETRY_DELAY}s..." + echo "Got HTTP $HTTP_STATUS; retrying in ${RETRY_DELAY}s..." sleep $RETRY_DELAY done + echo "ERROR: Health gate failed after $MAX_RETRIES attempts on ${{ inputs.environment }}." + exit 1 - echo "HEALTH GATE FAILED after $MAX_RETRIES attempts" - echo "Note: If you haven't set up Render for this fork, this is expected." - # Only allow failure on feature branches; fail on main/dev - if [[ "${{ github.ref }}" == "refs/heads/main" || "${{ github.ref }}" == "refs/heads/dev" ]]; then - echo "ERROR: Health check failed on protected branch. Deployment verification required." - exit 1 - else - echo "Allowing health check failure on feature branch (infra may not be set up)" - exit 0 - fi - - # ── Smoke tests ──────────────────────────────────────────────────── - name: Run smoke tests against live deployment - if: steps.check_config.outputs.is_configured == 'true' || github.event_name == 'workflow_dispatch' + if: inputs.run_smoke_tests env: - API_URL: ${{ secrets.API_URL || 'https://openshield-api.onrender.com' }} JWT_SECRET: ${{ secrets.JWT_SECRET }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} RUN_REAL_SCAN: "true" run: | - if [[ "${{ github.ref }}" == "refs/heads/main" && -z "${{ secrets.JWT_SECRET }}" ]]; then - echo "ERROR: Cannot run smoke tests on main branch without JWT_SECRET configured" - exit 1 - fi echo "Running smoke tests against: $API_URL" - python tests/smoke_test.py \ No newline at end of file + python tests/smoke_test.py diff --git a/.github/workflows/terraform-plan.yml b/.github/workflows/terraform-plan.yml new file mode 100644 index 0000000..82efcfe --- /dev/null +++ b/.github/workflows/terraform-plan.yml @@ -0,0 +1,62 @@ +name: Terraform Plan + +on: + pull_request: + paths: + - "infra/terraform/**" + +permissions: + contents: read + pull-requests: write + +jobs: + plan: + name: Terraform fmt / validate / plan + runs-on: ubuntu-latest + defaults: + run: + working-directory: infra/terraform + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - name: Set up Terraform + uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2 + with: + terraform_version: "~1.6" + terraform_wrapper: false + + - name: Terraform fmt + run: terraform fmt -check -recursive + + - name: Check for Terraform Cloud token + id: check_token + run: | + if [ -n "${{ secrets.TF_API_TOKEN }}" ]; then + echo "has_token=true" >> "$GITHUB_OUTPUT" + else + echo "has_token=false" >> "$GITHUB_OUTPUT" + fi + + - name: Terraform validate (no credentials) + if: steps.check_token.outputs.has_token == 'false' + run: | + echo "TF_API_TOKEN not configured — skipping real plan, running structural validation only." + echo 'terraform {' > override.tf + echo ' backend "local" {}' >> override.tf + echo '}' >> override.tf + terraform init -backend=false + terraform validate + rm override.tf + + - name: Terraform init (real backend) + if: steps.check_token.outputs.has_token == 'true' + env: + TF_TOKEN_app_terraform_io: ${{ secrets.TF_API_TOKEN }} + run: terraform init + + - name: Terraform plan + if: steps.check_token.outputs.has_token == 'true' + env: + TF_TOKEN_app_terraform_io: ${{ secrets.TF_API_TOKEN }} + run: terraform plan -no-color diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7424506..aad7d71 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -213,22 +213,29 @@ Most list methods return an empty list on failure. Methods that fetch one resour ```bash # Python 3.10+ pip install -r requirements.txt -# Installs Flask, Azure SDK clients, requests, psycopg2, PyJWT, and PyYAML for CI workflow validation. +# Installs Flask, Alembic, Azure SDK clients, requests, psycopg2, PyJWT, and PyYAML for CI workflow validation. # Frontend # The frontend directory is currently a scaffold. The React dashboard MVP is on the roadmap. -# API -FLASK_APP=api/app.py flask run --debug - # Database (Docker) docker run --name openshield-db \ -e POSTGRES_USER=openshield \ -e POSTGRES_PASSWORD=openshield \ -e POSTGRES_DB=openshield \ -p 5432:5432 -d postgres + +export DATABASE_URL=postgresql://openshield:openshield@localhost:5432/openshield +alembic upgrade head + +# API +FLASK_APP=api/app.py flask run --debug ``` +See [Database Migrations](docs/database-migrations.md) before creating or applying +a schema change. Migration revisions are written explicitly because OpenShield +does not use ORM metadata. + --- ## Code Standards diff --git a/Dockerfile b/Dockerfile index 0725bfa..a88354d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,12 @@ -FROM python:3.11-slim-bookworm +FROM python:3.11-slim-trixie WORKDIR /app COPY requirements.txt . -RUN pip install --no-cache-dir --upgrade pip && \ +RUN pip install --no-cache-dir --upgrade \ + pip==26.1.2 \ + setuptools==83.0.0 \ + wheel==0.46.3 && \ pip install --no-cache-dir -r requirements.txt COPY . . diff --git a/README.md b/README.md index de0b752..0999c5b 100644 --- a/README.md +++ b/README.md @@ -162,6 +162,10 @@ export AZURE_CLIENT_ID=your-client-id export AZURE_CLIENT_SECRET=your-client-secret export AZURE_TENANT_ID=your-tenant-id export JWT_SECRET=your-strong-secret # used to protect write endpoints (scan trigger, AI) +export DATABASE_URL=postgresql://openshield:openshield@localhost:5432/openshield + +# Create or update the database schema +alembic upgrade head # Run a scan python -c " @@ -175,6 +179,9 @@ print(json.dumps(result, indent=2)) FLASK_APP=api/app.py flask run ``` +See [Database Migrations](docs/database-migrations.md) for schema changes and the +one-time onboarding step required for existing production databases. + **Frontend (React dashboard)** ```bash diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..1ac0d3f --- /dev/null +++ b/alembic.ini @@ -0,0 +1,42 @@ +[alembic] +script_location = %(here)s/alembic +prepend_sys_path = . +path_separator = os + +# DATABASE_URL is read from the environment by alembic/env.py. + +[post_write_hooks] + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..f1ce2f8 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,54 @@ +"""Alembic migration environment for OpenShield.""" + +import os +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import create_engine, pool + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# OpenShield intentionally uses raw SQL rather than ORM metadata. Migration +# revisions therefore define every schema operation explicitly. +target_metadata = None + + +def get_database_url() -> str: + """Return the configured PostgreSQL URL or fail before migration work.""" + database_url = os.environ.get("DATABASE_URL") + if not database_url: + raise RuntimeError("DATABASE_URL environment variable is required") + return database_url + + +def run_migrations_offline() -> None: + """Generate SQL without opening a database connection.""" + context.configure( + url=get_database_url(), + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations against the configured database.""" + connectable = create_engine(get_database_url(), poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..e3ac2e8 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# Revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Apply this migration.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Revert this migration.""" + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/b3f1a2c4d5e6_baseline_schema.py b/alembic/versions/b3f1a2c4d5e6_baseline_schema.py new file mode 100644 index 0000000..0f2002b --- /dev/null +++ b/alembic/versions/b3f1a2c4d5e6_baseline_schema.py @@ -0,0 +1,93 @@ +"""Create the baseline OpenShield schema. + +Revision ID: b3f1a2c4d5e6 +Revises: +Create Date: 2026-07-05 00:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# Revision identifiers, used by Alembic. +revision: str = "b3f1a2c4d5e6" +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create the current scans and findings tables.""" + op.create_table( + "scans", + sa.Column("scan_id", postgresql.UUID(), nullable=False), + sa.Column("subscription_id", sa.Text(), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("claimed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("total_findings", sa.Integer(), server_default=sa.text("0"), nullable=True), + sa.Column("score", sa.Integer(), server_default=sa.text("NULL"), nullable=True), + sa.Column( + "cve_enrichment_status", + sa.Text(), + server_default=sa.text("'PENDING'"), + nullable=True, + ), + sa.Column("status", sa.Text(), server_default=sa.text("'pending'"), nullable=True), + sa.Column("attempt_count", sa.Integer(), server_default=sa.text("0"), nullable=True), + sa.Column("error_message", sa.Text(), nullable=True), + sa.PrimaryKeyConstraint("scan_id", name="scans_pkey"), + ) + + op.create_table( + "findings", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("scan_id", postgresql.UUID(), nullable=True), + sa.Column("rule_id", sa.Text(), nullable=False), + sa.Column("rule_name", sa.Text(), nullable=False), + sa.Column("severity", sa.Text(), nullable=False), + sa.Column("category", sa.Text(), nullable=True), + sa.Column("resource_id", sa.Text(), nullable=True), + sa.Column("resource_name", sa.Text(), nullable=True), + sa.Column("resource_type", sa.Text(), nullable=True), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("remediation", sa.Text(), nullable=True), + sa.Column("playbook", sa.Text(), nullable=True), + sa.Column("frameworks", postgresql.JSONB(), nullable=True), + sa.Column("metadata", postgresql.JSONB(), nullable=True), + sa.Column( + "cve_references", + postgresql.JSONB(), + server_default=sa.text("'[]'::jsonb"), + nullable=True, + ), + sa.Column("cvss_score", sa.Float(), server_default=sa.text("NULL"), nullable=True), + sa.Column( + "exploit_available", + sa.Boolean(), + server_default=sa.text("false"), + nullable=True, + ), + sa.Column("detected_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint( + ["scan_id"], + ["scans.scan_id"], + name="findings_scan_id_fkey", + ), + sa.PrimaryKeyConstraint("id", name="findings_pkey"), + ) + + op.create_index("idx_findings_scan_id", "findings", ["scan_id"], unique=False) + op.create_index("idx_findings_severity", "findings", ["severity"], unique=False) + op.create_index("idx_findings_rule_id", "findings", ["rule_id"], unique=False) + + +def downgrade() -> None: + """Drop the current findings and scans tables.""" + op.drop_index("idx_findings_rule_id", table_name="findings") + op.drop_index("idx_findings_severity", table_name="findings") + op.drop_index("idx_findings_scan_id", table_name="findings") + op.drop_table("findings") + op.drop_table("scans") diff --git a/alembic/versions/c7a2e9f1b3d4_rate_limit_hits.py b/alembic/versions/c7a2e9f1b3d4_rate_limit_hits.py new file mode 100644 index 0000000..b69c617 --- /dev/null +++ b/alembic/versions/c7a2e9f1b3d4_rate_limit_hits.py @@ -0,0 +1,35 @@ +"""Add the rate_limit_hits table for the Postgres-backed rate limiter. + +Revision ID: c7a2e9f1b3d4 +Revises: b3f1a2c4d5e6 +Create Date: 2026-07-10 00:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# Revision identifiers, used by Alembic. +revision: str = "c7a2e9f1b3d4" +down_revision: Union[str, Sequence[str], None] = "b3f1a2c4d5e6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create the rate_limit_hits table used by api.rate_limit.""" + op.create_table( + "rate_limit_hits", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("limit_key", sa.Text(), nullable=False), + sa.Column("hit_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id", name="rate_limit_hits_pkey"), + ) + op.create_index("idx_rate_limit_hits_key_time", "rate_limit_hits", ["limit_key", "hit_at"], unique=False) + + +def downgrade() -> None: + """Drop the rate_limit_hits table.""" + op.drop_index("idx_rate_limit_hits_key_time", table_name="rate_limit_hits") + op.drop_table("rate_limit_hits") diff --git a/api/app.py b/api/app.py index 942a6f3..738826a 100644 --- a/api/app.py +++ b/api/app.py @@ -8,6 +8,7 @@ from dotenv import load_dotenv from flask import Flask, g, jsonify, request from flask_cors import CORS +from werkzeug.middleware.proxy_fix import ProxyFix from api.models.finding import DatabaseManager from api.observability import configure_logging, get_request_id, init_app, init_sentry @@ -22,6 +23,10 @@ # /ready and /metrics are probe/scrape endpoints and must never require auth. _ALWAYS_PUBLIC = {"/", "/health", "/ready", "/metrics"} +# Maximum accepted request body size (bytes). Guards AI endpoints and the rest +# of the API against oversized payloads tying up worker threads. +_MAX_CONTENT_LENGTH = 2 * 1024 * 1024 # 2 MB + _INSECURE_JWT_DEFAULT = "change-me-in-production" _MIN_JWT_SECRET_LENGTH = 32 _GENERATE_CMD = 'python -c "import secrets; print(secrets.token_urlsafe(32))"' @@ -92,6 +97,12 @@ def create_app() -> Flask: """ app = Flask(__name__) + # Trust exactly one reverse-proxy hop (Render's edge) for the client IP + # and scheme, so request.remote_addr reflects the real caller instead of + # collapsing every client onto Render's proxy address. Rate limiting and + # any other per-IP logic depend on this being accurate. + app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_proto=1) + # ------------------------------------------------------------------ # # Observability # # ------------------------------------------------------------------ # @@ -104,6 +115,7 @@ def create_app() -> Flask: # Configuration & Security # # ------------------------------------------------------------------ # app.config["JWT_SECRET"] = _resolve_jwt_secret() + app.config["MAX_CONTENT_LENGTH"] = _MAX_CONTENT_LENGTH # ------------------------------------------------------------------ # # CORS # @@ -128,31 +140,15 @@ def create_app() -> Flask: "Do not use this setting with real Azure scan data in production." ) - # ------------------------------------------------------------------ # - # Database Management # - # ------------------------------------------------------------------ # - # Skip migrations when DATABASE_URL is not set (e.g. during unit tests). - # Deployment startup always sets DATABASE_URL so migrations still run in - # production and staging. Tests that need a real database should set - # DATABASE_URL explicitly in their environment. - if os.environ.get("DATABASE_URL"): - with app.app_context(): - db = DatabaseManager() - db.run_migrations() - else: - logger.info("DATABASE_URL not set — skipping database migrations. Set DATABASE_URL to connect to PostgreSQL.") - @app.teardown_appcontext def close_db(error=None): - """Ensure the database connection is closed after the request.""" + """Return the request's pooled database connection after the request.""" for key in ("db", "db_conn"): db = g.pop(key, None) if db is None: continue try: - if hasattr(db, "conn") and db.conn is not None: - db.conn.close() - logger.debug("Database connection closed gracefully") + db.close() except Exception as exc: logger.error("Error closing database connection: %s", exc) @@ -262,6 +258,10 @@ def forbidden(exc): def not_found(exc): return jsonify({"error": "Not found", "request_id": get_request_id()}), 404 + @app.errorhandler(413) + def payload_too_large(exc): + return jsonify({"error": "Request body too large"}), 413 + @app.errorhandler(500) def internal_error(exc): logger.error("Unhandled exception: %s", exc) diff --git a/api/models/finding.py b/api/models/finding.py index f901f19..83dd9e3 100644 --- a/api/models/finding.py +++ b/api/models/finding.py @@ -3,17 +3,41 @@ import json import logging import os +import threading from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, List, Optional import psycopg2 import psycopg2.extras +import psycopg2.pool logger = logging.getLogger(__name__) FRAMEWORKS_DIR = Path(__file__).parent.parent.parent / "compliance" / "frameworks" +# One pool per DSN, shared across all DatabaseManager instances in this +# process. Routes create a new DatabaseManager per request, but they all +# borrow from the same small set of long-lived connections instead of +# opening a fresh PostgreSQL connection on every request. +_POOLS: Dict[str, "psycopg2.pool.ThreadedConnectionPool"] = {} +_POOLS_LOCK = threading.Lock() + + +_POOL_MAX_CONN = int(os.environ.get("DB_POOL_MAX_CONN", "10")) + + +def _get_pool(dsn: str) -> "psycopg2.pool.ThreadedConnectionPool": + # Pool creation happens at most once per DSN per process lifetime, so a + # plain lock (no unlocked fast path) is simpler and just as cheap here. + with _POOLS_LOCK: + pool = _POOLS.get(dsn) + if pool is None: + pool = psycopg2.pool.ThreadedConnectionPool(1, _POOL_MAX_CONN, dsn) + _POOLS[dsn] = pool + return pool + + SEVERITY_WEIGHTS = {"HIGH": 10, "MEDIUM": 5, "LOW": 2, "INFO": 0} FRAMEWORK_FILE_MAP = { @@ -73,8 +97,8 @@ def to_dict(self) -> Dict[str, Any]: class DatabaseManager: """Manages PostgreSQL persistence for scans, findings, and scoring. - All public methods open a new connection on first use. Call connect() - explicitly if you want to pre-warm the connection. + Database operations open a new connection on first use. Call connect() + explicitly to pre-warm the connection. """ def __init__(self, dsn: Optional[str] = None) -> None: @@ -86,10 +110,10 @@ def __init__(self, dsn: Optional[str] = None) -> None: # ------------------------------------------------------------------ # def connect(self) -> None: - """Open a persistent database connection.""" - self.conn = psycopg2.connect(self.dsn) + """Acquire a connection from this DSN's shared pool.""" + self.conn = _get_pool(self.dsn).getconn() self.conn.autocommit = False - logger.info("Database connection established") + logger.debug("Database connection acquired from pool") def _get_conn(self) -> Any: if self.conn is None or self.conn.closed: @@ -97,11 +121,16 @@ def _get_conn(self) -> Any: return self.conn def close(self) -> None: - """Close the database connection.""" - if self.conn and not self.conn.closed: - self.conn.close() + """Return the connection to its pool (or discard it if broken).""" + if self.conn is None: + return + try: + _get_pool(self.dsn).putconn(self.conn, close=bool(self.conn.closed)) + logger.debug("Database connection returned to pool") + except Exception as exc: + logger.error("Error returning connection to pool: %s", exc) + finally: self.conn = None - logger.debug("Database connection closed") def ping(self) -> bool: """Execute a trivial query to confirm database connectivity. @@ -115,128 +144,12 @@ def ping(self) -> bool: cur.fetchone() return True - # ------------------------------------------------------------------ # - # Schema # - # ------------------------------------------------------------------ # - def init_db(self) -> None: - """Alias for run_migrations. Called by startup.sh on every boot. - - Calling this is always safe; run_migrations() handles both fresh - databases and existing ones via IF NOT EXISTS guards throughout. - """ - self.run_migrations() - - def create_tables(self) -> None: - """Create the findings, scans, and rules tables if they do not exist. - - Includes all columns, including CVE columns, so fresh databases - never need the ALTER TABLE path in run_migrations(). - """ - conn = self._get_conn() - with conn.cursor() as cur: - cur.execute(""" - CREATE TABLE IF NOT EXISTS scans ( - scan_id UUID PRIMARY KEY, - subscription_id TEXT NOT NULL, - started_at TIMESTAMPTZ NOT NULL, - claimed_at TIMESTAMPTZ, - completed_at TIMESTAMPTZ, - total_findings INTEGER DEFAULT 0, - score INTEGER DEFAULT NULL, - cve_enrichment_status TEXT DEFAULT 'PENDING', - status TEXT DEFAULT 'pending', - attempt_count INTEGER DEFAULT 0, - error_message TEXT - ); - """) - cur.execute(""" - CREATE TABLE IF NOT EXISTS findings ( - id SERIAL PRIMARY KEY, - scan_id UUID REFERENCES scans(scan_id), - rule_id TEXT NOT NULL, - rule_name TEXT NOT NULL, - severity TEXT NOT NULL, - category TEXT, - resource_id TEXT, - resource_name TEXT, - resource_type TEXT, - description TEXT, - remediation TEXT, - playbook TEXT, - frameworks JSONB, - metadata JSONB, - cve_references JSONB DEFAULT '[]', - cvss_score FLOAT DEFAULT NULL, - exploit_available BOOLEAN DEFAULT FALSE, - detected_at TIMESTAMPTZ NOT NULL - ); - """) - cur.execute(""" - CREATE INDEX IF NOT EXISTS idx_findings_scan_id - ON findings(scan_id); - CREATE INDEX IF NOT EXISTS idx_findings_severity - ON findings(severity); - CREATE INDEX IF NOT EXISTS idx_findings_rule_id - ON findings(rule_id); - """) - conn.commit() - logger.info("Database tables created / verified") - - def run_migrations(self) -> None: - """Ensure the schema is fully current. Safe to call on every startup. - - Calls create_tables() first so the call order never matters; this - method is safe whether the database is brand new or has existing data. - - On a fresh database: - create_tables() creates all tables including CVE columns. - The ALTER TABLE below is a no-op (IF NOT EXISTS). - - On a pre-CVE database (existed before this feature was merged): - create_tables() verifies tables exist and skips creation. - The ALTER TABLE adds the three CVE columns. - - Concurrent startup safety: - Both CREATE TABLE IF NOT EXISTS and ALTER TABLE ADD COLUMN IF NOT - EXISTS are atomic at the PostgreSQL catalog level. Two Render - instances racing at boot will not error; the second call silently - no-ops on whichever statement the first already completed. - """ - self.create_tables() - - conn = self._get_conn() - try: - with conn.cursor() as cur: - cur.execute(""" - ALTER TABLE findings - ADD COLUMN IF NOT EXISTS cve_references JSONB DEFAULT '[]', - ADD COLUMN IF NOT EXISTS cvss_score FLOAT DEFAULT NULL, - ADD COLUMN IF NOT EXISTS exploit_available BOOLEAN DEFAULT FALSE - """) - cur.execute(""" - ALTER TABLE scans - ADD COLUMN IF NOT EXISTS cve_enrichment_status TEXT DEFAULT 'COMPLETED', - ADD COLUMN IF NOT EXISTS status TEXT DEFAULT 'completed', - ADD COLUMN IF NOT EXISTS attempt_count INTEGER DEFAULT 0, - ADD COLUMN IF NOT EXISTS error_message TEXT, - ADD COLUMN IF NOT EXISTS claimed_at TIMESTAMPTZ - """) - # Fix: If status already existed but was backfilled as 'pending' (e.g. from - # a previous buggy deploy), force it to 'completed' for all historical - # scans that have already finished. - cur.execute( - "UPDATE scans SET status = 'completed' WHERE status = 'pending' AND completed_at IS NOT NULL" - ) - - # Backfill claimed_at for any currently running scans so they don't get - # immediately marked as stale by the new recovery logic. - cur.execute("UPDATE scans SET claimed_at = started_at WHERE status = 'running' AND claimed_at IS NULL") - conn.commit() - logger.info("CVE migrations applied successfully") - except Exception as e: - logger.error("Failed to run CVE migrations: %s", e) - conn.rollback() + """Deprecated compatibility hook; schema changes are managed by Alembic.""" + logger.warning( + "DatabaseManager.init_db() is deprecated and no longer manages the schema; " + "run 'alembic upgrade head' instead" + ) # ------------------------------------------------------------------ # # Write # @@ -560,7 +473,6 @@ def get_score(self) -> int: SELECT scan_id FROM scans WHERE status = 'completed' ORDER BY started_at DESC LIMIT 1 ) GROUP BY severity - GROUP BY severity """ ) rows = cur.fetchall() diff --git a/api/rate_limit.py b/api/rate_limit.py new file mode 100644 index 0000000..3a54551 --- /dev/null +++ b/api/rate_limit.py @@ -0,0 +1,103 @@ +"""Postgres-backed sliding-window rate limiter for Flask routes. + +Deployments run gunicorn with multiple worker processes (see startup.sh), +so a plain in-memory counter would give each worker its own independent +budget instead of enforcing one shared limit per client. State is kept in +the `rate_limit_hits` table instead, with the count-then-insert made atomic +across all workers via a Postgres advisory transaction lock keyed on the +same rate-limit key. +""" + +import logging +import os +from functools import wraps +from typing import Any + +from flask import current_app, g, jsonify, request + +from api.models.finding import DatabaseManager + +logger = logging.getLogger(__name__) + + +def _get_db() -> DatabaseManager: + """Reuse the same `g.db` connection every other route module populates, + so it's cleaned up by api.app's existing close_db teardown instead of + leaking a second, unmanaged connection per request.""" + if "db" not in g: + g.db = DatabaseManager(os.environ["DATABASE_URL"]) + g.db.connect() + return g.db + + +def _check_and_record(key: str, max_requests: int, window_seconds: int) -> bool: + """Return True if this request is within the limit, recording it if so. + + The advisory lock is scoped to the current transaction (released on the + commit/rollback below), which serializes concurrent requests for the + same key across every worker process sharing this database — not just + threads within one process. + """ + db = _get_db() + conn: Any = db._get_conn() + try: + with conn.cursor() as cur: + cur.execute("SELECT pg_advisory_xact_lock(hashtext(%s))", (key,)) + cur.execute( + "DELETE FROM rate_limit_hits WHERE limit_key = %s AND hit_at < now() - make_interval(secs => %s)", + (key, window_seconds), + ) + cur.execute("SELECT COUNT(*) FROM rate_limit_hits WHERE limit_key = %s", (key,)) + (count,) = cur.fetchone() + allowed = count < max_requests + if allowed: + cur.execute("INSERT INTO rate_limit_hits (limit_key, hit_at) VALUES (%s, now())", (key,)) + conn.commit() + except Exception: + conn.rollback() + raise + return allowed + + +def rate_limit(max_requests: int, window_seconds: int = 60): + """Limit a view to ``max_requests`` per ``window_seconds`` per client IP. + + Disabled automatically when the Flask app is in testing mode. Fails + open (allows the request) if the rate-limit check itself errors, since + a database hiccup here shouldn't take down an otherwise-healthy AI + endpoint — this limiter blunts abuse, it isn't the primary safeguard. + """ + + def decorator(fn): + @wraps(fn) + def wrapped(*args, **kwargs): + if current_app.testing: + return fn(*args, **kwargs) + + if "DATABASE_URL" not in os.environ: + # Distinct from the generic fail-open below: a transient DB + # hiccup is fine to wave through, but a deployment that's + # simply missing its DB config would otherwise silently and + # permanently run this metered endpoint with no rate + # limiting at all, which should be loud, not silent. + logger.error( + "DATABASE_URL is not set; refusing %s without rate limiting", + request.path, + ) + return jsonify({"error": "Service temporarily unavailable"}), 503 + + key = f"{request.remote_addr}:{request.path}" + try: + allowed = _check_and_record(key, max_requests, window_seconds) + except Exception: + logger.exception("Rate limit check failed for %s; failing open", key) + allowed = True + + if not allowed: + return jsonify({"error": "Rate limit exceeded, try again later"}), 429 + + return fn(*args, **kwargs) + + return wrapped + + return decorator diff --git a/api/routes/ai.py b/api/routes/ai.py index ed437b3..5059ea4 100644 --- a/api/routes/ai.py +++ b/api/routes/ai.py @@ -5,6 +5,7 @@ from flask import Blueprint, jsonify, request +from api.rate_limit import rate_limit from api.services.ai_provider import PROVIDERS as SUPPORTED_PROVIDERS from api.services.ai_provider import get_completion from ai.retriever import retrieve, VectorStoreNotBuilt @@ -12,6 +13,8 @@ ai_bp = Blueprint("ai", __name__) logger = logging.getLogger(__name__) +_AI_RATE_LIMIT = 20 # requests per minute per client IP, per endpoint + _SEVERITY_RANK = { "CRITICAL": 5, "HIGH": 4, @@ -156,6 +159,7 @@ def _read_request(): @ai_bp.post("/api/ai/insights") +@rate_limit(_AI_RATE_LIMIT) def insights(): data = request.get_json(silent=True) if data is None: @@ -206,6 +210,7 @@ def insights(): @ai_bp.post("/api/ai/summary") +@rate_limit(_AI_RATE_LIMIT) def ai_summary(): body, error = _read_request() if error: @@ -244,6 +249,7 @@ def ai_summary(): @ai_bp.post("/api/ai/prioritise") +@rate_limit(_AI_RATE_LIMIT) def ai_prioritise(): body, error = _read_request() if error: @@ -289,6 +295,7 @@ def ai_prioritise(): @ai_bp.post("/api/ai/ask") +@rate_limit(_AI_RATE_LIMIT) def ai_ask(): body, error = _read_request() if error: @@ -330,6 +337,7 @@ def ai_ask(): @ai_bp.post("/api/ai/threat-simulation") +@rate_limit(_AI_RATE_LIMIT) def ai_threat_simulation(): body, error = _read_request() if error: diff --git a/api/routes/findings.py b/api/routes/findings.py index dcb1f13..3a4939b 100644 --- a/api/routes/findings.py +++ b/api/routes/findings.py @@ -2,14 +2,18 @@ import logging import os +import re from pathlib import Path from flask import Blueprint, g, jsonify, request from api.models.finding import DatabaseManager -_PLAYBOOKS_DIR = Path(__file__).parent.parent.parent / "playbooks" / "cli" +_PLAYBOOKS_DIR = (Path(__file__).parent.parent.parent / "playbooks" / "cli").resolve() -_PLAYBOOKS_DIR = Path(__file__).parent.parent.parent / "playbooks" / "cli" +# Known rule_id shape, e.g. AZ-STOR-001. Anything else is rejected before it +# ever reaches the filesystem, closing off path traversal via a crafted or +# corrupted rule_id. +_RULE_ID_RE = re.compile(r"^[A-Z0-9]+(?:-[A-Z0-9]+)*$") findings_bp = Blueprint("findings", __name__) logger = logging.getLogger(__name__) @@ -73,12 +77,18 @@ def get_playbook(finding_id: int): remediation = finding.get("remediation", "") cve_refs = finding.get("cve_references") or [] - # Map rule_id (e.g. AZ-STOR-001) to script filename (fix_az_stor_001.sh) - script_name = "fix_" + rule_id.lower().replace("-", "_") + ".sh" - script_path = _PLAYBOOKS_DIR / script_name - cli_commands = [] - if script_path.exists(): + script_path = None + if _RULE_ID_RE.match(rule_id or ""): + # Map rule_id (e.g. AZ-STOR-001) to script filename (fix_az_stor_001.sh) + script_name = "fix_" + rule_id.lower().replace("-", "_") + ".sh" + candidate = (_PLAYBOOKS_DIR / script_name).resolve() + if candidate.is_relative_to(_PLAYBOOKS_DIR): + script_path = candidate + else: + logger.warning("Rejected malformed rule_id for playbook lookup: %r", rule_id) + + if script_path is not None and script_path.exists(): raw = script_path.read_text() # Strip comment-only lines and blank lines; join multi-line commands lines = raw.splitlines() diff --git a/api/routes/scans.py b/api/routes/scans.py index 691c262..c87914a 100644 --- a/api/routes/scans.py +++ b/api/routes/scans.py @@ -2,6 +2,7 @@ import logging import os +import threading import uuid from flask import Blueprint, g, jsonify, request @@ -83,9 +84,45 @@ def trigger_scan(): return jsonify({"error": "Critical route failure", "detail": str(exc)}), 500 +def _run_enrichment_in_background(scan_id: str, findings: list, db_url: str) -> None: + """Run CVE enrichment off the request thread and persist the result. + + Runs outside the Flask request/app context (it's started via + threading.Thread), so it opens its own DatabaseManager rather than + reusing flask.g. + """ + db = DatabaseManager(db_url) + try: + enriched = enrich_findings(findings) + db.update_cve_fields(enriched) + db.update_scan_enrichment_status(scan_id, "COMPLETED") + logger.info("Background CVE enrichment complete for scan %s (%d findings)", scan_id, len(enriched)) + except Exception as exc: + logger.error("Background enrichment failed for scan %s: %s", scan_id, exc) + try: + # A failed write (e.g. in update_cve_fields) can leave db.conn in + # an aborted-transaction state. Roll back first, or this status + # update itself raises InFailedSqlTransaction and gets swallowed + # below, leaving the scan stuck at ENRICHING forever. + if db.conn is not None: + db.conn.rollback() + db.update_scan_enrichment_status(scan_id, "FAILED") + except Exception as status_exc: + logger.error("Failed to record FAILED status for scan %s: %s", scan_id, status_exc) + finally: + db.close() + + @scans_bp.post("/api/scans//enrich") def enrich_scan(scan_id): - """Trigger CVE enrichment for an existing scan.""" + """Kick off CVE enrichment for an existing scan in the background. + + Returns immediately with 202 and status ENRICHING; poll + GET /api/scans/ (cve_enrichment_status) for completion. + Running enrichment synchronously here previously caused the request to + time out on scans spanning many rule categories, since NVD lookups are + rate-limited to one every ~7 seconds. + """ try: db = _get_db() @@ -106,20 +143,23 @@ def enrich_scan(scan_id): if not findings: return jsonify({"error": "No findings found for this scan"}), 404 - logger.info("Enriching %d findings for scan %s", len(findings), scan_id) + logger.info("Starting background CVE enrichment for %d findings in scan %s", len(findings), scan_id) db.update_scan_enrichment_status(scan_id, "ENRICHING") - try: - enriched = enrich_findings(findings) - db.update_cve_fields(enriched) - db.update_scan_enrichment_status(scan_id, "COMPLETED") - except Exception as exc: - logger.error("Enrichment failed for scan %s: %s", scan_id, exc) - db.update_scan_enrichment_status(scan_id, "FAILED") - return jsonify({"error": "Enrichment failed", "detail": str(exc)}), 500 + threading.Thread( + target=_run_enrichment_in_background, + args=(scan_id, findings, db.dsn), + daemon=True, + ).start() - return jsonify({"scan_id": scan_id, "status": "COMPLETED", "enriched_count": len(enriched)}) + return jsonify( + { + "scan_id": scan_id, + "status": "ENRICHING", + "message": "CVE enrichment started; poll GET /api/scans/ for completion.", + } + ), 202 except Exception as exc: - logger.error("Failed to enrich scan %s: %s", scan_id, exc) + logger.error("Failed to start enrichment for scan %s: %s", scan_id, exc) return jsonify({"error": "Internal server error", "detail": str(exc)}), 500 diff --git a/api/services/ai_provider.py b/api/services/ai_provider.py index 9af4c16..4a92ebb 100644 --- a/api/services/ai_provider.py +++ b/api/services/ai_provider.py @@ -94,7 +94,7 @@ def _gemini(api_key: str, prompt: str, model: str) -> str: try: resp = requests.post( f"https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent", - params={"key": api_key}, + headers={"x-goog-api-key": api_key, "content-type": "application/json"}, json={"contents": [{"parts": [{"text": prompt}]}]}, timeout=30, ) diff --git a/docs/api-render-deploy.md b/docs/api-render-deploy.md index d5dc2bd..6585660 100644 --- a/docs/api-render-deploy.md +++ b/docs/api-render-deploy.md @@ -1,19 +1,16 @@ -# Test Plan — API-DEP-001 -# Render API Deployment and CI Smoke Testing -# ============================================================ +# Deprecated Render deployment guide -## 1. Overview +This guide describes the previous single-service deployment architecture and is no +longer current. -This test plan covers the verification of the OpenShield API deployment -to Render (Starter instance or higher). The goal is to confirm: +OpenShield now deploys the API and scan worker as separate Render services through +the deterministic manual workflow documented here: -- The Render Web Service builds and deploys the Flask app successfully. -- The database is automatically initialized on startup via `init_db`. -- The pre-commit hook and GitHub Actions CI pipeline gate the code properly. -- The CI pipeline is **community-friendly**, allowing forks to pass even without custom secrets. -- Real Azure scan tests are gated behind `RUN_REAL_SCAN=true` so contributor CI never depends on live Azure credentials. -- All 32 API test cases (health, findings, score, scans, compliance, dashboard contract, auth, edge cases) pass against the live deployment. +- [Current Render deployment guide](deployment/render.md) +- [Render Blueprint](../render.yaml) +- [Deployment workflow](../.github/workflows/deploy.yml) +Do not use the instructions previously contained in this document. --- ## 2. Methodology and Test Rationale @@ -22,7 +19,7 @@ To ensure the highest reliability of the deployment while remaining community-fr ### 2.1 Infrastructure and Pipeline Strategy * **Targeting Render over Azure F1:** Azure App Service's F1 tier imposes a strict 60 CPU-minute daily cap. Render provides unmetered CPU and always-on availability on paid instances, making it significantly more reliable for production environments. -* **Database Initialization:** The `api/models/finding.py` was updated with an `init_db` method. This method ensures that all required tables (`scans`, `findings`) are created automatically during the first deployment, preventing HTTP 500 errors. +* **Database Migrations:** `startup.sh` runs `alembic upgrade head` before starting the worker and Gunicorn. Existing production databases must complete the documented one-time `alembic stamp head` step before this release is deployed. * **Pre-commit Hook:** Fails fast. By running syntax checks and local API smoke tests *before* the commit is allowed, we prevent broken code from polluting the remote branch. * **Community-Friendly CI Gate:** The GitHub Action is designed to be zero-friction for contributors. * **Optional Smoke Tests:** If `JWT_SECRET` is not set (typical for forks), the smoke test step is gracefully skipped rather than failing the build. @@ -81,8 +78,8 @@ API_URL=https://openshield-api.onrender.com JWT_SECRET= \ | File | Purpose | |---|---| -| `startup.sh` | Container startup script, DB initialization, and Gunicorn execution | -| `api/models/finding.py` | Added `init_db` to ensure schema existence on startup | +| `startup.sh` | Runs Alembic once, then starts the worker and Gunicorn | +| `alembic/` | Versioned PostgreSQL schema migrations | | `.github/workflows/deploy.yml` | Flexible GitHub Actions workflow (optional smoke tests) | | `tests/smoke_test.py` | 23-case functional test suite with default secret support | | `.git/hooks/pre-commit` | Local Git hook enforcing syntax checks and local smoke tests | @@ -120,12 +117,14 @@ To enable the automated smoke tests in the CI/CD pipeline, you **must** add the |---|---|---| | `JWT_SECRET` | All smoke tests | Must match the value set in Render. Used to sign tokens for test requests. | | `API_URL` | All smoke tests (optional) | Your Render Service URL. Defaults to the main production instance if not set. | -| `AZURE_SUBSCRIPTION_ID` | Real scan tests | Azure Subscription ID passed to the scan trigger endpoint. | -| `AZURE_CLIENT_ID` | Real scan tests | Service principal client ID for `DefaultAzureCredential`. | -| `AZURE_CLIENT_SECRET` | Real scan tests | Service principal secret for `DefaultAzureCredential`. | -| `AZURE_TENANT_ID` | Real scan tests | Azure AD tenant ID for `DefaultAzureCredential`. | +| `AZURE_SUBSCRIPTION_ID` | Real scan tests | Azure Subscription ID passed to the scan trigger endpoint, and to the `azure/login` OIDC step. | +| `AZURE_CLIENT_ID` | Real scan tests | Service principal client ID for `DefaultAzureCredential` and the `azure/login` OIDC step. | +| `AZURE_TENANT_ID` | Real scan tests | Azure AD tenant ID for `DefaultAzureCredential` and the `azure/login` OIDC step. | > **Note:** `RUN_REAL_SCAN=true` is set automatically by `deploy.yml` on `dev` and `main` branches. Forks and contributor PRs never set this flag, so TC-13 and TC-14 are always skipped in fork CI regardless of which secrets are present. +> +> As of #160, Azure authentication for real-scan smoke tests uses GitHub OIDC federation +> instead of a stored client secret. See `docs/ci-oidc-setup.md` for the one-time setup. --- @@ -139,7 +138,7 @@ To enable the automated smoke tests in the CI/CD pipeline, you **must** add the **DP-02 — Render executes startup script successfully** * **Steps:** Push code to GitHub and monitor Render deployment logs. -* **Expected:** Logs show DB initialization (`Database initialized.`) and Gunicorn starting. +* **Expected:** Logs show Alembic upgrading to `head` before the worker and Gunicorn start. **DP-03 — GitHub Actions CI pipeline passes** * **Steps:** Push a commit and monitor the GitHub Actions tab. diff --git a/docs/azure-setup.md b/docs/azure-setup.md index 7aeef2a..63e345e 100644 --- a/docs/azure-setup.md +++ b/docs/azure-setup.md @@ -137,7 +137,14 @@ brew services start postgresql@15 createdb openshield ``` -The `DatabaseManager.create_tables()` call in the scan trigger will create the schema automatically on first run. +Create the database schema with Alembic before running a scan or starting the API: + +```bash +set -a; source .env; set +a +alembic upgrade head +``` + +See [Database Migrations](database-migrations.md) before onboarding an existing production database. --- @@ -197,7 +204,7 @@ Render is recommended for hosting the OpenShield API. Use the Starter instance o - Name: `openshield-api` - Branch: `main` - Build Command: `pip install -r requirements.txt` - - Start Command: `gunicorn api.app:create_app()` + - Start Command: `./startup.sh` - Instance Type: `Free` 5. Add environment variables under Environment: diff --git a/docs/ci-oidc-setup.md b/docs/ci-oidc-setup.md new file mode 100644 index 0000000..c7aad46 --- /dev/null +++ b/docs/ci-oidc-setup.md @@ -0,0 +1,58 @@ +# GitHub OIDC Federation for Azure (CI smoke tests) + +This replaces the long-lived `AZURE_CLIENT_SECRET` GitHub secret used by `deploy.yml`'s smoke-test +job with short-lived tokens minted per workflow run via GitHub's OIDC provider. It applies **only** +to the maintainer's own smoke-test Azure subscription in CI — it has no effect on how the deployed +product authenticates to end users' own Azure subscriptions when they configure OpenShield to scan +their environment (that always uses credentials the end user supplies). + +## Prerequisite + +The existing `openshield-scanner` service principal from `docs/azure-setup.md` (Reader role on the +maintainer's test subscription). You are adding a federated credential to it, not creating a new +principal. + +## One-time setup (run once, by whoever holds Azure AD admin rights) + +```bash +# Get the existing app's object ID +APP_ID=$(az ad app list --display-name "openshield-scanner" --query "[0].id" --output tsv) + +# Trust GitHub Actions runs from this repo's dev and main branches +az ad app federated-credential create \ + --id "$APP_ID" \ + --parameters '{ + "name": "openshield-dev-branch", + "issuer": "https://token.actions.githubusercontent.com", + "subject": "repo:openshield-org/openshield:ref:refs/heads/dev", + "audiences": ["api://AzureADTokenExchange"] + }' + +az ad app federated-credential create \ + --id "$APP_ID" \ + --parameters '{ + "name": "openshield-main-branch", + "issuer": "https://token.actions.githubusercontent.com", + "subject": "repo:openshield-org/openshield:ref:refs/heads/main", + "audiences": ["api://AzureADTokenExchange"] + }' +``` + +Add one more `federated-credential create` call per branch or trigger type you need — for example, +`"subject": "repo:openshield-org/openshield:pull_request"` if `deploy.yml` is ever triggered on pull +requests, not just `dev`/`main` pushes. `deploy.yml` currently only runs on `workflow_dispatch`, so +confirm which ref it's dispatched against and add a matching federated credential subject for that +ref before relying on this in production. + +## Verifying it works + +1. Trigger `deploy.yml` manually (`workflow_dispatch`) from the Actions tab. +2. Confirm the "Azure login (OIDC)" step succeeds without any `client-secret` input. +3. Confirm the smoke test's real-scan steps (TC-13/TC-14) still pass — this proves + `DefaultAzureCredential()` picked up the federated token correctly. + +## Last step — only after verification passes + +Delete the `AZURE_CLIENT_SECRET` repository secret (**Settings → Secrets and variables → Actions**). +This is a manual, deliberate step — do not automate it, and do not do it before confirming the OIDC +login actually works, or the smoke-test workflow breaks with no working fallback. diff --git a/docs/cve_correlation_feature.md b/docs/cve_correlation_feature.md index 40052dd..a082081 100644 --- a/docs/cve_correlation_feature.md +++ b/docs/cve_correlation_feature.md @@ -21,7 +21,7 @@ The CVE Correlation feature integrates the MITRE National Vulnerability Database | scanner/engine.py | Decoupled Scan. Removed synchronous enrichment from the scan lifecycle. | Performance: Azure scans now return immediately without waiting for NVD rate limits (7s per resource type). | | api/routes/scans.py | New Endpoint. Added `POST /api/scans//enrich`. | Flexibility: CVE enrichment can now be triggered on-demand or by a background job after the scan completes. | | api/models/finding.py | Updated Scan model and added enrichment status tracking. | Persistence: Adds `cve_enrichment_status` to track `PENDING`, `COMPLETED`, or `FAILED` states. | -| api/app.py | Added db.run_migrations call at startup. | Auto-Deployment: Ensures the database schema is updated automatically on any environment where the app is launched. | +| alembic/versions/ | Defines CVE columns in the versioned database schema. | Deployment: Alembic owns schema changes independently of Flask application startup. | | api/routes/score.py | Added GET /api/score/cve-summary endpoint. | Dashboard UI: Provides the frontend with high-level data like Total Known Exploits and enrichment status. | | api/routes/findings.py | Returns findings from the database without JIT enrichment. | Performance: Ensures predictable and fast API responses for findings. | diff --git a/docs/database-migrations.md b/docs/database-migrations.md new file mode 100644 index 0000000..66791cb --- /dev/null +++ b/docs/database-migrations.md @@ -0,0 +1,107 @@ +# Database Migrations + +OpenShield manages its PostgreSQL schema with Alembic. Application queries remain +raw SQL; migration files define schema operations explicitly and do not introduce +ORM models or SQLAlchemy metadata. + +All commands below run from the repository root and require `DATABASE_URL` in the +environment: + +```bash +export DATABASE_URL=postgresql://openshield:openshield@localhost:5432/openshield +``` + +## Fresh databases + +Create the complete current schema by upgrading to the latest revision: + +```bash +alembic upgrade head +alembic current +``` + +Production startup runs `alembic upgrade head` once, before starting the worker +and Gunicorn. The Flask application does not create or migrate tables. + +## Deployment checklist + +For a brand-new database: + +1. Set `DATABASE_URL`. +2. Deploy normally; `startup.sh` runs `alembic upgrade head`. +3. Confirm `alembic current` reports the head revision. + +For an existing production database that already has OpenShield tables: + +1. Back up the database. +2. Verify the live schema matches the baseline revision in `alembic/versions/`. +3. Run `alembic stamp head` once before the first Alembic-enabled deployment. +4. Deploy normally; `startup.sh` must continue to run only `alembic upgrade head`. + +Do not automate `alembic stamp head` in application code, startup scripts, +containers, or deployment configuration. + +## Existing production databases + +The baseline migration represents the schema that OpenShield already created in +production before Alembic was adopted. It must not be run against an unversioned +database that already contains the `scans` and `findings` tables. + +Before deploying this Alembic-enabled release to an existing production database: + +1. Back up the database. +2. Verify that `scans`, `findings`, their columns, constraints, and indexes match + the baseline revision in `alembic/versions/`. +3. Record the existing schema at the baseline revision without executing DDL: + + ```bash + alembic stamp head + alembic current + ``` + +This is a one-time operational step for existing databases. Never add automatic +stamping to application code, startup scripts, containers, or deployment +configuration. After stamping, normal deployments use only: + +```bash +alembic upgrade head +``` + +## Creating a migration + +Create an empty revision with a short, descriptive, imperative message: + +```bash +alembic revision -m "add finding source" +``` + +Because OpenShield has no ORM metadata, do not use `--autogenerate`. Implement +both `upgrade()` and `downgrade()` explicitly with Alembic operations, then review +the generated file before applying it. + +Migration files use Alembic's standard `_.py` naming. Keep the +message lowercase and specific to one schema change, for example +`add_scan_region` or `index_findings_detected_at`. Each revision must point to the +current head through `down_revision`; do not create parallel heads. + +## Contributor workflow + +1. Start PostgreSQL and export `DATABASE_URL`. +2. Update to the current schema with `alembic upgrade head`. +3. Create and implement the new revision. +4. Test the upgrade from the previous revision. +5. Test `alembic downgrade -1` and then `alembic upgrade head` on disposable data. +6. Test the full migration chain against a fresh empty database. +7. Run the application test suite. + +Useful inspection commands: + +```bash +alembic current +alembic history --verbose +alembic heads +alembic upgrade head --sql +``` + +Treat downgrades as destructive operations and run them only against a database +whose data can be restored. diff --git a/docs/deployment/render.md b/docs/deployment/render.md new file mode 100644 index 0000000..6a53c2f --- /dev/null +++ b/docs/deployment/render.md @@ -0,0 +1,215 @@ +# OpenShield Render deployment + +OpenShield uses a manual, deterministic GitHub Actions workflow to deploy the API +and scan worker to Render. Deployments are coordinated around one immutable GitHub +SHA, but they are not atomic. + +- Blueprint: [`render.yaml`](../../render.yaml) +- Workflow: [`.github/workflows/deploy.yml`](../../.github/workflows/deploy.yml) +- Deploy client: [`scripts/render_deploy.py`](../../scripts/render_deploy.py) + +## Service layout + +| Environment | Service | Type | Branch | Start command | +|---|---|---|---|---| +| Staging | `openshield-api-staging` | web | `dev` | `./startup.sh` | +| Staging | `openshield-worker-staging` | worker | `dev` | `python -m scanner.worker` | +| Production | `openshield-api` | web | `main` | `./startup.sh` | +| Production | `openshield-worker` | worker | `main` | `python -m scanner.worker` | + +All four services use `autoDeployTrigger: "off"`. Render does not independently +deploy either worker or API service; GitHub Actions is the sole deployment +controller. The workflow may only be dispatched as follows: + +- `staging` from the `dev` branch; +- `production` from the `main` branch. + +Any other branch/environment combination fails during preflight before a Render +deployment is created. The job uses the selected GitHub Environment, allowing +maintainers to add approval gates and protection rules for staging or production. + +The API startup applies database migrations with `alembic upgrade head` and then +executes Gunicorn. The worker runs only in its separate Render worker service. + +## Required configuration + +Configure these GitHub Actions secrets for every deployment: + +| Secret | Purpose | +|---|---| +| `RENDER_API_KEY` | Authenticate Render API requests | +| `RENDER_STAGING_SERVICE_ID` | Staging API service ID | +| `RENDER_STAGING_WORKER_SERVICE_ID` | Staging worker service ID | +| `RENDER_PRODUCTION_SERVICE_ID` | Production API service ID | +| `RENDER_PRODUCTION_WORKER_SERVICE_ID` | Production worker service ID | +| `STAGING_API_URL` | Staging API health and smoke-test URL | +| `PRODUCTION_API_URL` | Production API health and smoke-test URL | + +When `run_smoke_tests` is enabled, these existing secrets are also mandatory: + +- `JWT_SECRET` +- `AZURE_SUBSCRIPTION_ID` +- `AZURE_CLIENT_ID` +- `AZURE_CLIENT_SECRET` +- `AZURE_TENANT_ID` + +They are not required for a health-only deployment. The preflight validates every +value required for the selected operation before either deployment POST and never +prints secret values. + +Each Render service also needs its application environment configured. Within an +environment, the API and worker must use matching `DATABASE_URL` and `JWT_SECRET` +values and the required Azure credentials. Values declared with `sync: false` in +the Blueprint must be configured in Render and are never stored in this repository. + +The two API services also require `ALLOWED_ORIGINS`. Set the staging API to its +staging frontend origin and the production API to its production frontend origin. +When more than one origin is required, use the comma-separated format accepted by +the Flask application. Leaving this value unset enables wildcard CORS and is not +acceptable for a real staging or production environment. Do not add this setting to +worker services. + +## Coordinated deterministic deployment + +For the selected environment, the workflow: + +1. validates the selected branch, service IDs, API URL, SHA, Render key, and any + enabled smoke-test credentials; +2. creates one API deployment pinned to `github.sha` and records its exact ID; +3. creates one worker deployment pinned to the same `github.sha` and records its + exact ID; +4. polls each exact deployment ID independently until both are `live`; +5. runs the API `/health` gate only after both services are live; +6. optionally runs the smoke suite only after both deployments and health succeed. + +The deployment-creation POST is attempted once. An ambiguous POST network failure +may mean Render accepted the request even though Actions did not receive its ID, so +automatically repeating the POST could create a duplicate deployment. + +Polling GET requests retry transient network failures, HTTP 429, and selected HTTP +5xx responses with bounded backoff. Permanent errors, malformed responses, terminal +Render failures, and the overall timeout fail immediately. Retry time counts against +the overall timeout. Logs identify the service, SHA, deployment ID when known, and +last status without printing credentials. + +## Migration and worker startup ordering + +API startup owns database migration execution through `alembic upgrade head`. The +worker can begin querying PostgreSQL while a migration is still completing, so it +may temporarily encounter database errors; the current worker loop catches these +errors and retries. This does not make every future migration safe. Non-backward- +compatible schema changes require deliberate API/worker rollout planning, and +maintainers should monitor worker logs during the first deployment after a schema +migration. Do not run migrations independently in both services. + +## Partial failure and recovery + +API and worker deployment is coordinated, not atomic. Possible partial states +include: + +- the API deployment is created but worker creation fails; +- one deployment reaches `live` while the other fails; +- Actions loses contact while Render continues an accepted deployment; +- a rerun creates new deployment IDs for the same SHA. + +When this happens: + +1. inspect both services in Render and record the API and worker deployment IDs; +2. confirm the commit SHA attached to each deployment; +3. rerun the workflow from the correct branch for a current-branch deployment, or + use the manual rollback procedure below for a historical SHA; +4. do not manually deploy a different commit to only one service; +5. confirm both services converge on the same SHA before treating the environment + as healthy. + +If the initial POST failed ambiguously, first inspect Render for a deployment at the +requested SHA before rerunning. A rerun is safe operationally only when maintainers +understand whether the earlier request was accepted. + +## Worker verification limitation + +Render reporting the worker deployment as `live` proves that Render deployed and +started the service. It does not prove that the worker can connect to the database, +claim queued scans, process them, or persist completed results. This repository does +not currently expose a worker heartbeat or queue-processing health endpoint. +Application-level worker verification therefore remains a separate live step: queue +a controlled scan and confirm it is claimed and completed successfully. + +## Blueprint validation + +Local YAML parsing verifies syntax only. It is not equivalent to validation against +Render's current Blueprint schema. Maintainers may validate the Blueprint through +the Render dashboard preview or Render's supported Blueprint validation API before +syncing services. Live validation requires Render credentials and must not be +reported as completed unless the live API or dashboard was actually used. + +## Manual deployment and verification + +1. Open **Actions > Deploy API and worker to Render > Run workflow**. +2. Select `dev` with `staging`, or `main` with `production`. +3. Choose whether to run the real-scan smoke tests. +4. Record the GitHub SHA and both Render deployment IDs from the workflow logs. +5. Confirm the API health result and, if enabled, the smoke-test result. +6. Perform the application-level worker verification described above. +7. Confirm both Render services report the same SHA. + +A normal workflow dispatch always deploys `github.sha` from the selected current +`dev` or `main` ref. It cannot deploy an arbitrary historical SHA. + +## Rollback + +Historical rollback must currently be performed with the deployment client or the +Render dashboard/API. The rollback is coordinated but not atomic: invoke it for +both API and worker using the same known-good SHA, capture both deployment IDs, and +confirm both services converge before treating the environment as healthy. + +Create and wait for the API deployment: + +```bash +RENDER_API_KEY="" \ +RENDER_SERVICE_ID="" \ +RENDER_SERVICE_NAME="API" \ +GITHUB_SHA="" \ +python scripts/render_deploy.py create +# deploy_id= + +RENDER_API_KEY="" \ +RENDER_SERVICE_ID="" \ +RENDER_SERVICE_NAME="API" \ +RENDER_DEPLOY_ID="" \ +GITHUB_SHA="" \ +python scripts/render_deploy.py wait +``` + +Repeat the equivalent commands for the worker: + +```bash +RENDER_API_KEY="" \ +RENDER_SERVICE_ID="" \ +RENDER_SERVICE_NAME="worker" \ +GITHUB_SHA="" \ +python scripts/render_deploy.py create +# deploy_id= + +RENDER_API_KEY="" \ +RENDER_SERVICE_ID="" \ +RENDER_SERVICE_NAME="worker" \ +RENDER_DEPLOY_ID="" \ +GITHUB_SHA="" \ +python scripts/render_deploy.py wait +``` + +For a simpler one-service operation, default `deploy` mode creates and waits for a +single service: + +```bash +RENDER_API_KEY="" \ +RENDER_SERVICE_ID="" \ +RENDER_SERVICE_NAME="" \ +GITHUB_SHA="" \ +python scripts/render_deploy.py deploy +``` + +Run that command separately for both matching services and confirm both report the +same SHA. These examples are documented procedures only; no live rollback was run +as part of this change. diff --git a/docs/secrets-inventory.md b/docs/secrets-inventory.md new file mode 100644 index 0000000..e1c8d95 --- /dev/null +++ b/docs/secrets-inventory.md @@ -0,0 +1,39 @@ +# Secrets Inventory + +Every credential-shaped value in this project, and exactly where it lives. Three categories: +GitHub Actions secrets, Render environment variables, and Terraform variables. A value can +legitimately live in more than one place (e.g. `JWT_SECRET` must match between Render and any +GitHub secret that signs tokens for smoke tests) — that overlap is called out explicitly below, +not treated as a bug. + +## GitHub Actions secrets + +| Secret | Used by | Purpose | +|---|---|---| +| `JWT_SECRET` | `deploy.yml` smoke tests | Must match the value configured on the Render web service; signs test JWTs | +| `API_URL` | `deploy.yml` (optional) | Overrides the default production API URL for smoke tests | +| `AZURE_SUBSCRIPTION_ID` | `deploy.yml` real-scan tests | Target subscription for the maintainer's own smoke-test scan | +| `AZURE_CLIENT_ID` | `deploy.yml` real-scan tests, `azure/login@v3` | Service principal client ID (OIDC, no secret needed as of #160) | +| `AZURE_TENANT_ID` | `deploy.yml` real-scan tests, `azure/login@v3` | Azure AD tenant ID (OIDC) | +| ~~`AZURE_CLIENT_SECRET`~~ | — | Removed by #160; see `docs/ci-oidc-setup.md` for the OIDC replacement and the manual deletion step | +| `TF_API_TOKEN` | `terraform-plan.yml` (once configured) | Terraform Cloud API token for `terraform plan` on infra PRs | + +## Render environment variables (managed via Terraform going forward — `infra/terraform/render.tf`) + +| Variable | Source in Terraform | Notes | +|---|---|---| +| `DATABASE_URL` | `var.database_url` | Points at the `render_postgres.db` instance's connection string | +| `JWT_SECRET` | `var.jwt_secret` (via `render_env_group.api_secrets`) | Must match the `JWT_SECRET` GitHub secret used for smoke tests | +| `ALLOWED_ORIGINS` | `var.allowed_origins` | CORS allowlist — the two Vercel project URLs | +| `ANTHROPIC_API_KEY` / `GROQ_API_KEY` / `GEMINI_API_KEY` | matching `var.*_api_key` | AI insights providers, at least one required | +| `NVD_API_KEY` | `var.nvd_api_key` | Optional, raises NVD rate limit | +| `SENTRY_DSN` | `var.sentry_dsn` | Optional error tracking | +| `AZURE_SUBSCRIPTION_ID` / `AZURE_CLIENT_ID` / `AZURE_TENANT_ID` | matching `var.azure_*` | For the maintainer's own smoke-test identity only — **not** end-user scan credentials | +| `AZURE_CLIENT_SECRET` | *(not in Terraform)* | Configured directly in the Render dashboard if the product itself still needs a standing credential for its own scheduled scans — operationally distinct from the CI smoke-test identity above, and out of scope for OIDC (OIDC only works for GitHub Actions workflows) | + +## Terraform variables (`infra/terraform/variables.tf`) + +Every variable above that flows into a Render or Vercel resource is declared `sensitive = true` with +no default, supplied via a local `terraform.tfvars` (gitignored) or CI secrets — see +`infra/terraform/terraform.tfvars.example` for the full list and `infra/terraform/README.md` for the +plan/apply flow. diff --git a/docs/superpowers/plans/2026-07-12-terraform-oidc.md b/docs/superpowers/plans/2026-07-12-terraform-oidc.md new file mode 100644 index 0000000..7d80c6b --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-terraform-oidc.md @@ -0,0 +1,962 @@ +# Terraform IaC + Azure OIDC Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Codify Render/Vercel infrastructure as Terraform (current live topology only) and switch the +Azure smoke-test workflow from a long-lived client secret to GitHub OIDC federation. + +**Architecture:** A new `infra/terraform/` root with the `render-oss/render` and `vercel/vercel` +providers, state in Terraform Cloud (`cloud {}` block, org/workspace supplied by the user later — not +applied in this PR). A `terraform-plan.yml` workflow validates and (once `TF_API_TOKEN` exists) +plans on PRs touching `infra/terraform/**`. `deploy.yml`'s Azure smoke-test step switches to +`azure/login@v3` with OIDC; no application code changes since `DefaultAzureCredential()` already +picks up workload-identity federation automatically. + +**Tech Stack:** Terraform >= 1.6 (for the `cloud` block), `render-oss/render` provider, +`vercel/vercel` provider (>= 4.8), `azure/login@v3` GitHub Action. + +## Global Constraints + +- No live Render/Vercel/Azure/Terraform Cloud credentials exist yet. Every credential-shaped + Terraform variable is declared `sensitive = true` with **no default** — real values are supplied + later via `terraform.tfvars` (gitignored) or CI secrets, never committed. +- No `terraform apply` happens in this work. Verification is `terraform fmt -check`, + `terraform init` (downloads providers, no auth needed), and `terraform validate`. +- Model the *current* live topology only (from `docs/api-render-deploy.md`): one Render web service + `openshield-api` (Docker runtime, `./startup.sh` start command), one Render Postgres + `openshield-db`, two Vercel projects (`frontend` = dashboard, `website` = docs site). Do not add + the staging environment or a separate worker service — that's #157/#172's job, done later. +- OIDC applies only to `deploy.yml`'s smoke-test Azure login, not to the product's own + `DefaultAzureCredential()` usage for scanning end-user subscriptions (`scanner/azure_client.py` — + do not touch this file). +- Deleting the `AZURE_CLIENT_SECRET` GitHub secret is a manual, user-performed step after they + confirm OIDC works in a real run. Do not attempt it via `gh` or any tool. +- Verified resource schemas (fetched from the providers' own docs, not assumed): + - `render_web_service`: required `name`, `plan`, `region`, `runtime_source` block. `runtime_source.docker` + block requires `branch`, `repo_url`; optional `dockerfile_path`. Optional top-level: `env_vars` + (map of `{ value = "..." }`), `start_command`. + - `render_postgres`: required `name`, `plan`, `region`, `version`. Optional `database_name`, + `database_user`. + - `render_env_group`: required `name`. Optional `env_vars` (map of `{ value = "..." }` or + `{ generate_value = true }`). + - `render_env_group_link`: required `env_group_id`, `service_ids` (set). + - Render provider block: `api_key`, `owner_id` (from `RENDER_API_KEY`, `RENDER_OWNER_ID` env vars). + - `vercel_project`: required `name`. Optional `framework`, `root_directory`, + `git_repository = { type = "github", repo = "..." }`, `environment` (set of + `{ key, value, target, sensitive }`). + - Vercel provider block: `api_token` (from `VERCEL_API_TOKEN` env var), optional `team`. + - `azure/login@v3` OIDC inputs: `client-id`, `tenant-id`, `subscription-id` (no `client-secret`). + Requires workflow-level `permissions: { id-token: write, contents: read }`. + - Terraform Cloud remote state: `terraform { cloud { organization = "..." workspaces { name = "..." } } }`. + +--- + +### Task 1: Terraform skeleton — providers, backend, core variables + +**Files:** +- Create: `infra/terraform/versions.tf` +- Create: `infra/terraform/variables.tf` +- Create: `infra/terraform/terraform.tfvars.example` +- Create: `infra/terraform/.gitignore` + +**Interfaces:** +- Produces: `var.render_api_key`, `var.render_owner_id`, `var.vercel_api_token`, `var.vercel_team`, + `var.tfc_organization`, `var.tfc_workspace_name` — every later task's provider config and resource + block consumes these. + +- [ ] **Step 1: Create the backend + provider requirements file** + +```hcl +# infra/terraform/versions.tf +terraform { + required_version = ">= 1.6" + + cloud { + organization = "REPLACE_WITH_TFC_ORG" + workspaces { + name = "openshield" + } + } + + required_providers { + render = { + source = "render-oss/render" + version = "~> 1.8" + } + vercel = { + source = "vercel/vercel" + version = ">= 4.8" + } + } +} + +provider "render" { + api_key = var.render_api_key + owner_id = var.render_owner_id +} + +provider "vercel" { + api_token = var.vercel_api_token + team = var.vercel_team +} +``` + +The `cloud` block's `organization` is a placeholder string, not a variable — HCP Terraform does not +allow interpolation inside the `cloud` block. Document in `infra/terraform/README.md` (Task 4) that +the user must replace `REPLACE_WITH_TFC_ORG` with their real Terraform Cloud organization name before +`terraform init` will succeed. + +- [ ] **Step 2: Create the variables file** + +```hcl +# infra/terraform/variables.tf +variable "render_api_key" { + description = "Render API key. Create at https://dashboard.render.com/u/settings#api-keys" + type = string + sensitive = true +} + +variable "render_owner_id" { + description = "Render owner (user or team) ID that owns the managed resources" + type = string + sensitive = true +} + +variable "vercel_api_token" { + description = "Vercel API token. Create at https://vercel.com/account/tokens" + type = string + sensitive = true +} + +variable "vercel_team" { + description = "Vercel team slug or ID that owns the managed projects" + type = string + default = null +} + +variable "database_url" { + description = "Full Postgres connection string for the API service (sourced from the render_postgres resource's connection_info in normal use; declared as a variable here since we are not applying yet)" + type = string + sensitive = true + default = null +} + +variable "jwt_secret" { + description = "JWT signing secret for the API service — must match between Render and any GitHub Actions secrets that also need it" + type = string + sensitive = true + default = null +} + +variable "allowed_origins" { + description = "Comma-separated list of allowed CORS origins for the API service" + type = string + default = null +} + +variable "anthropic_api_key" { + description = "Anthropic API key for the AI insights feature" + type = string + sensitive = true + default = null +} + +variable "groq_api_key" { + description = "Groq API key for the AI insights feature" + type = string + sensitive = true + default = null +} + +variable "gemini_api_key" { + description = "Gemini API key for the AI insights feature" + type = string + sensitive = true + default = null +} + +variable "nvd_api_key" { + description = "NVD API key for CVE enrichment (optional, raises the NVD rate limit)" + type = string + sensitive = true + default = null +} + +variable "sentry_dsn" { + description = "Sentry DSN for error tracking (optional)" + type = string + sensitive = true + default = null +} + +variable "azure_subscription_id" { + description = "Azure subscription ID for the maintainer's own smoke-test scan (NOT end-user scan credentials)" + type = string + sensitive = true + default = null +} + +variable "azure_client_id" { + description = "Azure service principal client ID for the maintainer's own smoke-test scan" + type = string + sensitive = true + default = null +} + +variable "azure_tenant_id" { + description = "Azure AD tenant ID for the maintainer's own smoke-test scan" + type = string + sensitive = true + default = null +} + +variable "render_postgres_plan" { + description = "Render Postgres plan tier. Confirm the real live tier against the Render dashboard before applying — this default is a guess based on docs/api-render-deploy.md, not a verified value" + type = string + default = "free" +} + +variable "render_web_service_plan" { + description = "Render web service plan tier. docs/api-render-deploy.md says \"Starter instance or higher\" — confirm the exact live tier against the dashboard before applying" + type = string + default = "starter" +} + +variable "render_region" { + description = "Render region for all managed services" + type = string + default = "oregon" +} +``` + +- [ ] **Step 3: Create the tfvars example (no real values, ever)** + +```hcl +# infra/terraform/terraform.tfvars.example +# +# Copy this file to terraform.tfvars and fill in real values. terraform.tfvars +# is gitignored (see infra/terraform/.gitignore) — never commit real credentials. + +render_api_key = "rnd_xxxxxxxxxxxxxxxxxxxxxxxxxxxx" +render_owner_id = "tea-xxxxxxxxxxxxxxxxxxxx" + +vercel_api_token = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +vercel_team = "your-team-slug" + +database_url = "postgresql://user:password@host:5432/openshield" +jwt_secret = "generate-with: python -c \"import secrets; print(secrets.token_urlsafe(32))\"" +allowed_origins = "https://openshield-gules.vercel.app,https://openshield-website.vercel.app" + +anthropic_api_key = "" +groq_api_key = "" +gemini_api_key = "" +nvd_api_key = "" +sentry_dsn = "" + +azure_subscription_id = "" +azure_client_id = "" +azure_tenant_id = "" + +render_postgres_plan = "free" +render_web_service_plan = "starter" +render_region = "oregon" +``` + +- [ ] **Step 4: Gitignore real tfvars and local state** + +``` +# infra/terraform/.gitignore +terraform.tfvars +*.tfstate +*.tfstate.* +.terraform/ +.terraform.lock.hcl +``` + +`.terraform.lock.hcl` is deliberately ignored here since this PR never runs a real `terraform init` +against a live backend to generate a committed lock file — see Task 5's verification note. + +- [ ] **Step 5: Commit** + +```bash +git add infra/terraform/versions.tf infra/terraform/variables.tf infra/terraform/terraform.tfvars.example infra/terraform/.gitignore +git commit -m "infra: add Terraform provider/backend skeleton for Render and Vercel" +``` + +--- + +### Task 2: Render resources — Postgres, web service, env group + +**Files:** +- Create: `infra/terraform/render.tf` + +**Interfaces:** +- Consumes: `var.render_postgres_plan`, `var.render_web_service_plan`, `var.render_region`, + `var.database_url`, `var.jwt_secret`, `var.allowed_origins`, `var.anthropic_api_key`, + `var.groq_api_key`, `var.gemini_api_key`, `var.nvd_api_key`, `var.sentry_dsn`, + `var.azure_subscription_id`, `var.azure_client_id`, `var.azure_tenant_id` (all from Task 1). +- Produces: `render_postgres.db`, `render_web_service.api`, `render_env_group.api_secrets` — Task 4's + `outputs.tf` references these. + +- [ ] **Step 1: Write the Postgres resource** + +```hcl +# infra/terraform/render.tf + +resource "render_postgres" "db" { + name = "openshield-db" + plan = var.render_postgres_plan + region = var.render_region + version = "16" + + database_name = "openshield" + database_user = "openshield" +} +``` + +- [ ] **Step 2: Write the shared env group for secret-shaped values** + +```hcl +resource "render_env_group" "api_secrets" { + name = "openshield-api-secrets" + + env_vars = { + JWT_SECRET = { value = var.jwt_secret } + ANTHROPIC_API_KEY = { value = var.anthropic_api_key } + GROQ_API_KEY = { value = var.groq_api_key } + GEMINI_API_KEY = { value = var.gemini_api_key } + NVD_API_KEY = { value = var.nvd_api_key } + SENTRY_DSN = { value = var.sentry_dsn } + AZURE_SUBSCRIPTION_ID = { value = var.azure_subscription_id } + AZURE_CLIENT_ID = { value = var.azure_client_id } + AZURE_TENANT_ID = { value = var.azure_tenant_id } + } +} +``` + +Note: `AZURE_CLIENT_SECRET` is deliberately absent from this env group. It stays out of Terraform +entirely — the whole point of this issue is to stop needing it in the smoke-test workflow, and the +Render web service's own Azure credentials (for the product's real scanning feature) are configured +directly by the maintainer in the Render dashboard, not through this shared secrets group, since they +are operationally distinct from the CI smoke-test identity (see Global Constraints). + +- [ ] **Step 3: Write the web service resource** + +```hcl +resource "render_web_service" "api" { + name = "openshield-api" + plan = var.render_web_service_plan + region = var.render_region + + runtime_source = { + docker = { + repo_url = "https://github.com/openshield-org/openshield" + branch = "main" + dockerfile_path = "Dockerfile" + } + } + + start_command = "./startup.sh" + + env_vars = { + DATABASE_URL = { value = var.database_url } + ALLOWED_ORIGINS = { value = var.allowed_origins } + OPENSHIELD_ENV = { value = "production" } + RENDER = { value = "true" } + } +} + +resource "render_env_group_link" "api_secrets_link" { + env_group_id = render_env_group.api_secrets.id + service_ids = [render_web_service.api.id] +} +``` + +`OPENSHIELD_ENV=production` and `RENDER=true` are set explicitly here because `api/app.py`'s +`_is_production()` checks exactly these two environment variables to enforce the strong-`JWT_SECRET` +fail-closed behavior described in `docs/api-render-deploy.md`. + +- [ ] **Step 4: Commit** + +```bash +git add infra/terraform/render.tf +git commit -m "infra: define Render Postgres, web service, and shared secrets env group" +``` + +--- + +### Task 3: Vercel resources — dashboard and website projects + +**Files:** +- Create: `infra/terraform/vercel.tf` + +**Interfaces:** +- Consumes: nothing beyond the `vercel` provider configured in Task 1. +- Produces: `vercel_project.dashboard`, `vercel_project.website` — Task 4's `outputs.tf` references + these. + +- [ ] **Step 1: Write the dashboard (frontend/) project** + +```hcl +# infra/terraform/vercel.tf + +resource "vercel_project" "dashboard" { + name = "openshield-dashboard" + framework = "vite" + root_directory = "frontend" + + git_repository = { + type = "github" + repo = "openshield-org/openshield" + } + + environment = [ + { + key = "VITE_API_BASE_URL" + value = "https://openshield-api.onrender.com" + target = ["production", "preview"] + sensitive = false + } + ] +} +``` + +`framework = "vite"` matches `frontend/package.json`'s `vite`/`vite build` scripts confirmed during +design. `name` uses a descriptive slug rather than assuming the exact live project name/URL +(`openshield-gules`, per `README.md`) since Vercel project names become part of the deployment URL +and renaming an existing live project via Terraform on first apply could change that URL — flagged in +`infra/terraform/README.md` (Task 4) as something the user must reconcile (likely via +`terraform import` against the *existing* project rather than creating a new one). + +- [ ] **Step 2: Write the website project** + +```hcl +resource "vercel_project" "website" { + name = "openshield-website" + root_directory = "website" + + git_repository = { + type = "github" + repo = "openshield-org/openshield" + } +} +``` + +No `framework` value — the website is static HTML with a Tailwind CDN `