Features/improve interaction

.env.exemple

@@ -30,6 +30,14 @@ STORAGE_ACCESS_KEY=minioadmin
 STORAGE_SECRET_KEY=minioadmin
 STORAGE_REGION=us-east-1
 
+# ── Realtime (Redis pub/sub) ──
+# Fire-and-forget SSE hints consumed by crm-backend's realtime hub. Optional
+# everywhere, including production: an empty URL makes core.realtime.emit() a
+# silent no-op, so a broker outage degrades the UI to polling and never affects
+# a business write. db 1 on purpose — db 0 is crm-backend's cache.
+REALTIME_ENABLED=false
+REALTIME_REDIS_URL=redis://:changeme_redis_password@localhost:6379/1
+
 # ── CORS ──
 ALLOW_ORIGIN=*
 

.gitignore

@@ -210,7 +210,11 @@ pyrightconfig.json
 [Ll]ib
 [Ll]ib64
 [Ll]ocal
-[Ss]cripts
+# [Ss]cripts — REMOVED. This unanchored toptal venv entry matched this
+# project's own top-level scripts/ directory, which silently kept every
+# file under scripts/sql/migrations/ out of the repository. The virtualenv
+# case it was meant to cover is already handled by the `.venv` and `venv/`
+# rules above, so the entry was pure downside.
 pyvenv.cfg
 pip-selfcheck.json
 

api/simulation/mappers.py

@@ -1,10 +1,15 @@
 from api.simulation.schemas import (
     ConsumerResult,
+    CrmDataPreview,
+    IncompleteMeter,
     IterationResult,
     KeyResult,
+    PreviewBlocker,
     Simulation,
     SimulationDetail,
 )
+from core.i18n import translate
+from shared.crm_preflight import Preflight
 from shared.models.local_models import (
     SimulationConsumerResultModel,
     SimulationIterationResultModel,
@@ -80,3 +85,44 @@ def to_simulation_detail(simulation: SimulationModel) -> SimulationDetail:
         has_timeseries=simulation.result_storage_key is not None,
         key_result=(to_key_result_schema(simulation.key_result) if simulation.key_result else None),
     )
+
+
+def to_crm_data_preview(preflight: Preflight, locale: str) -> CrmDataPreview:
+    """Render a pre-flight verdict for the manager's screen.
+
+    Blocker messages are localised here rather than in the service so that
+    translation stays at the API edge — the worker reaches the same verdict via
+    ``Preflight`` and needs no locale at all.
+    """
+    summary = preflight.summary
+    return CrmDataPreview(
+        can_simulate=preflight.ok,
+        matched_participants=preflight.matched,
+        unmatched_participants=preflight.unmatched,
+        meter_count=len(summary.eans),
+        reading_count=summary.total_rows,
+        first_timestamp=summary.first_timestamp,
+        last_timestamp=summary.last_timestamp,
+        total_consumption_kwh=summary.total_consumption_kwh,
+        total_injection_kwh=summary.total_injection_kwh,
+        # Only participants are reported: a gap in a meter this key ignores
+        # changes nothing about the simulation.
+        incomplete_meters=[
+            IncompleteMeter(
+                ean=e.ean,
+                readings=e.distinct_ts,
+                expected=summary.grid_size,
+                missing=summary.grid_size - e.distinct_ts,
+            )
+            for e in summary.incomplete
+            if e.ean in set(preflight.matched)
+        ],
+        blockers=[
+            PreviewBlocker(
+                error_code=b.error.code,
+                message=translate(b.error.key, locale=locale),
+                detail=b.detail,
+            )
+            for b in preflight.blockers
+        ],
+    )

api/simulation/routes.py

@@ -1,9 +1,13 @@
+from datetime import date
 from typing import Annotated, Any
 
-from fastapi import APIRouter, Depends, File, Form, Query, UploadFile
+from fastapi import APIRouter, Body, Depends, File, Form, Query, UploadFile
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from api.simulation.mappers import to_crm_data_preview
 from api.simulation.schemas import (
+    CrmDataPreview,
+    SimulateFromCrmRequest,
     SimulateRequest,
     SimulateResponse,
     Simulation,
@@ -12,7 +16,7 @@
 )
 from api.simulation.service import SimulationService
 from core.api_response import ApiResponse, ApiResponsePaginated
-from core.context_vars import current_internal_community_id
+from core.context_vars import current_internal_community_id, current_locale
 from core.database.database import get_crm_session, get_local_session
 from core.errors.errors import ErrorException
 from core.errors.with_default_error import with_default_error
@@ -45,6 +49,37 @@ async def get_simulations(
     return ApiResponsePaginated[list[Simulation]](data=data, pagination=pagination)
 
 
+# GET (/crm-data-preview) : What the CRM holds for an operation + period, and
+# whether the key's participants match those meters.
+#
+# Declared BEFORE `GET /{id}`: FastAPI matches in declaration order, so putting
+# this after the integer catch-all would make "/crm-data-preview" try to parse
+# as an id and 422.
+@simulation_routes.get("/crm-data-preview", response_model=ApiResponse[CrmDataPreview])
+@with_default_error(default_error=errors.simulation.GET_CRM_PREVIEW)
+async def get_crm_data_preview(
+    local_session: Annotated[AsyncSession, Depends(get_local_session)],
+    crm_session: Annotated[AsyncSession, Depends(get_crm_session)],
+    id_key: Annotated[int, Query(description="CRM allocation key to be simulated.")],
+    id_sharing_operation: Annotated[int, Query(description="CRM sharing operation id.")],
+    period_start: Annotated[date, Query(description="First day of the period (inclusive).")],
+    period_end: Annotated[date, Query(description="Last day of the period (inclusive).")],
+):
+    internal_community_id = current_internal_community_id.get()
+    if internal_community_id is None:
+        raise ErrorException(error=errors.auth.UNAUTHORIZED, status_code=401)
+    service = SimulationService(local_session, crm_session)
+    preflight = await service.preview_crm_data(
+        id_key=id_key,
+        id_sharing_operation=id_sharing_operation,
+        period_start=period_start,
+        period_end=period_end,
+        community_id=internal_community_id,
+    )
+    locale = current_locale.get().split("_")[0]
+    return ApiResponse[CrmDataPreview](data=to_crm_data_preview(preflight, locale))
+
+
 # GET (/{id}) : One simulation with its scalar result tree.
 @simulation_routes.get("/{id}", response_model=ApiResponse[SimulationDetail])
 @with_default_error(default_error=errors.simulation.GET_SIMULATION)
@@ -95,6 +130,25 @@ async def start_simulation(
     return ApiResponse[SimulateResponse](data=data)
 
 
+# POST (/from-crm) Simulate against CRM meter data
+#
+# Plain JSON, unlike POST / — with no file part nothing forces multipart. It
+# also correctly keeps the default 2 MB body cap rather than the upload one.
+@simulation_routes.post("/from-crm", response_model=ApiResponse[SimulateResponse])
+@with_default_error(default_error=errors.simulation.START_SIMULATION)
+async def start_simulation_from_crm(
+    body: Annotated[SimulateFromCrmRequest, Body()],
+    local_session: Annotated[AsyncSession, Depends(get_local_session)],
+    crm_session: Annotated[AsyncSession, Depends(get_crm_session)],
+):
+    internal_community_id = current_internal_community_id.get()
+    if internal_community_id is None:
+        raise ErrorException(error=errors.auth.UNAUTHORIZED, status_code=401)
+    service = SimulationService(local_session, crm_session)
+    data = await service.start_simulation_from_crm(body, internal_community_id)
+    return ApiResponse[SimulateResponse](data=data)
+
+
 # DELETE (/{id}) : Delete a simulation and its results.
 @simulation_routes.delete("/{id}", response_model=ApiResponse[str])
 @with_default_error(default_error=errors.simulation.DELETE_SIMULATION)

api/simulation/schemas.py

@@ -1,3 +1,5 @@
+import datetime
+
 from pydantic import BaseModel, Field
 
 from shared.const import SimulationStatus
@@ -107,3 +109,64 @@ class SimulateRequest(BaseModel):
 class SimulateResponse(BaseModel):
     id: int = Field(..., description="ID of the freshly created simulation row.")
     status: SimulationStatus = Field(..., description="Initial status (PENDING on success).")
+
+
+# ---------------------------------------------------------------------------
+# CRM-sourced simulation (source = DataSource.CRM)
+# ---------------------------------------------------------------------------
+
+
+class SimulateFromCrmRequest(BaseModel):
+    """Body of ``POST /from-crm``.
+
+    Unlike ``SimulateRequest`` this *is* the FastAPI body model: with no file
+    part there is nothing forcing multipart, so the request is plain JSON.
+
+    ``injection_name`` is deliberately absent — the production profile is summed
+    from the meters themselves, which is the whole reason this path is simpler
+    for the user than uploading a file.
+    """
+
+    name: str = Field(..., min_length=1, description="User-facing label for the simulation.")
+    id_key: int = Field(..., description="CRM allocation key id to stress-test.")
+    id_sharing_operation: int = Field(..., description="CRM sharing operation to read meters from.")
+    period_start: datetime.date = Field(..., description="First day of the period (inclusive).")
+    period_end: datetime.date = Field(..., description="Last day of the period (inclusive).")
+
+
+class IncompleteMeter(BaseModel):
+    """A participant's meter missing part of the period. Zero-filled, not fatal."""
+
+    ean: str
+    readings: int = Field(..., description="Distinct timestamps this meter actually has.")
+    expected: int = Field(..., description="Distinct timestamps across the whole operation.")
+    missing: int = Field(..., description="expected - readings.")
+
+
+class PreviewBlocker(BaseModel):
+    """A reason the period cannot be used, already localised."""
+
+    error_code: int = Field(..., description="Matches the error_code of the eventual 4xx.")
+    message: str = Field(..., description="Localised, manager-facing explanation.")
+    detail: str = Field(..., description="Which meters/participants triggered it.")
+
+
+class CrmDataPreview(BaseModel):
+    """What ``GET /crm-data-preview`` shows before the manager commits to a run.
+
+    ``matched`` / ``unmatched`` are the heart of it: the key's participant names
+    must be meter EANs, and this is where the manager finds out that they are
+    not, rather than after a failed run.
+    """
+
+    can_simulate: bool
+    matched_participants: list[str]
+    unmatched_participants: list[str]
+    meter_count: int = Field(..., description="Meters with readings in the period.")
+    reading_count: int
+    first_timestamp: datetime.datetime | None
+    last_timestamp: datetime.datetime | None
+    total_consumption_kwh: float
+    total_injection_kwh: float
+    incomplete_meters: list[IncompleteMeter]
+    blockers: list[PreviewBlocker]