diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 87c1eb52..332af000 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -93,6 +93,20 @@ jobs: run: | cd server/installer cp .env.example .env + mkdir -p .ci-slicks/src/slicks + cat > .ci-slicks/pyproject.toml <<'EOF' + [project] + name = "slicks" + version = "0.0.0" + [build-system] + requires = ["setuptools>=68"] + build-backend = "setuptools.build_meta" + [tool.setuptools.packages.find] + where = ["src"] + EOF + touch .ci-slicks/README.md + echo '__version__ = "0.0.0"' > .ci-slicks/src/slicks/__init__.py + echo "SLICKS_HOST_PATH=${PWD}/.ci-slicks" >> .env - name: Pull pre-built images run: | diff --git a/server/installer/.env.example b/server/installer/.env.example index 6b677908..71983d5a 100644 --- a/server/installer/.env.example +++ b/server/installer/.env.example @@ -4,12 +4,15 @@ DBC_FILE_PATH=example.dbc # ------------------------------------------------------------ -# File uploader — team DBCs from GitHub (optional) +# Team DBC from GitHub (used by file-uploader and data-downloader) # ------------------------------------------------------------ # Fine-grained PAT or classic PAT with contents:read on Western-Formula-Racing/DBC GITHUB_DBC_TOKEN= # GITHUB_DBC_REPO=Western-Formula-Racing/DBC # GITHUB_DBC_BRANCH=main +# Data-downloader auto-selects the most recently committed .dbc in the repo. +# Set GITHUB_DBC_PATH to pin a specific file instead (e.g. WFR26.dbc). +# GITHUB_DBC_PATH= # Optional limits for .zip uploads (file-uploader); defaults are generous for team use # UPLOAD_ZIP_MAX_ARCHIVE_BYTES=2147483648 @@ -86,10 +89,19 @@ SENSOR_LOOKBACK_DAYS=365 SCAN_INTERVAL_SECONDS=3600 VITE_API_BASE_URL=http://localhost:8000 -ALLOWED_ORIGINS=http://localhost:3000,http://localhost:5173 +ALLOWED_ORIGINS=http://localhost:3000,http://localhost:5173,https://daq.westernformularacing.org # End Data Downloader configuration +# ------------------------------------------------------------ +# Health monitor +# ------------------------------------------------------------ +# All defaults match the standard docker-compose stack; only override if needed. +# HEALTH_MONITOR_INTERVAL_SECONDS=60 +# HEALTH_MONITOR_TIMESCALEDB_CONTAINER=timescaledb +# HEALTH_MONITOR_SCANNER_CONTAINER=data-downloader-scanner +# HEALTH_MONITOR_SCANNER_API_URL=http://data-downloader-api:8000 + # ------------------------------------------------------------ # AI Code Generation — MiniMax (Anthropic-compatible SDK) # ------------------------------------------------------------ diff --git a/server/installer/data-downloader/.env.example b/server/installer/data-downloader/.env.example deleted file mode 100644 index a7cb1b8f..00000000 --- a/server/installer/data-downloader/.env.example +++ /dev/null @@ -1,20 +0,0 @@ -POSTGRES_DSN=postgresql://wfr:wfr_password@timescaledb:5432/wfr -DEFAULT_SEASON_TABLE=wfr25 -DATA_DIR=/app/data -SCANNER_YEAR=2025 -SCANNER_BIN=hour -SCANNER_INCLUDE_COUNTS=true -SCANNER_INITIAL_CHUNK_DAYS=31 -SENSOR_WINDOW_DAYS=7 -SENSOR_LOOKBACK_DAYS=30 -SCAN_INTERVAL_SECONDS=3600 -VITE_API_BASE_URL=http://localhost:8000 -ALLOWED_ORIGINS=http://localhost:3000,http://localhost:5173,https://daq.westernformularacing.org - -# Health monitor (optional — defaults work for standard docker-compose stack) -TIMESCALE_HEALTH_TABLE=monitoring -HEALTH_MONITOR_INTERVAL_SECONDS=60 -HEALTH_MONITOR_TIMESCALE_CONTAINER=timescaledb -HEALTH_MONITOR_SCANNER_CONTAINER=data-downloader-scanner -HEALTH_MONITOR_SCANNER_API_URL=http://data-downloader-api:8000 -HEALTH_MONITOR_TIMESCALE_VOLUME_SUFFIX=timescaledb-data diff --git a/server/installer/data-downloader/backend/app.py b/server/installer/data-downloader/backend/app.py index 23dfd4a5..40bde0db 100644 --- a/server/installer/data-downloader/backend/app.py +++ b/server/installer/data-downloader/backend/app.py @@ -15,6 +15,7 @@ from pydantic import BaseModel from backend.config import get_settings +from backend.dbc_utils import group_sensors_by_message, load_dbc_db, refresh_dbc from backend.services import DataDownloaderService @@ -124,6 +125,36 @@ def list_sensors(season: str | None = None) -> dict: return service.get_sensors(season=season) +@app.get("/api/sensors/grouped") +def list_sensors_grouped(season: str | None = None) -> dict: + """ + Return sensors grouped by their DBC CAN message and transmitter node. + + Each entry in ``messages`` contains only signals that actually have data in + TimescaleDB (left-join: DB is truth, DBC is the categorisation guide). + Sensors with no matching DBC entry appear in ``ungrouped``. + + Falls back gracefully to ``{"messages": [], "ungrouped": all_sensors}`` + when no DBC is configured. + """ + sensor_payload = service.get_sensors(season=season) + sensor_names: list[str] = sensor_payload.get("sensors", []) + db, source = load_dbc_db(settings) + grouped = group_sensors_by_message(sensor_names, db) + return { + "updated_at": sensor_payload.get("updated_at"), + "dbc_source": source, + **grouped, + } + + +@app.post("/api/dbc/refresh") +def dbc_refresh() -> dict: + """Force-reload the DBC from GitHub (or local file). Returns the new source.""" + source = refresh_dbc(settings) + return {"status": "ok", "dbc_source": source} + + @app.get("/api/scanner-status") def scanner_status() -> dict: return service.get_scanner_status() diff --git a/server/installer/data-downloader/backend/config.py b/server/installer/data-downloader/backend/config.py index 5c7fbcef..5b4d47dd 100644 --- a/server/installer/data-downloader/backend/config.py +++ b/server/installer/data-downloader/backend/config.py @@ -110,6 +110,25 @@ class Settings(BaseModel): default_factory=lambda: _parse_origins(os.getenv("ALLOWED_ORIGINS", "*")) ) + # DBC source — GitHub takes priority over local file + github_dbc_token: str = Field( + default_factory=lambda: os.getenv("GITHUB_DBC_TOKEN", "") + ) + github_dbc_repo: str = Field( + default_factory=lambda: os.getenv("GITHUB_DBC_REPO", "Western-Formula-Racing/DBC") + ) + github_dbc_branch: str = Field( + default_factory=lambda: os.getenv("GITHUB_DBC_BRANCH", "main") + ) + # Specific file path within the repo, e.g. "WFR26.dbc" + github_dbc_path: str = Field( + default_factory=lambda: os.getenv("GITHUB_DBC_PATH", "") + ) + # Fallback: local DBC file path + dbc_file_path: str | None = Field( + default_factory=lambda: os.getenv("DBC_FILE_PATH") or None + ) + @lru_cache(maxsize=1) def get_settings() -> Settings: diff --git a/server/installer/data-downloader/backend/dbc_utils.py b/server/installer/data-downloader/backend/dbc_utils.py new file mode 100644 index 00000000..11da945f --- /dev/null +++ b/server/installer/data-downloader/backend/dbc_utils.py @@ -0,0 +1,262 @@ +""" +dbc_utils.py — DBC acquisition and sensor grouping for data-downloader. + +Priority order for DBC source: + 1. GitHub (GITHUB_DBC_TOKEN + GITHUB_DBC_PATH are set) + 2. Local file (DBC_FILE_PATH is set) + 3. None → grouped endpoint returns everything as ungrouped + +The loaded cantools DB is cached as a module-level singleton. +Call refresh_dbc(settings) to bust the cache (e.g. after a DBC push). +""" +from __future__ import annotations + +import logging +import threading +from pathlib import Path +from typing import Optional +from urllib.parse import quote + +logger = logging.getLogger(__name__) + +_VECTOR_PLACEHOLDER = "Vector__XXX" + +_db_lock = threading.Lock() +_db_cache = None # cantools.database.Database | None +_db_source: str = "none" # "github" | "github-cached" | "file" | "none" + + +# ── GitHub helpers ──────────────────────────────────────────────────────────── + +def _github_headers(token: str) -> dict: + h = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + if token: + h["Authorization"] = f"Bearer {token}" + return h + + +def _list_github_dbc_paths(token: str, repo: str, branch: str) -> list[str]: + """Return all .dbc blob paths in the repo.""" + import requests + url = f"https://api.github.com/repos/{repo}/git/trees/{branch}?recursive=1" + r = requests.get(url, headers=_github_headers(token), timeout=20) + r.raise_for_status() + tree = r.json().get("tree", []) + return [ + item["path"] for item in tree + if item.get("type") == "blob" and item["path"].lower().endswith(".dbc") + ] + + +def _last_commit_date(token: str, repo: str, branch: str, path: str) -> str: + """Return the ISO-8601 committer date of the most recent commit touching path.""" + import requests + enc = quote(path, safe="") + url = f"https://api.github.com/repos/{repo}/commits?path={enc}&sha={branch}&per_page=1" + r = requests.get(url, headers=_github_headers(token), timeout=20) + r.raise_for_status() + commits = r.json() + if not commits: + return "" + return commits[0].get("commit", {}).get("committer", {}).get("date", "") + + +def _fetch_github_dbc_bytes(token: str, repo: str, branch: str, path: str) -> bytes: + import requests + enc = quote(path, safe="") + url = f"https://api.github.com/repos/{repo}/contents/{enc}?ref={branch}" + headers = {**_github_headers(token), "Accept": "application/vnd.github.raw"} + r = requests.get(url, headers=headers, timeout=120) + r.raise_for_status() + return r.content + + +def _resolve_github_dbc_path(token: str, repo: str, branch: str, explicit_path: str) -> str: + """ + Return the DBC path to download. + - If explicit_path is set, use it directly. + - Otherwise find all .dbc files in the repo and return the one most + recently modified (by last commit date), using one commits API call + per file. + """ + if explicit_path: + return explicit_path + paths = _list_github_dbc_paths(token, repo, branch) + if not paths: + raise ValueError(f"No .dbc files found in {repo}@{branch}") + if len(paths) == 1: + return paths[0] + # Fetch last-commit date for each and pick the most recent + dated = [] + for p in paths: + try: + date = _last_commit_date(token, repo, branch, p) + except Exception: + date = "" + dated.append((date, p)) + logger.debug("DBC candidate: %s last commit: %s", p, date or "unknown") + dated.sort(key=lambda x: x[0], reverse=True) + chosen = dated[0][1] + logger.info( + "Auto-selected most recently modified DBC: %s (last commit: %s)", + chosen, dated[0][0] or "unknown", + ) + return chosen + + +# ── DB load / cache ─────────────────────────────────────────────────────────── + +def load_dbc_db(settings) -> tuple[Optional[object], str]: + """Return (cantools_db, source_label). Source: github | github-cached | file | none.""" + global _db_cache, _db_source + with _db_lock: + if _db_cache is not None: + return _db_cache, _db_source + return _reload_locked(settings) + + +def refresh_dbc(settings) -> str: + """Bust the cache and re-load. Returns the new source label.""" + global _db_cache, _db_source + with _db_lock: + _db_cache = None + _db_source = "none" + _, source = _reload_locked(settings) + return source + + +def _reload_locked(settings) -> tuple[Optional[object], str]: + """Must be called with _db_lock held.""" + global _db_cache, _db_source + try: + import cantools + except ImportError: + logger.warning("cantools not installed — DBC grouping disabled") + return None, "none" + + cache_path = Path(settings.data_dir) / "dbc_cache.dbc" + + # 1. GitHub — token alone is sufficient; path is auto-discovered if not set + if settings.github_dbc_token: + try: + path = _resolve_github_dbc_path( + settings.github_dbc_token, + settings.github_dbc_repo, + settings.github_dbc_branch, + settings.github_dbc_path, + ) + raw = _fetch_github_dbc_bytes( + settings.github_dbc_token, + settings.github_dbc_repo, + settings.github_dbc_branch, + path, + ) + cache_path.parent.mkdir(parents=True, exist_ok=True) + cache_path.write_bytes(raw) + db = cantools.database.load_file(str(cache_path)) + _db_cache, _db_source = db, "github" + logger.info("DBC loaded from GitHub: %s/%s", settings.github_dbc_repo, settings.github_dbc_path) + return db, "github" + except Exception as exc: + logger.warning("GitHub DBC fetch failed (%s); trying disk cache", exc) + if cache_path.exists(): + try: + db = cantools.database.load_file(str(cache_path)) + _db_cache, _db_source = db, "github-cached" + logger.info("DBC loaded from disk cache: %s", cache_path) + return db, "github-cached" + except Exception as exc2: + logger.warning("Disk cache load failed: %s", exc2) + + # 2. Local file + if settings.dbc_file_path: + fp = Path(settings.dbc_file_path) + if fp.exists(): + try: + db = cantools.database.load_file(str(fp)) + _db_cache, _db_source = db, "file" + logger.info("DBC loaded from file: %s", fp) + return db, "file" + except Exception as exc: + logger.warning("Local DBC load failed (%s): %s", fp, exc) + + _db_cache, _db_source = None, "none" + return None, "none" + + +# ── Grouping ────────────────────────────────────────────────────────────────── + +def _subsystem_for_message(message) -> str: + """ + Prefer the transmitter node declared in the DBC (message.senders[0]). + Falls back to the first '_'-delimited prefix of the message name. + """ + senders = getattr(message, "senders", None) or [] + for sender in senders: + if sender and sender != _VECTOR_PLACEHOLDER: + return sender.upper() + name: str = message.name + return name.split("_")[0].upper() if "_" in name else name.upper() + + +def group_sensors_by_message(sensor_names: list[str], db) -> dict: + """ + Left-join sensor_names (DB truth) against DBC message/signal tree. + + Only sensors that exist in the DB are included; DBC signals with no DB + data are silently ignored. + + Returns:: + + { + "messages": [ + { + "name": "BMS_Current_Limit", + "subsystem": "MOBO", + "can_id": 514, + "can_id_hex": "0x202", + "signals": ["BMS_Max_Charge_Current", "BMS_Max_Discharge_Current"] + }, + ... + ], + "ungrouped": ["GPS_Lat", ...] # DB sensors with no DBC entry + } + """ + if db is None: + return {"messages": [], "ungrouped": sorted(sensor_names)} + + # Build signal_name → (message, subsystem) lookup + signal_to_msg: dict[str, tuple] = {} + for message in db.messages: + subsystem = _subsystem_for_message(message) + for signal in message.signals: + signal_to_msg[signal.name] = (message, subsystem) + + # Group DB sensors by their DBC message + msg_groups: dict[str, dict] = {} + ungrouped: list[str] = [] + + for sensor in sensor_names: + if sensor in signal_to_msg: + message, subsystem = signal_to_msg[sensor] + key = message.name + if key not in msg_groups: + msg_groups[key] = { + "name": message.name, + "subsystem": subsystem, + "can_id": message.frame_id, + "can_id_hex": f"0x{message.frame_id:03X}", + "signals": [], + } + msg_groups[key]["signals"].append(sensor) + else: + ungrouped.append(sensor) + + messages = sorted(msg_groups.values(), key=lambda m: m["can_id"]) + for m in messages: + m["signals"].sort() + + return {"messages": messages, "ungrouped": sorted(ungrouped)} diff --git a/server/installer/data-downloader/backend/requirements.txt b/server/installer/data-downloader/backend/requirements.txt index bf9a6227..3d934fa5 100644 --- a/server/installer/data-downloader/backend/requirements.txt +++ b/server/installer/data-downloader/backend/requirements.txt @@ -3,3 +3,5 @@ uvicorn[standard]==0.23.2 psycopg2-binary>=2.9.9 pydantic==2.9.2 docker>=7.0.0 +cantools>=39.0.0 +requests>=2.31.0 diff --git a/server/installer/data-downloader/frontend/src/App.tsx b/server/installer/data-downloader/frontend/src/App.tsx index d4f88cc1..a58c2bd2 100644 --- a/server/installer/data-downloader/frontend/src/App.tsx +++ b/server/installer/data-downloader/frontend/src/App.tsx @@ -1,9 +1,10 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { fetchRuns, fetchSensors, fetchScannerStatus, triggerScan, updateNote, fetchSeasons } from "./api"; +import { fetchRuns, fetchSensors, fetchSensorsGrouped, fetchScannerStatus, triggerScan, updateNote, fetchSeasons } from "./api"; import { Moon, Sun } from "lucide-react"; -import { RunRecord, RunsResponse, ScannerStatus, SensorsResponse, Season } from "./types"; +import { RunRecord, RunsResponse, ScannerStatus, SensorsGroupedResponse, SensorsResponse, Season } from "./types"; import { RunTable } from "./components/RunTable"; import { DataDownload } from "./components/data-download"; +import { SensorGroupedGrid } from "./components/SensorGroupedGrid"; type ScanState = "idle" | "running" | "success" | "error"; type Theme = "light" | "dark"; @@ -28,6 +29,7 @@ export default function App() { const [selectedSeason, setSelectedSeason] = useState(""); // season name const [runs, setRuns] = useState(null); const [sensors, setSensors] = useState(null); + const [sensorsGrouped, setSensorsGrouped] = useState(null); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); const [noteDrafts, setNoteDrafts] = useState>({}); @@ -61,12 +63,14 @@ export default function App() { // If we still don't have a season (e.g. no seasons configured), fetch with default (undefined) const seasonArg = currentSeason || undefined; - const [runsData, sensorsData] = await Promise.all([ + const [runsData, sensorsData, groupedData] = await Promise.all([ fetchRuns(seasonArg), - fetchSensors(seasonArg) + fetchSensors(seasonArg), + fetchSensorsGrouped(seasonArg).catch(() => null), ]); setRuns(runsData); setSensors(sensorsData); + setSensorsGrouped(groupedData); setError(null); } catch (err) { console.error(err); @@ -359,10 +363,23 @@ export default function App() {
-

Unique Sensors

-

Last refresh: {lastSensorRefresh}

+
+

Unique Sensors

+ {sensorsGrouped?.dbc_source && sensorsGrouped.dbc_source !== "none" && ( + + DBC: {sensorsGrouped.dbc_source} + + )} +
+

Last refresh: {lastSensorRefresh}

{loading && !sensors ? (

Loading sensors...

+ ) : sensorsGrouped && sensorsGrouped.messages.length > 0 ? ( + ) : (
{sensorsPreview.length === 0 &&

No sensors captured.

} @@ -384,6 +401,7 @@ export default function App() { { return request(`/api/sensors${query}`); } +export function fetchSensorsGrouped(season?: string): Promise { + const query = season ? `?season=${encodeURIComponent(season)}` : ""; + return request(`/api/sensors/grouped${query}`); +} + +export function refreshDbc(): Promise<{ status: string; dbc_source: string }> { + return request("/api/dbc/refresh", { method: "POST" }); +} + export function fetchScannerStatus(): Promise { return request("/api/scanner-status"); } diff --git a/server/installer/data-downloader/frontend/src/components/SensorGroupedGrid.tsx b/server/installer/data-downloader/frontend/src/components/SensorGroupedGrid.tsx new file mode 100644 index 00000000..faa2615b --- /dev/null +++ b/server/installer/data-downloader/frontend/src/components/SensorGroupedGrid.tsx @@ -0,0 +1,211 @@ +import { useState, type ReactNode } from "react"; +import { MessageGroup, SensorsGroupedResponse } from "../types"; + +// ── Subsystem colour palette (8 hues, light + dark) ────────────────────────── + +interface PaletteEntry { + border: string; + bg: string; + badgeBg: string; + badgeText: string; +} + +const PALETTE: Array<{ light: PaletteEntry; dark: PaletteEntry }> = [ + // 0 Blue + { + light: { border: "#2563eb", bg: "#eef2ff", badgeBg: "#dbeafe", badgeText: "#1d4ed8" }, + dark: { border: "#60a5fa", bg: "#1e2638", badgeBg: "#1e3150", badgeText: "#93c5fd" }, + }, + // 1 Orange + { + light: { border: "#c2410c", bg: "#fff7ed", badgeBg: "#ffedd5", badgeText: "#9a3412" }, + dark: { border: "#fb923c", bg: "#2c1a10", badgeBg: "#3a2010", badgeText: "#fdba74" }, + }, + // 2 Teal + { + light: { border: "#0d9488", bg: "#f0fdfa", badgeBg: "#ccfbf1", badgeText: "#0f766e" }, + dark: { border: "#2dd4bf", bg: "#0d2522", badgeBg: "#123230", badgeText: "#5eead4" }, + }, + // 3 Violet + { + light: { border: "#7c3aed", bg: "#f5f3ff", badgeBg: "#ede9fe", badgeText: "#5b21b6" }, + dark: { border: "#a78bfa", bg: "#1e1530", badgeBg: "#261b3e", badgeText: "#c4b5fd" }, + }, + // 4 Amber + { + light: { border: "#b45309", bg: "#fffbeb", badgeBg: "#fef3c7", badgeText: "#92400e" }, + dark: { border: "#fbbf24", bg: "#2a1f0a", badgeBg: "#352710", badgeText: "#fde68a" }, + }, + // 5 Green + { + light: { border: "#15803d", bg: "#f0fdf4", badgeBg: "#dcfce7", badgeText: "#14532d" }, + dark: { border: "#4ade80", bg: "#0d2118", badgeBg: "#122818", badgeText: "#86efac" }, + }, + // 6 Rose + { + light: { border: "#be185d", bg: "#fdf2f8", badgeBg: "#fce7f3", badgeText: "#9d174d" }, + dark: { border: "#f472b6", bg: "#2c1020", badgeBg: "#3a1428", badgeText: "#f9a8d4" }, + }, + // 7 Cyan + { + light: { border: "#0e7490", bg: "#ecfeff", badgeBg: "#cffafe", badgeText: "#155e75" }, + dark: { border: "#22d3ee", bg: "#0a2028", badgeBg: "#0f2a38", badgeText: "#67e8f9" }, + }, +]; + +/** djb2 hash → stable palette index for a subsystem name. */ +function paletteIndex(subsystem: string): number { + let h = 5381; + const s = subsystem.toUpperCase(); + for (let i = 0; i < s.length; i++) { + h = (((h << 5) + h) ^ s.charCodeAt(i)) >>> 0; + } + return h % PALETTE.length; +} + +function subsystemColor(subsystem: string, theme: "light" | "dark"): PaletteEntry { + return PALETTE[paletteIndex(subsystem)][theme]; +} + +// ── Component ───────────────────────────────────────────────────────────────── + +interface Props { + grouped: SensorsGroupedResponse; + theme: "light" | "dark"; + onPick: (sensor: string) => void; +} + +interface GroupRowProps { + groupKey: string; + name: string; + signals: string[]; + colors: PaletteEntry; + count: number; + collapsed: boolean; + onToggle: () => void; + onPick: (s: string) => void; + badge?: ReactNode; +} + +function GroupRow({ groupKey, name, signals, colors, count, collapsed, onToggle, onPick, badge }: GroupRowProps) { + return ( +
+ + {!collapsed && ( +
+
+ {signals.map((signal) => ( + + ))} +
+
+ )} +
+ ); +} + +export function SensorGroupedGrid({ grouped, theme, onPick }: Props) { + // Track which groups are collapsed; default is all expanded so signals are + // visible (and findable via browser Ctrl+F) without clicking into each group. + const [collapsed, setCollapsed] = useState>(new Set()); + + const toggle = (key: string) => + setCollapsed((prev) => { + const next = new Set(prev); + if (next.has(key)) { + next.delete(key); + } else { + next.add(key); + } + return next; + }); + + const hasUngrouped = grouped.ungrouped.length > 0; + + return ( +
+ {grouped.messages.map((msg: MessageGroup) => { + const colors = subsystemColor(msg.subsystem, theme); + return ( + toggle(msg.name)} + onPick={onPick} + badge={ + <> + + {msg.subsystem} + + + {msg.can_id_hex} · {msg.can_id} + + + } + /> + ); + })} + + {hasUngrouped && ( + toggle("__ungrouped__")} + onPick={onPick} + /> + )} +
+ ); +} diff --git a/server/installer/data-downloader/frontend/src/components/data-download.tsx b/server/installer/data-downloader/frontend/src/components/data-download.tsx index f8060cf8..6e2996b5 100644 --- a/server/installer/data-downloader/frontend/src/components/data-download.tsx +++ b/server/installer/data-downloader/frontend/src/components/data-download.tsx @@ -4,7 +4,7 @@ import Papa from "papaparse"; import { Download } from "lucide-react"; import Plot from "react-plotly.js"; -import { RunRecord, SensorDataPoint, SensorDataResponse } from "../types"; +import { RunRecord, SensorDataPoint, SensorDataResponse, SensorsGroupedResponse } from "../types"; import { querySensorData } from "../api"; interface ExternalSelection { @@ -18,6 +18,7 @@ interface ExternalSelection { interface Props { runs: RunRecord[]; sensors: string[]; + sensorsGrouped?: SensorsGroupedResponse; season?: string; externalSelection?: ExternalSelection; theme?: "light" | "dark"; @@ -62,7 +63,7 @@ const toUtcTooltip = (value: string) => { return dt.isValid ? `${dt.toFormat("yyyy-LL-dd HH:mm:ss")} UTC` : value; }; -export function DataDownload({ runs, sensors, season, externalSelection, theme = "light" }: Props) { +export function DataDownload({ runs, sensors, sensorsGrouped, season, externalSelection, theme = "light" }: Props) { const [selectedRunKey, setSelectedRunKey] = useState(""); const [selectedRunTimezone, setSelectedRunTimezone] = useState(null); const [selectedSensor, setSelectedSensor] = useState(""); @@ -370,7 +371,24 @@ export function DataDownload({ runs, sensors, season, externalSelection, theme = value={selectedSensor} onChange={(event) => setSelectedSensor(event.target.value)} > - {sensors.length === 0 ? ( + {sensorsGrouped && sensorsGrouped.messages.length > 0 ? ( + <> + {sensorsGrouped.messages.map((msg) => ( + + {msg.signals.map((signal) => ( + + ))} + + ))} + {sensorsGrouped.ungrouped.length > 0 && ( + + {sensorsGrouped.ungrouped.map((signal) => ( + + ))} + + )} + + ) : sensors.length === 0 ? ( ) : ( sensors.map((sensor) => ( diff --git a/server/installer/data-downloader/frontend/src/styles.css b/server/installer/data-downloader/frontend/src/styles.css index f23e1fe8..c589b77e 100644 --- a/server/installer/data-downloader/frontend/src/styles.css +++ b/server/installer/data-downloader/frontend/src/styles.css @@ -453,3 +453,111 @@ body { font-weight: 600; box-shadow: 0 6px 20px -12px rgba(148, 107, 33, 0.6); } + +/* ── Grouped sensor browser ───────────────────────────────────────────────── */ + +.message-groups-container { + display: flex; + flex-direction: column; + gap: 0.4rem; + margin-top: 1rem; + max-height: 440px; + overflow-y: auto; + padding-right: 2px; +} + +.message-group { + flex-shrink: 0; + border: 1px solid var(--border); + border-radius: 10px; + background: var(--surface); + overflow: hidden; +} + +.message-group-header { + display: flex; + align-items: center; + gap: 0.5rem; + width: 100%; + padding: 0.5rem 0.7rem; + border: none; + border-left: 3px solid var(--border-strong); + background: transparent; + cursor: pointer; + text-align: left; + font: inherit; + color: var(--text); + transition: background 0.12s ease; + appearance: none; +} + +.message-group-header:hover { + background: var(--row-hover) !important; +} + +.message-group-header:focus-visible { + outline: 2px solid var(--accent); + outline-offset: -2px; +} + +.message-group-chevron { + font-size: 0.7rem; + color: var(--text-subtle); + flex-shrink: 0; + display: inline-block; + transition: transform 0.15s ease; + line-height: 1; +} + +.message-group-name { + font-weight: 600; + font-size: 0.88rem; + flex: 1; + min-width: 0; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.subsystem-badge { + flex-shrink: 0; + font-size: 0.72rem; + font-weight: 700; + letter-spacing: 0.04em; + padding: 0.15rem 0.45rem; + border-radius: 5px; + border: 1px solid transparent; + white-space: nowrap; +} + +.can-id-badge { + flex-shrink: 0; + font-family: ui-monospace, "Cascadia Code", "Fira Code", monospace; + font-size: 0.76rem; + padding: 0.15rem 0.45rem; + border-radius: 5px; + border: 1px solid transparent; + white-space: nowrap; + color: var(--text-muted); + background: var(--surface-2); +} + +.message-group-count { + flex-shrink: 0; + font-size: 0.72rem; + color: var(--text-subtle); + min-width: 1.4rem; + text-align: right; +} + +.message-group-body { + padding: 0.45rem 0.6rem 0.55rem; + border-top: 1px solid var(--border); + background: var(--surface-2); +} + +/* Compact chip grid inside a group — no max-height (container scrolls instead) */ +.sensor-grid--compact { + max-height: none; + margin-top: 0; +} diff --git a/server/installer/data-downloader/frontend/src/types.ts b/server/installer/data-downloader/frontend/src/types.ts index f379b2d3..e39195f7 100644 --- a/server/installer/data-downloader/frontend/src/types.ts +++ b/server/installer/data-downloader/frontend/src/types.ts @@ -52,3 +52,18 @@ export interface Season { database: string; color?: string; } + +export interface MessageGroup { + name: string; + subsystem: string; + can_id: number; + can_id_hex: string; + signals: string[]; +} + +export interface SensorsGroupedResponse { + updated_at: string | null; + dbc_source: string; + messages: MessageGroup[]; + ungrouped: string[]; +} diff --git a/server/installer/docker-compose.yml b/server/installer/docker-compose.yml index 635442ac..7f278448 100644 --- a/server/installer/docker-compose.yml +++ b/server/installer/docker-compose.yml @@ -171,6 +171,14 @@ services: environment: POSTGRES_DSN: "${POSTGRES_DSN:-postgresql://wfr:wfr_password@timescaledb:5432/wfr}" SEASONS: "${SEASONS:-}" + # DBC grouping — same token as file-uploader; most recently committed .dbc auto-selected + GITHUB_DBC_TOKEN: "${GITHUB_DBC_TOKEN:-}" + GITHUB_DBC_REPO: "${GITHUB_DBC_REPO:-Western-Formula-Racing/DBC}" + GITHUB_DBC_BRANCH: "${GITHUB_DBC_BRANCH:-main}" + # Optional: pin a specific file (e.g. WFR26.dbc); leave unset to auto-pick newest + GITHUB_DBC_PATH: "${GITHUB_DBC_PATH:-}" + # Fallback: local DBC file (mounted below or already on disk) + DBC_FILE_PATH: "${DBC_FILE_PATH:-}" ports: - "8000:8000" volumes: diff --git a/server/installer/sandbox/Dockerfile b/server/installer/sandbox/Dockerfile index efd5b32f..ebbd6810 100644 --- a/server/installer/sandbox/Dockerfile +++ b/server/installer/sandbox/Dockerfile @@ -6,13 +6,14 @@ WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -# Pre-download the FastEmbed ONNX model so first startup is instant -RUN python -c "from langchain_community.embeddings import FastEmbedEmbeddings; FastEmbedEmbeddings()" +# Best-effort FastEmbed ONNX prefetch. Do not fail image builds when external +# model hosts rate-limit CI; the service can download the model at runtime. +RUN python -c "from langchain_community.embeddings import FastEmbedEmbeddings; FastEmbedEmbeddings()" \ + || echo "FastEmbed model prefetch failed; continuing build" # Copy application code COPY code_generator.py . COPY stats_report.py . -COPY anomaly_scan.py . # Use the wildcard so an existing prompt-guide.txt is preferred; otherwise # fall back to .example. The previous version only copied .example, which # silently ignored the active prompt-guide.txt customization.