From 9360d2c1fde595326bde3350cd943d8e6703ad6a Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 12 Jun 2026 11:04:57 -0700 Subject: [PATCH 01/19] add Python/daily-insight: railtracks-powered status-page insight demo Ports the CSharp DailyInsightService functionality to a standalone Python FastAPI service. Drives an Anthropic Claude agent through Railtracks with a single `@rt.function_node` tool that calls the Railengine Python SDK directly (replaces the C# MCP attachment). POST /insight returns the same one-line-per- metric plain-text summary so the existing /api/insight card can render it unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- Python/daily-insight/.env.example | 12 +++ Python/daily-insight/README.md | 97 +++++++++++++++++++ Python/daily-insight/pyproject.toml | 30 ++++++ Python/daily-insight/src/__init__.py | 3 + Python/daily-insight/src/agents/__init__.py | 3 + .../daily-insight/src/agents/insight_agent.py | 42 ++++++++ Python/daily-insight/src/agents/tools.py | 22 +++++ Python/daily-insight/src/config/__init__.py | 3 + Python/daily-insight/src/config/env.py | 35 +++++++ .../daily-insight/src/controllers/__init__.py | 0 Python/daily-insight/src/controllers/api.py | 61 ++++++++++++ Python/daily-insight/src/models/__init__.py | 4 + Python/daily-insight/src/models/insight.py | 17 ++++ Python/daily-insight/src/models/metric.py | 17 ++++ .../src/repositories/__init__.py | 3 + .../src/repositories/metric_repository.py | 44 +++++++++ Python/daily-insight/src/services/__init__.py | 3 + .../src/services/insight_service.py | 50 ++++++++++ 18 files changed, 446 insertions(+) create mode 100644 Python/daily-insight/.env.example create mode 100644 Python/daily-insight/README.md create mode 100644 Python/daily-insight/pyproject.toml create mode 100644 Python/daily-insight/src/__init__.py create mode 100644 Python/daily-insight/src/agents/__init__.py create mode 100644 Python/daily-insight/src/agents/insight_agent.py create mode 100644 Python/daily-insight/src/agents/tools.py create mode 100644 Python/daily-insight/src/config/__init__.py create mode 100644 Python/daily-insight/src/config/env.py create mode 100644 Python/daily-insight/src/controllers/__init__.py create mode 100644 Python/daily-insight/src/controllers/api.py create mode 100644 Python/daily-insight/src/models/__init__.py create mode 100644 Python/daily-insight/src/models/insight.py create mode 100644 Python/daily-insight/src/models/metric.py create mode 100644 Python/daily-insight/src/repositories/__init__.py create mode 100644 Python/daily-insight/src/repositories/metric_repository.py create mode 100644 Python/daily-insight/src/services/__init__.py create mode 100644 Python/daily-insight/src/services/insight_service.py diff --git a/Python/daily-insight/.env.example b/Python/daily-insight/.env.example new file mode 100644 index 0000000..b228f6e --- /dev/null +++ b/Python/daily-insight/.env.example @@ -0,0 +1,12 @@ +# Retrieval (PAT + Engine ID for the same engine the status page reads from) +ENGINE_ID="[Your Engine ID]" +ENGINE_PAT="[Your Engine PAT]" + +# Optional: override API host (must match the stack ENGINE_PAT targets) +# RAILTOWN_API_URL="https://cndr.railtown.ai/api" + +# Railtracks LLM provider +ANTHROPIC_API_KEY="[Your Anthropic API key]" + +# Optional: override the default Claude model (defaults to claude-haiku-4-5-20251001 in code) +# INSIGHT_MODEL="claude-haiku-4-5-20251001" diff --git a/Python/daily-insight/README.md b/Python/daily-insight/README.md new file mode 100644 index 0000000..9a127ce --- /dev/null +++ b/Python/daily-insight/README.md @@ -0,0 +1,97 @@ +# Daily Insight (Railtracks) + +A small Python service that mirrors the **Daily Insight** feature from [`CSharp/Examples/RailenginePoweredStatusPage`](../../CSharp/Examples/RailenginePoweredStatusPage/): given a Railengine of metric records, ask an LLM to produce a one-line-per-metric plain-text summary of recent values. + +Where the C# version drives the LLM through an Anthropic MCP server attached as a tool source, this version drives it through [Railtracks](https://github.com/RailtownAI/railtracks) with a single `@rt.function_node` tool that calls the [Railengine Python SDK](https://pypi.org/project/rail-engine/) directly. + +## Before you start + +- A [Railengine](https://railengine.ai/) engine populated with `MetricRecord`-shaped documents (`metric`, `timestamp`, `value` — see [`MetricRecord`](src/models/metric.py)). The C# status page example produces records in exactly this shape. +- An Anthropic API key with access to `claude-haiku-4-5-20251001` (or another model you set via `INSIGHT_MODEL`). + +## Quick start + +```bash +cd Python/daily-insight +cp .env.example .env # fill ENGINE_ID, ENGINE_PAT, ANTHROPIC_API_KEY +uv sync +uv run uvicorn daily_insight.controllers.api:app --reload --port 8000 +``` + +Then: + +```bash +# Liveness +curl http://127.0.0.1:8000/health + +# Generate a fresh insight +curl -X POST http://127.0.0.1:8000/insight +``` + +`POST /insight` runs the Railtracks agent against your engine and returns: + +```json +{ + "text": "latency-p95: Latest reading 87.3 ms, trending down over the last 50 samples.\nerror-rate: Holding flat near 2.4 errors/min.\n...", + "generated_at": "2026-06-12T18:42:11.123456+00:00", + "metric_count": 4, + "error": null +} +``` + +Expect each call to take a few seconds — the agent makes one Anthropic call plus one Railengine GET. The exact latency depends on the model. + +## Environment variables + +| Variable | Required | Purpose | +|---|---|---| +| `ENGINE_ID` | Yes | Railengine engine GUID — same one the C# status page reads from | +| `ENGINE_PAT` | Yes | Railengine PAT used for retrieval | +| `ANTHROPIC_API_KEY` | Yes | Anthropic API key used by `rt.llm.AnthropicLLM` | +| `INSIGHT_MODEL` | No | Override the default Claude model (`claude-haiku-4-5-20251001`) | +| `RAILTOWN_API_URL` | No | Override the Railengine API host (defaults to production) | + +Variables are read from `.env` next to `pyproject.toml`, then from the process environment. + +## Project layout + +```text +controllers (FastAPI) → services → agents (Railtracks) → repositories → rail-engine + ↓ + models +``` + +| Path | Role | +|---|---| +| [`src/models/`](src/models/) | Pydantic types — `MetricRecord`, `DailyInsight` | +| [`src/repositories/`](src/repositories/) | [`MetricRepository`](src/repositories/metric_repository.py) — wraps `Railengine.list_storage_documents` | +| [`src/agents/`](src/agents/) | [`tools.py`](src/agents/tools.py) defines the `get_recent_metrics` function node; [`insight_agent.py`](src/agents/insight_agent.py) wires it to `rt.llm.AnthropicLLM` | +| [`src/services/`](src/services/) | [`InsightService`](src/services/insight_service.py) — runs the `rt.Flow` and shapes the response | +| [`src/controllers/`](src/controllers/) | [`api.py`](src/controllers/api.py) — FastAPI app with `/health` and `/insight` | +| [`src/config/`](src/config/) | `.env` loading and required-env validation | + +## How this differs from the C# version + +The C# [`DailyInsightService`](../../CSharp/Examples/RailenginePoweredStatusPage/Services/DailyInsightService.cs) is a `BackgroundService` that wakes once every 24 hours and posts to `/v1/messages` with the Railengine MCP server (`mcp_servers`) attached, then streams the response. The same prompt and output format are used here. + +This Python version: + +- Replaces the MCP attachment with a Railtracks `@rt.function_node` tool (`get_recent_metrics`) that calls the Railengine Python SDK directly. The LLM still chooses when to call it, but the schema and execution are local. +- Replaces the 24h `BackgroundService` with an on-demand HTTP endpoint. A scheduler (cron, GitHub Actions, Azure Logic Apps, etc.) can POST `/insight` daily if you want the same cadence. +- Keeps the strict plain-text output rules so the existing C# `Daily Insight` card can render the result unchanged. + +## Debug and visualize the agent (optional) + +After at least one insight run: + +```bash +cd Python/daily-insight +railtracks update +railtracks viz +``` + +(`railtracks[visual]` is in `pyproject.toml`; run `uv sync` if you have not already.) Opens the local visualization app for inspecting tool calls and prompts. + +## Local only + +Don't expose this service on the public internet without authentication — `/insight` triggers a paid LLM call and a Railengine read on every invocation. diff --git a/Python/daily-insight/pyproject.toml b/Python/daily-insight/pyproject.toml new file mode 100644 index 0000000..39d2a10 --- /dev/null +++ b/Python/daily-insight/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "daily-insight" +version = "0.1.0" +description = "Daily status-page insight demo: Railengine retrieval + Railtracks agent + FastAPI" +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "rail-engine>=0.2.1", + "railtracks[visual]>=1.3.0", + "pydantic>=2.0", + "python-dotenv>=1.0.0", + "fastapi>=0.110.0", + "uvicorn[standard]>=0.27.0", +] + +[tool.setuptools] +package-dir = { "daily_insight" = "src" } +packages = [ + "daily_insight", + "daily_insight.agents", + "daily_insight.config", + "daily_insight.controllers", + "daily_insight.models", + "daily_insight.repositories", + "daily_insight.services", +] diff --git a/Python/daily-insight/src/__init__.py b/Python/daily-insight/src/__init__.py new file mode 100644 index 0000000..81c697c --- /dev/null +++ b/Python/daily-insight/src/__init__.py @@ -0,0 +1,3 @@ +"""Daily status-page insight demo: Railengine + Railtracks + FastAPI.""" + +__version__ = "0.1.0" diff --git a/Python/daily-insight/src/agents/__init__.py b/Python/daily-insight/src/agents/__init__.py new file mode 100644 index 0000000..5919831 --- /dev/null +++ b/Python/daily-insight/src/agents/__init__.py @@ -0,0 +1,3 @@ +from daily_insight.agents.insight_agent import build_insight_agent + +__all__ = ["build_insight_agent"] diff --git a/Python/daily-insight/src/agents/insight_agent.py b/Python/daily-insight/src/agents/insight_agent.py new file mode 100644 index 0000000..c37591e --- /dev/null +++ b/Python/daily-insight/src/agents/insight_agent.py @@ -0,0 +1,42 @@ +"""Railtracks insight agent definition.""" + +from __future__ import annotations + +import os + +import railtracks as rt + +from daily_insight.agents.tools import get_recent_metrics + + +DEFAULT_MODEL = "claude-haiku-4-5-20251001" + +SYSTEM_PROMPT = """You are an automated reviewer for a status page dashboard. + +Use AT MOST 1 tool call. Call `get_recent_metrics` with limit=50 to fetch the most recent metric records, then summarize. Do not make additional exploratory calls. + +Output format (strict): +- One single line per metric, nothing else. +- Format: "{Metric name}: {one-sentence insight including the latest value and any notable trend}" +- Plain text only. No markdown, no headers, no bullets, no emoji, no bold. +- No preamble, no commentary about your process, no closing remarks. +- If a metric is flat, say so concisely.""" + + +def build_insight_agent(): + """Create the Railtracks insight agent backed by AnthropicLLM.""" + api_key = os.environ.get("ANTHROPIC_API_KEY", "").strip() + if not api_key: + raise RuntimeError( + "ANTHROPIC_API_KEY is not set. Add it to .env or export it before running." + ) + + model = os.environ.get("INSIGHT_MODEL", "").strip() or DEFAULT_MODEL + llm = rt.llm.AnthropicLLM(model) + + return rt.agent_node( + "Daily Insight Agent", + tool_nodes=(get_recent_metrics,), + llm=llm, + system_message=SYSTEM_PROMPT, + ) diff --git a/Python/daily-insight/src/agents/tools.py b/Python/daily-insight/src/agents/tools.py new file mode 100644 index 0000000..805d1a7 --- /dev/null +++ b/Python/daily-insight/src/agents/tools.py @@ -0,0 +1,22 @@ +"""Railtracks tool nodes backed by MetricRepository.""" + +from __future__ import annotations + +import json + +import railtracks as rt + +from daily_insight.repositories import MetricRepository + + +@rt.function_node +async def get_recent_metrics(limit: int = 50) -> str: + """ + Fetch the most recent metric records from Railengine. + + Args: + limit: Maximum number of records to return (capped at 100 by the SDK). + """ + repo = MetricRepository() + records = await repo.list_recent(limit=limit) + return json.dumps([r.model_dump() for r in records], ensure_ascii=False) diff --git a/Python/daily-insight/src/config/__init__.py b/Python/daily-insight/src/config/__init__.py new file mode 100644 index 0000000..4c3a33d --- /dev/null +++ b/Python/daily-insight/src/config/__init__.py @@ -0,0 +1,3 @@ +from daily_insight.config.env import ensure_dotenv_loaded, validate_required_env, MissingEnvVarsError + +__all__ = ["ensure_dotenv_loaded", "validate_required_env", "MissingEnvVarsError"] diff --git a/Python/daily-insight/src/config/env.py b/Python/daily-insight/src/config/env.py new file mode 100644 index 0000000..88fb8dd --- /dev/null +++ b/Python/daily-insight/src/config/env.py @@ -0,0 +1,35 @@ +"""Load `.env` and validate required environment variables.""" + +from __future__ import annotations + +import os +from pathlib import Path + +from dotenv import load_dotenv + + +REQUIRED_ENV_VARS = ("ENGINE_PAT", "ENGINE_ID", "ANTHROPIC_API_KEY") + + +class MissingEnvVarsError(RuntimeError): + def __init__(self, missing: list[str]) -> None: + self.missing = missing + names = ", ".join(missing) + super().__init__( + f"Missing required environment variables: {names}. " + f"Set them in .env (see .env.example)." + ) + + +def ensure_dotenv_loaded() -> None: + """Load env from `Python/daily-insight/.env`, then cwd `.env` (override).""" + project_root = Path(__file__).resolve().parents[2] + load_dotenv(project_root / ".env") + load_dotenv() + + +def validate_required_env() -> None: + """Raise MissingEnvVarsError if any required variable is unset or blank.""" + missing = [name for name in REQUIRED_ENV_VARS if not os.environ.get(name, "").strip()] + if missing: + raise MissingEnvVarsError(missing) diff --git a/Python/daily-insight/src/controllers/__init__.py b/Python/daily-insight/src/controllers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Python/daily-insight/src/controllers/api.py b/Python/daily-insight/src/controllers/api.py new file mode 100644 index 0000000..395ea60 --- /dev/null +++ b/Python/daily-insight/src/controllers/api.py @@ -0,0 +1,61 @@ +"""FastAPI service that runs the daily insight agent on demand. + +Run:: + + uv run uvicorn daily_insight.controllers.api:app --reload --port 8000 + +Endpoints: +- GET /health — liveness probe +- POST /insight — run the Railtracks agent against Railengine and return a DailyInsight +""" + +from __future__ import annotations + +import logging +from contextlib import asynccontextmanager + +from fastapi import FastAPI, HTTPException + +from daily_insight.config import ( + MissingEnvVarsError, + ensure_dotenv_loaded, + validate_required_env, +) +from daily_insight.models import DailyInsight +from daily_insight.services import InsightService + + +logger = logging.getLogger(__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + ensure_dotenv_loaded() + try: + validate_required_env() + except MissingEnvVarsError as exc: + logger.error(str(exc)) + raise + yield + + +app = FastAPI( + title="Daily Insight", + description="Railtracks-powered status-page insight over Railengine metrics.", + version="0.1.0", + lifespan=lifespan, +) + + +@app.get("/health") +async def health() -> dict[str, str]: + return {"status": "ok"} + + +@app.post("/insight", response_model=DailyInsight) +async def generate_insight() -> DailyInsight: + try: + return await InsightService().run() + except Exception as exc: + logger.exception("Failed to generate daily insight") + raise HTTPException(status_code=500, detail=str(exc)) from exc diff --git a/Python/daily-insight/src/models/__init__.py b/Python/daily-insight/src/models/__init__.py new file mode 100644 index 0000000..4ab9b67 --- /dev/null +++ b/Python/daily-insight/src/models/__init__.py @@ -0,0 +1,4 @@ +from daily_insight.models.insight import DailyInsight +from daily_insight.models.metric import MetricRecord + +__all__ = ["DailyInsight", "MetricRecord"] diff --git a/Python/daily-insight/src/models/insight.py b/Python/daily-insight/src/models/insight.py new file mode 100644 index 0000000..76ad479 --- /dev/null +++ b/Python/daily-insight/src/models/insight.py @@ -0,0 +1,17 @@ +"""Daily insight result shape returned to API callers.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel + + +class DailyInsight(BaseModel): + """One-line-per-metric plain-text summary of recent engine data.""" + + text: str + generated_at: datetime + metric_count: int + error: Optional[str] = None diff --git a/Python/daily-insight/src/models/metric.py b/Python/daily-insight/src/models/metric.py new file mode 100644 index 0000000..ba73368 --- /dev/null +++ b/Python/daily-insight/src/models/metric.py @@ -0,0 +1,17 @@ +"""Metric record schema — matches the C# RailenginePoweredStatusPage example.""" + +from __future__ import annotations + +from pydantic import BaseModel + + +class MetricRecord(BaseModel): + """One metric reading as stored in Railengine. + + Fields mirror the camelCase JSON keys produced by the status-page example + (`metric`, `timestamp`, `value`), so no field aliases are needed. + """ + + metric: str + timestamp: int + value: float diff --git a/Python/daily-insight/src/repositories/__init__.py b/Python/daily-insight/src/repositories/__init__.py new file mode 100644 index 0000000..040b481 --- /dev/null +++ b/Python/daily-insight/src/repositories/__init__.py @@ -0,0 +1,3 @@ +from daily_insight.repositories.metric_repository import MetricRepository + +__all__ = ["MetricRepository"] diff --git a/Python/daily-insight/src/repositories/metric_repository.py b/Python/daily-insight/src/repositories/metric_repository.py new file mode 100644 index 0000000..3278e44 --- /dev/null +++ b/Python/daily-insight/src/repositories/metric_repository.py @@ -0,0 +1,44 @@ +"""Railengine retrieval for MetricRecord documents.""" + +from __future__ import annotations + +from typing import Any + +from railtown.engine import Railengine + +from daily_insight.models import MetricRecord + + +def _as_metric(item: Any) -> MetricRecord | None: + if isinstance(item, MetricRecord): + return item + if isinstance(item, dict): + try: + return MetricRecord.model_validate(item) + except Exception: + return None + return None + + +class MetricRepository: + """Single-page reads against the Railengine storage API.""" + + async def list_recent(self, limit: int = 50) -> list[MetricRecord]: + """Return up to ``limit`` metric records from the most recent page. + + ``page_size`` is capped at 100 by the SDK; callers wanting more should + page through ``list_storage_documents`` directly. + """ + page_size = max(1, min(int(limit), 100)) + async with Railengine(model=MetricRecord) as client: + page = await client.list_storage_documents( + page_number=1, + page_size=page_size, + ) + + out: list[MetricRecord] = [] + for item in page.items: + metric = _as_metric(item) + if metric is not None: + out.append(metric) + return out diff --git a/Python/daily-insight/src/services/__init__.py b/Python/daily-insight/src/services/__init__.py new file mode 100644 index 0000000..a5e2cbc --- /dev/null +++ b/Python/daily-insight/src/services/__init__.py @@ -0,0 +1,3 @@ +from daily_insight.services.insight_service import InsightService + +__all__ = ["InsightService"] diff --git a/Python/daily-insight/src/services/insight_service.py b/Python/daily-insight/src/services/insight_service.py new file mode 100644 index 0000000..b034a16 --- /dev/null +++ b/Python/daily-insight/src/services/insight_service.py @@ -0,0 +1,50 @@ +"""Orchestrate the insight Railtracks flow and shape the result.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +import railtracks as rt + +from daily_insight.agents import build_insight_agent +from daily_insight.models import DailyInsight +from daily_insight.repositories import MetricRepository + + +PROMPT = ( + "Summarize the current metric data. Call get_recent_metrics(limit=50), " + "then produce the strict per-metric output described in your system prompt." +) + + +def _result_text(result: object) -> str: + if hasattr(result, "content"): + return str(result.content) + return str(result) + + +class InsightService: + """Run the daily insight agent and return a DailyInsight.""" + + async def run(self) -> DailyInsight: + agent_cls = build_insight_agent() + flow = rt.Flow(name="DailyInsight", entry_point=agent_cls) + result = await flow.ainvoke(PROMPT) + + text = _result_text(result).strip() + metric_count = await _count_metrics() + + return DailyInsight( + text=text, + generated_at=datetime.now(timezone.utc), + metric_count=metric_count, + ) + + +async def _count_metrics() -> int: + """Cheap second read for the API response — does not block the agent.""" + try: + records = await MetricRepository().list_recent(limit=50) + return len({r.metric for r in records}) + except Exception: + return 0 From d578dc29e3800d9fe7684adb94e0c936ae8de7fc Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 12 Jun 2026 11:05:06 -0700 Subject: [PATCH 02/19] let DailyInsightService offload to a Railtracks agent via DailyInsight:AgentUrl When DailyInsight:AgentUrl is set, the 24h loop POSTs to {AgentUrl}/insight instead of calling Anthropic + MCP inline; Anthropic:ApiKey becomes optional since the agent owns the LLM call. Leave AgentUrl empty (or unset) to keep the existing inline path unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../RailenginePoweredStatusPage/README.md | 6 ++ .../Services/DailyInsightService.cs | 58 ++++++++++++++++++- .../appsettings.Development.sample.json | 4 ++ .../appsettings.json | 3 + 4 files changed, 68 insertions(+), 3 deletions(-) diff --git a/CSharp/Examples/RailenginePoweredStatusPage/README.md b/CSharp/Examples/RailenginePoweredStatusPage/README.md index b5beb85..b9ead8c 100644 --- a/CSharp/Examples/RailenginePoweredStatusPage/README.md +++ b/CSharp/Examples/RailenginePoweredStatusPage/README.md @@ -98,6 +98,12 @@ The prompt in `Services/DailyInsightService.cs` is deliberately tuned for short, If `Anthropic:ApiKey` is not set, the service logs a notice and exits cleanly; the insight card simply doesn't appear. +### Offloading to a Railtracks agent (optional) + +Set `DailyInsight:AgentUrl` to the base URL of a service that exposes `POST /insight` returning `{ text, generated_at, error? }` — for example the sibling [`Python/daily-insight`](../../../Python/daily-insight/) project, which runs the same prompt through a Railtracks agent backed by the Railengine Python SDK instead of MCP. + +When `AgentUrl` is set, `DailyInsightService` POSTs to `{AgentUrl}/insight` every 24 hours instead of calling Anthropic + MCP inline. `Anthropic:ApiKey` is then optional — the agent owns the LLM call. Leave `AgentUrl` empty (or unset) to keep the inline path. + > **Heads-up for local development:** the 24-hour timer is in-memory only, so each app restart triggers a fresh generation and a corresponding Anthropic API call. If you're iterating on the app you may want to comment out `AddHostedService()` in `Program.cs` until you're ready to test it. Once deployed to a long-running host, restarts are rare and this isn't a concern. ## Configuration diff --git a/CSharp/Examples/RailenginePoweredStatusPage/Services/DailyInsightService.cs b/CSharp/Examples/RailenginePoweredStatusPage/Services/DailyInsightService.cs index 9e6fb83..174efe9 100644 --- a/CSharp/Examples/RailenginePoweredStatusPage/Services/DailyInsightService.cs +++ b/CSharp/Examples/RailenginePoweredStatusPage/Services/DailyInsightService.cs @@ -16,6 +16,7 @@ public class DailyInsightService : BackgroundService private readonly string pat; private readonly string mcpServerBaseUrl; private readonly string mcpServerName; + private readonly string agentUrl; private static readonly TimeSpan Interval = TimeSpan.FromHours(24); private static readonly TimeSpan RequestTimeout = TimeSpan.FromMinutes(10); @@ -47,13 +48,15 @@ public DailyInsightService( pat = configuration["RailEngine:PAT"]!; mcpServerBaseUrl = configuration["RailEngine:McpServerBaseUrl"]!.TrimEnd('/'); mcpServerName = configuration["RailEngine:McpServerName"]!; + agentUrl = (configuration["DailyInsight:AgentUrl"] ?? "").TrimEnd('/'); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - if (string.IsNullOrEmpty(apiKey)) + var useAgent = !string.IsNullOrEmpty(agentUrl); + if (!useAgent && string.IsNullOrEmpty(apiKey)) { - logger.LogInformation("Anthropic:ApiKey not configured; daily insight disabled."); + logger.LogInformation("Neither DailyInsight:AgentUrl nor Anthropic:ApiKey configured; daily insight disabled."); return; } @@ -61,7 +64,14 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { - await GenerateAsync(stoppingToken); + if (useAgent) + { + await GenerateFromAgentAsync(stoppingToken); + } + else + { + await GenerateAsync(stoppingToken); + } state.Error = null; } catch (Exception ex) @@ -75,6 +85,48 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } + private async Task GenerateFromAgentAsync(CancellationToken ct) + { + logger.LogInformation("Starting daily insight generation via agent at {Url}…", agentUrl); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + cts.CancelAfter(RequestTimeout); + + using var client = httpClientFactory.CreateClient(); + client.Timeout = Timeout.InfiniteTimeSpan; + + using var request = new HttpRequestMessage(HttpMethod.Post, $"{agentUrl}/insight"); + request.Content = new StringContent("{}", Encoding.UTF8, "application/json"); + + using var response = await client.SendAsync(request, cts.Token); + + if (!response.IsSuccessStatusCode) + { + var errBody = await response.Content.ReadAsStringAsync(cts.Token); + throw new InvalidOperationException($"Agent returned {(int)response.StatusCode}: {errBody}"); + } + + var body = await response.Content.ReadAsStringAsync(cts.Token); + using var doc = JsonDocument.Parse(body); + var root = doc.RootElement; + + var text = root.TryGetProperty("text", out var t) ? t.GetString() ?? "" : ""; + var generatedAt = root.TryGetProperty("generated_at", out var ga) && ga.ValueKind == JsonValueKind.String + ? DateTimeOffset.Parse(ga.GetString()!) + : DateTimeOffset.UtcNow; + var agentError = root.TryGetProperty("error", out var err) && err.ValueKind == JsonValueKind.String + ? err.GetString() + : null; + + state.Text = text; + state.GeneratedAt = generatedAt; + if (!string.IsNullOrEmpty(agentError)) + { + state.Error = agentError; + } + logger.LogInformation("Daily insight generated via agent ({Length} chars)", text.Length); + } + private async Task GenerateAsync(CancellationToken ct) { logger.LogInformation("Starting daily insight generation…"); diff --git a/CSharp/Examples/RailenginePoweredStatusPage/appsettings.Development.sample.json b/CSharp/Examples/RailenginePoweredStatusPage/appsettings.Development.sample.json index 4d1fdca..0056a44 100644 --- a/CSharp/Examples/RailenginePoweredStatusPage/appsettings.Development.sample.json +++ b/CSharp/Examples/RailenginePoweredStatusPage/appsettings.Development.sample.json @@ -12,5 +12,9 @@ "Anthropic": { "ApiKey": "[your-anthropic-api-key-here]" }, + "DailyInsight": { + "// AgentUrl": "Optional — when set, posts to {AgentUrl}/insight (e.g. the Python/daily-insight service) instead of calling Anthropic + MCP inline. Leave empty/unset to use the inline path.", + "AgentUrl": "" + }, "AllowedIPs": [ "127.0.0.1", "::1" ] } diff --git a/CSharp/Examples/RailenginePoweredStatusPage/appsettings.json b/CSharp/Examples/RailenginePoweredStatusPage/appsettings.json index 9254a89..b0eebfe 100644 --- a/CSharp/Examples/RailenginePoweredStatusPage/appsettings.json +++ b/CSharp/Examples/RailenginePoweredStatusPage/appsettings.json @@ -17,5 +17,8 @@ "Beta": "mcp-client-2025-04-04", "Model": "claude-haiku-4-5-20251001" }, + "DailyInsight": { + "AgentUrl": "" + }, "AllowedIPs": [ "127.0.0.1", "::1" ] } From f37cbd070e2956a54e128ff863518b7a6b5c1ce9 Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 12 Jun 2026 11:53:21 -0700 Subject: [PATCH 03/19] attach bearer token to the agent POST when DailyInsight:AgentBearerToken is set Lets the C# app talk to a daily-insight agent endpoint sitting behind a bearer-auth reverse proxy. Token is read from configuration and attached as Authorization: Bearer on every /insight POST when non-empty; left blank the request goes unauthenticated, matching the existing local-dev behavior. Co-Authored-By: Claude Opus 4.7 (1M context) --- CSharp/Examples/RailenginePoweredStatusPage/README.md | 2 ++ .../Services/DailyInsightService.cs | 6 ++++++ .../appsettings.Development.sample.json | 4 +++- .../Examples/RailenginePoweredStatusPage/appsettings.json | 3 ++- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CSharp/Examples/RailenginePoweredStatusPage/README.md b/CSharp/Examples/RailenginePoweredStatusPage/README.md index b9ead8c..ebacdb5 100644 --- a/CSharp/Examples/RailenginePoweredStatusPage/README.md +++ b/CSharp/Examples/RailenginePoweredStatusPage/README.md @@ -104,6 +104,8 @@ Set `DailyInsight:AgentUrl` to the base URL of a service that exposes `POST /ins When `AgentUrl` is set, `DailyInsightService` POSTs to `{AgentUrl}/insight` every 24 hours instead of calling Anthropic + MCP inline. `Anthropic:ApiKey` is then optional — the agent owns the LLM call. Leave `AgentUrl` empty (or unset) to keep the inline path. +If the agent endpoint requires bearer authentication, set `DailyInsight:AgentBearerToken` to the token — `DailyInsightService` attaches it as `Authorization: Bearer ` on every POST. Leave it blank for an unauthenticated endpoint (e.g. local development on `:8000`). + > **Heads-up for local development:** the 24-hour timer is in-memory only, so each app restart triggers a fresh generation and a corresponding Anthropic API call. If you're iterating on the app you may want to comment out `AddHostedService()` in `Program.cs` until you're ready to test it. Once deployed to a long-running host, restarts are rare and this isn't a concern. ## Configuration diff --git a/CSharp/Examples/RailenginePoweredStatusPage/Services/DailyInsightService.cs b/CSharp/Examples/RailenginePoweredStatusPage/Services/DailyInsightService.cs index 174efe9..dee9e65 100644 --- a/CSharp/Examples/RailenginePoweredStatusPage/Services/DailyInsightService.cs +++ b/CSharp/Examples/RailenginePoweredStatusPage/Services/DailyInsightService.cs @@ -17,6 +17,7 @@ public class DailyInsightService : BackgroundService private readonly string mcpServerBaseUrl; private readonly string mcpServerName; private readonly string agentUrl; + private readonly string agentBearerToken; private static readonly TimeSpan Interval = TimeSpan.FromHours(24); private static readonly TimeSpan RequestTimeout = TimeSpan.FromMinutes(10); @@ -49,6 +50,7 @@ public DailyInsightService( mcpServerBaseUrl = configuration["RailEngine:McpServerBaseUrl"]!.TrimEnd('/'); mcpServerName = configuration["RailEngine:McpServerName"]!; agentUrl = (configuration["DailyInsight:AgentUrl"] ?? "").TrimEnd('/'); + agentBearerToken = configuration["DailyInsight:AgentBearerToken"] ?? ""; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -97,6 +99,10 @@ private async Task GenerateFromAgentAsync(CancellationToken ct) using var request = new HttpRequestMessage(HttpMethod.Post, $"{agentUrl}/insight"); request.Content = new StringContent("{}", Encoding.UTF8, "application/json"); + if (!string.IsNullOrEmpty(agentBearerToken)) + { + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", agentBearerToken); + } using var response = await client.SendAsync(request, cts.Token); diff --git a/CSharp/Examples/RailenginePoweredStatusPage/appsettings.Development.sample.json b/CSharp/Examples/RailenginePoweredStatusPage/appsettings.Development.sample.json index 0056a44..0cafdf2 100644 --- a/CSharp/Examples/RailenginePoweredStatusPage/appsettings.Development.sample.json +++ b/CSharp/Examples/RailenginePoweredStatusPage/appsettings.Development.sample.json @@ -14,7 +14,9 @@ }, "DailyInsight": { "// AgentUrl": "Optional — when set, posts to {AgentUrl}/insight (e.g. the Python/daily-insight service) instead of calling Anthropic + MCP inline. Leave empty/unset to use the inline path.", - "AgentUrl": "" + "AgentUrl": "", + "// AgentBearerToken": "Optional — when set, sent as Authorization: Bearer on every POST to {AgentUrl}/insight. Set this when the agent endpoint requires bearer authentication. Leave empty for an unauthenticated endpoint (e.g. local development).", + "AgentBearerToken": "" }, "AllowedIPs": [ "127.0.0.1", "::1" ] } diff --git a/CSharp/Examples/RailenginePoweredStatusPage/appsettings.json b/CSharp/Examples/RailenginePoweredStatusPage/appsettings.json index b0eebfe..646e75b 100644 --- a/CSharp/Examples/RailenginePoweredStatusPage/appsettings.json +++ b/CSharp/Examples/RailenginePoweredStatusPage/appsettings.json @@ -18,7 +18,8 @@ "Model": "claude-haiku-4-5-20251001" }, "DailyInsight": { - "AgentUrl": "" + "AgentUrl": "", + "AgentBearerToken": "" }, "AllowedIPs": [ "127.0.0.1", "::1" ] } From a0d4764980fab27363ca2f3c99eedc004bca6c5f Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 12 Jun 2026 11:53:38 -0700 Subject: [PATCH 04/19] add Dockerfile and switch to provider-neutral LLM_API_KEY / LLM_MODEL env vars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dockerfile installs the daily_insight package via pyproject.toml and runs uvicorn on :8000 — standard container shape for any deploy target. configure_runtime_env bridges LLM_API_KEY <-> ANTHROPIC_API_KEY at startup so the same setting works under either name; the Anthropic SDK still reads ANTHROPIC_API_KEY internally. INSIGHT_MODEL renamed to the conventional LLM_MODEL. README leads with the provider-neutral names and documents the mapping in one row instead of two. Co-Authored-By: Claude Opus 4.7 (1M context) --- Python/daily-insight/.env.example | 10 +++++--- Python/daily-insight/Dockerfile | 13 +++++++++++ Python/daily-insight/README.md | 19 +++++++++++---- .../daily-insight/src/agents/insight_agent.py | 2 +- Python/daily-insight/src/config/__init__.py | 14 +++++++++-- Python/daily-insight/src/config/env.py | 23 ++++++++++++++++--- Python/daily-insight/src/controllers/api.py | 2 ++ 7 files changed, 70 insertions(+), 13 deletions(-) create mode 100644 Python/daily-insight/Dockerfile diff --git a/Python/daily-insight/.env.example b/Python/daily-insight/.env.example index b228f6e..e90d6fa 100644 --- a/Python/daily-insight/.env.example +++ b/Python/daily-insight/.env.example @@ -5,8 +5,12 @@ ENGINE_PAT="[Your Engine PAT]" # Optional: override API host (must match the stack ENGINE_PAT targets) # RAILTOWN_API_URL="https://cndr.railtown.ai/api" -# Railtracks LLM provider -ANTHROPIC_API_KEY="[Your Anthropic API key]" +# LLM provider key. The agent uses the provider-neutral name LLM_API_KEY and +# copies it into ANTHROPIC_API_KEY on startup so the Anthropic SDK picks it up. +# (You can also set ANTHROPIC_API_KEY directly if you prefer the SDK-native +# name — the bridge runs in both directions.) +LLM_API_KEY="[Your Anthropic API key]" +# ANTHROPIC_API_KEY="[same key — SDK-native name]" # Optional: override the default Claude model (defaults to claude-haiku-4-5-20251001 in code) -# INSIGHT_MODEL="claude-haiku-4-5-20251001" +# LLM_MODEL="claude-haiku-4-5-20251001" diff --git a/Python/daily-insight/Dockerfile b/Python/daily-insight/Dockerfile new file mode 100644 index 0000000..1eae936 --- /dev/null +++ b/Python/daily-insight/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.10-slim + +WORKDIR /app + +# Install the package + deps. Copying pyproject.toml + src separately lets +# Docker cache the dependency install across source-only changes. +COPY pyproject.toml ./ +COPY src/ ./src/ +RUN pip install --no-cache-dir . + +EXPOSE 8000 + +CMD ["uvicorn", "daily_insight.controllers.api:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/Python/daily-insight/README.md b/Python/daily-insight/README.md index 9a127ce..a2dc7d9 100644 --- a/Python/daily-insight/README.md +++ b/Python/daily-insight/README.md @@ -7,13 +7,13 @@ Where the C# version drives the LLM through an Anthropic MCP server attached as ## Before you start - A [Railengine](https://railengine.ai/) engine populated with `MetricRecord`-shaped documents (`metric`, `timestamp`, `value` — see [`MetricRecord`](src/models/metric.py)). The C# status page example produces records in exactly this shape. -- An Anthropic API key with access to `claude-haiku-4-5-20251001` (or another model you set via `INSIGHT_MODEL`). +- An Anthropic API key with access to `claude-haiku-4-5-20251001` (or another model you set via `LLM_MODEL`). ## Quick start ```bash cd Python/daily-insight -cp .env.example .env # fill ENGINE_ID, ENGINE_PAT, ANTHROPIC_API_KEY +cp .env.example .env # fill ENGINE_ID, ENGINE_PAT, LLM_API_KEY uv sync uv run uvicorn daily_insight.controllers.api:app --reload --port 8000 ``` @@ -47,8 +47,8 @@ Expect each call to take a few seconds — the agent makes one Anthropic call pl |---|---|---| | `ENGINE_ID` | Yes | Railengine engine GUID — same one the C# status page reads from | | `ENGINE_PAT` | Yes | Railengine PAT used for retrieval | -| `ANTHROPIC_API_KEY` | Yes | Anthropic API key used by `rt.llm.AnthropicLLM` | -| `INSIGHT_MODEL` | No | Override the default Claude model (`claude-haiku-4-5-20251001`) | +| `LLM_API_KEY` | Yes | Anthropic API key. Provider-neutral name so the same setting works across agents; on startup the value is copied into `ANTHROPIC_API_KEY` for the Anthropic SDK to pick up. `ANTHROPIC_API_KEY` is also accepted directly if you prefer the SDK-native name. | +| `LLM_MODEL` | No | Override the default Claude model (`claude-haiku-4-5-20251001`) | | `RAILTOWN_API_URL` | No | Override the Railengine API host (defaults to production) | Variables are read from `.env` next to `pyproject.toml`, then from the process environment. @@ -80,6 +80,17 @@ This Python version: - Replaces the 24h `BackgroundService` with an on-demand HTTP endpoint. A scheduler (cron, GitHub Actions, Azure Logic Apps, etc.) can POST `/insight` daily if you want the same cadence. - Keeps the strict plain-text output rules so the existing C# `Daily Insight` card can render the result unchanged. +## Containerization + +A [`Dockerfile`](Dockerfile) is included so the service can be containerized for deployment. The image installs the package via `pyproject.toml` and runs `uvicorn daily_insight.controllers.api:app` on port 8000. Build and run locally with: + +```bash +docker build -t daily-insight . +docker run --rm -p 8000:8000 --env-file .env daily-insight +``` + +If you put the agent behind a reverse proxy that requires bearer auth, the C# caller in [`RailenginePoweredStatusPage`](../../CSharp/Examples/RailenginePoweredStatusPage/) can attach the token via the `DailyInsight:AgentBearerToken` setting — `DailyInsightService.GenerateFromAgentAsync` adds it as `Authorization: Bearer ` when non-empty. + ## Debug and visualize the agent (optional) After at least one insight run: diff --git a/Python/daily-insight/src/agents/insight_agent.py b/Python/daily-insight/src/agents/insight_agent.py index c37591e..6ed4064 100644 --- a/Python/daily-insight/src/agents/insight_agent.py +++ b/Python/daily-insight/src/agents/insight_agent.py @@ -31,7 +31,7 @@ def build_insight_agent(): "ANTHROPIC_API_KEY is not set. Add it to .env or export it before running." ) - model = os.environ.get("INSIGHT_MODEL", "").strip() or DEFAULT_MODEL + model = os.environ.get("LLM_MODEL", "").strip() or DEFAULT_MODEL llm = rt.llm.AnthropicLLM(model) return rt.agent_node( diff --git a/Python/daily-insight/src/config/__init__.py b/Python/daily-insight/src/config/__init__.py index 4c3a33d..03eea61 100644 --- a/Python/daily-insight/src/config/__init__.py +++ b/Python/daily-insight/src/config/__init__.py @@ -1,3 +1,13 @@ -from daily_insight.config.env import ensure_dotenv_loaded, validate_required_env, MissingEnvVarsError +from daily_insight.config.env import ( + MissingEnvVarsError, + configure_runtime_env, + ensure_dotenv_loaded, + validate_required_env, +) -__all__ = ["ensure_dotenv_loaded", "validate_required_env", "MissingEnvVarsError"] +__all__ = [ + "MissingEnvVarsError", + "configure_runtime_env", + "ensure_dotenv_loaded", + "validate_required_env", +] diff --git a/Python/daily-insight/src/config/env.py b/Python/daily-insight/src/config/env.py index 88fb8dd..3083b93 100644 --- a/Python/daily-insight/src/config/env.py +++ b/Python/daily-insight/src/config/env.py @@ -1,4 +1,4 @@ -"""Load `.env` and validate required environment variables.""" +"""Load `.env`, bridge env var aliases, and validate required environment variables.""" from __future__ import annotations @@ -17,7 +17,8 @@ def __init__(self, missing: list[str]) -> None: names = ", ".join(missing) super().__init__( f"Missing required environment variables: {names}. " - f"Set them in .env (see .env.example)." + f"Set them in .env (see .env.example). " + f"ANTHROPIC_API_KEY also accepts the provider-neutral alias LLM_API_KEY." ) @@ -28,8 +29,24 @@ def ensure_dotenv_loaded() -> None: load_dotenv() +def configure_runtime_env() -> None: + """Bridge LLM_API_KEY ↔ ANTHROPIC_API_KEY so deploy tooling that stores + keys under a provider-neutral name still satisfies the AnthropicLLM + SDK's lookup. Whichever is set on startup is mirrored to the other.""" + if not os.environ.get("LLM_API_KEY", "").strip(): + if anthropic := os.environ.get("ANTHROPIC_API_KEY", "").strip(): + os.environ["LLM_API_KEY"] = anthropic + + if llm_key := os.environ.get("LLM_API_KEY", "").strip(): + os.environ["ANTHROPIC_API_KEY"] = llm_key + + def validate_required_env() -> None: - """Raise MissingEnvVarsError if any required variable is unset or blank.""" + """Raise MissingEnvVarsError if any required variable is unset or blank. + + Run ``configure_runtime_env`` first so the LLM_API_KEY alias has a chance + to populate ANTHROPIC_API_KEY before this check. + """ missing = [name for name in REQUIRED_ENV_VARS if not os.environ.get(name, "").strip()] if missing: raise MissingEnvVarsError(missing) diff --git a/Python/daily-insight/src/controllers/api.py b/Python/daily-insight/src/controllers/api.py index 395ea60..27907c2 100644 --- a/Python/daily-insight/src/controllers/api.py +++ b/Python/daily-insight/src/controllers/api.py @@ -18,6 +18,7 @@ from daily_insight.config import ( MissingEnvVarsError, + configure_runtime_env, ensure_dotenv_loaded, validate_required_env, ) @@ -31,6 +32,7 @@ @asynccontextmanager async def lifespan(app: FastAPI): ensure_dotenv_loaded() + configure_runtime_env() try: validate_required_env() except MissingEnvVarsError as exc: From 9e3d04955ec5e665e07678ac3eb2db1672bb3462 Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 12 Jun 2026 14:31:20 -0700 Subject: [PATCH 05/19] wire railtownai observability + global exception handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes that together turn opaque 500s into actionable diagnostics: 1. railtownai.init() in the FastAPI lifespan when RAILTOWN_API_KEY is set. The SDK attaches a RailtownHandler to Python's root logger, so any logger.exception/error call downstream ships to Railtown automatically. No-op when the key is unset — agent runs normally. 2. @app.exception_handler(Exception) that logs the traceback and returns a JSON 500 body with the exception type and message instead of FastAPI's default plain-text "Internal Server Error". Replaces the per-endpoint try/except in /insight so unhandled errors from any route get the same treatment. Co-Authored-By: Claude Opus 4.7 (1M context) --- Python/daily-insight/.env.example | 4 ++ Python/daily-insight/README.md | 1 + Python/daily-insight/pyproject.toml | 1 + Python/daily-insight/src/controllers/api.py | 46 +++++++++++++++++---- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/Python/daily-insight/.env.example b/Python/daily-insight/.env.example index e90d6fa..3ec1c9c 100644 --- a/Python/daily-insight/.env.example +++ b/Python/daily-insight/.env.example @@ -14,3 +14,7 @@ LLM_API_KEY="[Your Anthropic API key]" # Optional: override the default Claude model (defaults to claude-haiku-4-5-20251001 in code) # LLM_MODEL="claude-haiku-4-5-20251001" + +# Optional: Railtown observability. When set, the agent's logs + any unhandled +# exception tracebacks ship to Railtown via the railtownai logging handler. +# RAILTOWN_API_KEY="[Your Railtown API key]" diff --git a/Python/daily-insight/README.md b/Python/daily-insight/README.md index a2dc7d9..247fc62 100644 --- a/Python/daily-insight/README.md +++ b/Python/daily-insight/README.md @@ -49,6 +49,7 @@ Expect each call to take a few seconds — the agent makes one Anthropic call pl | `ENGINE_PAT` | Yes | Railengine PAT used for retrieval | | `LLM_API_KEY` | Yes | Anthropic API key. Provider-neutral name so the same setting works across agents; on startup the value is copied into `ANTHROPIC_API_KEY` for the Anthropic SDK to pick up. `ANTHROPIC_API_KEY` is also accepted directly if you prefer the SDK-native name. | | `LLM_MODEL` | No | Override the default Claude model (`claude-haiku-4-5-20251001`) | +| `RAILTOWN_API_KEY` | No | Enables Railtown observability. When set, the agent ships its logs (and any unhandled-exception tracebacks from the global handler) to Railtown via the [`railtownai`](https://pypi.org/project/railtownai/) logging handler. Unset → no observability, agent runs normally. | | `RAILTOWN_API_URL` | No | Override the Railengine API host (defaults to production) | Variables are read from `.env` next to `pyproject.toml`, then from the process environment. diff --git a/Python/daily-insight/pyproject.toml b/Python/daily-insight/pyproject.toml index 39d2a10..2ec33d0 100644 --- a/Python/daily-insight/pyproject.toml +++ b/Python/daily-insight/pyproject.toml @@ -11,6 +11,7 @@ requires-python = ">=3.10" dependencies = [ "rail-engine>=0.2.1", "railtracks[visual]>=1.3.0", + "railtownai>=2.0.13", "pydantic>=2.0", "python-dotenv>=1.0.0", "fastapi>=0.110.0", diff --git a/Python/daily-insight/src/controllers/api.py b/Python/daily-insight/src/controllers/api.py index 27907c2..4f799f6 100644 --- a/Python/daily-insight/src/controllers/api.py +++ b/Python/daily-insight/src/controllers/api.py @@ -12,9 +12,12 @@ from __future__ import annotations import logging -from contextlib import asynccontextmanager +import os -from fastapi import FastAPI, HTTPException +import railtownai +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse +from contextlib import asynccontextmanager from daily_insight.config import ( MissingEnvVarsError, @@ -33,6 +36,16 @@ async def lifespan(app: FastAPI): ensure_dotenv_loaded() configure_runtime_env() + + # Wire Railtown observability when configured. railtownai.init attaches a + # logging handler to the root logger, so any logger.exception/error call + # downstream ships to Railtown automatically. + if railtown_key := os.environ.get("RAILTOWN_API_KEY", "").strip(): + railtownai.init(railtown_key) + logger.info("Railtown observability enabled") + else: + logger.info("RAILTOWN_API_KEY not set — Railtown observability disabled") + try: validate_required_env() except MissingEnvVarsError as exc: @@ -49,6 +62,29 @@ async def lifespan(app: FastAPI): ) +@app.exception_handler(Exception) +async def global_exception_handler(request: Request, exc: Exception) -> JSONResponse: + """Catch-all for unhandled exceptions. + + Logs the full traceback via Python's logging (Railtown's handler ships + it onward when initialized) and returns a JSON 500 body with the + exception type and message — far more useful than FastAPI's default + plain-text "Internal Server Error" for diagnosing live failures. + """ + logger.exception( + "Unhandled exception in %s %s", + request.method, + request.url.path, + ) + return JSONResponse( + status_code=500, + content={ + "type": type(exc).__name__, + "detail": str(exc), + }, + ) + + @app.get("/health") async def health() -> dict[str, str]: return {"status": "ok"} @@ -56,8 +92,4 @@ async def health() -> dict[str, str]: @app.post("/insight", response_model=DailyInsight) async def generate_insight() -> DailyInsight: - try: - return await InsightService().run() - except Exception as exc: - logger.exception("Failed to generate daily insight") - raise HTTPException(status_code=500, detail=str(exc)) from exc + return await InsightService().run() From 046696409010b5bebe822539cdf80bfea871fe86 Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 12 Jun 2026 14:44:48 -0700 Subject: [PATCH 06/19] upload Railtracks session payload to Conductr after each insight run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switches InsightService from rt.Flow(...).ainvoke() to the rt.Session + rt.call() pattern so we can grab session.payload() and hand it to railtownai.upload_agent_run(). The payload contains the nodes / edges / steps that drive the Railtracks viz UI in Conductr — useful for inspecting which tools the agent called and what prompts it used. Skipped silently when railtownai.init() hasn't run (no RAILTOWN_API_KEY) so local invocations without observability keep working unchanged. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/services/insight_service.py | 39 +++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/Python/daily-insight/src/services/insight_service.py b/Python/daily-insight/src/services/insight_service.py index b034a16..62afa91 100644 --- a/Python/daily-insight/src/services/insight_service.py +++ b/Python/daily-insight/src/services/insight_service.py @@ -1,10 +1,12 @@ -"""Orchestrate the insight Railtracks flow and shape the result.""" +"""Orchestrate the insight Railtracks session and shape the result.""" from __future__ import annotations +import logging from datetime import datetime, timezone import railtracks as rt +import railtownai from daily_insight.agents import build_insight_agent from daily_insight.models import DailyInsight @@ -16,6 +18,13 @@ "then produce the strict per-metric output described in your system prompt." ) +_SESSION_NAME = "daily-insight-session" +_FLOW_NAME = "daily-insight" +_FLOW_ID = "daily-insight-agent" + + +logger = logging.getLogger(__name__) + def _result_text(result: object) -> str: if hasattr(result, "content"): @@ -23,13 +32,37 @@ def _result_text(result: object) -> str: return str(result) +def _upload_session(session: "rt.Session") -> None: + """Upload the Railtracks session payload to Conductr when configured. + + Skips silently when ``railtownai.init`` hasn't run (no RAILTOWN_API_KEY), + so a local invocation without observability still succeeds. + """ + if railtownai.get_railtown_handler() is None: + return + try: + success = railtownai.upload_agent_run(session.payload()) + logger.info("Agent run uploaded to Conductr: %s", success) + except RuntimeError as exc: + logger.warning("Skipping Railtown upload: %s", exc) + except Exception: + logger.exception("Unexpected error uploading agent run to Railtown") + + class InsightService: """Run the daily insight agent and return a DailyInsight.""" async def run(self) -> DailyInsight: agent_cls = build_insight_agent() - flow = rt.Flow(name="DailyInsight", entry_point=agent_cls) - result = await flow.ainvoke(PROMPT) + message_history = rt.llm.MessageHistory([rt.llm.UserMessage(PROMPT)]) + + with rt.Session( + name=_SESSION_NAME, + flow_name=_FLOW_NAME, + flow_id=_FLOW_ID, + ) as session: + result = await rt.call(agent_cls, message_history) + _upload_session(session) text = _result_text(result).strip() metric_count = await _count_metrics() From c23b4746bfd68c19feb2b309259bba3bb7b68f30 Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 12 Jun 2026 15:02:36 -0700 Subject: [PATCH 07/19] add POST /evaluate: generate fresh insight runs and score them with Railtracks evaluators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-contained smoke test for the deployed agent. Body: { "sample_size": 1, "agent_run_id": "" } sample_size (default 1, capped at 5) drives how many fresh /insight runs the endpoint generates before scoring. agent_run_id is accepted for forward compatibility — currently logged and ignored, will identify a historical session to evaluate in a follow-up. Three evaluators per call: - ToolUseEvaluator (free) — checks the "at most 1 get_recent_metrics call" contract - LLMInferenceEvaluator (free) — checks LLM call latency/tokens/errors - JudgeEvaluator with two custom Categorical metrics: FormatCompliance (Compliant / MinorDeviation / MajorDeviation) FactualGrounding (FullyGrounded / PartiallyGrounded / Hallucinated) The judge uses AnthropicLLM with the agent's same key (LLM_API_KEY bridge); override the judge model via EVAL_JUDGE_MODEL. When RAILTOWN_API_KEY is set, each EvaluationResult uploads to Conductr via railtownai.upload_agent_evaluation through evals.evaluate's payload_callback hook. Sessions for extract_agent_data_points are staged in a request-scoped tempfile.TemporaryDirectory so concurrent /evaluate calls don't see each other's session payloads. agent_selection=False + agents=[...] keeps evals.evaluate headless (it would otherwise hang on rich.prompt.Prompt.ask). Bumps railtracks[visual] to >=1.4.0 for the tool-eval-requires-multiple- sessions bugfix. Co-Authored-By: Claude Opus 4.7 (1M context) --- Python/daily-insight/README.md | 17 ++++ Python/daily-insight/pyproject.toml | 2 +- .../daily-insight/src/agents/evaluations.py | 69 +++++++++++++++ Python/daily-insight/src/controllers/api.py | 45 +++++++++- Python/daily-insight/src/models/__init__.py | 3 +- Python/daily-insight/src/models/evaluation.py | 20 +++++ Python/daily-insight/src/services/__init__.py | 3 +- .../src/services/evaluation_service.py | 88 +++++++++++++++++++ .../src/services/insight_service.py | 34 ++++--- 9 files changed, 264 insertions(+), 17 deletions(-) create mode 100644 Python/daily-insight/src/agents/evaluations.py create mode 100644 Python/daily-insight/src/models/evaluation.py create mode 100644 Python/daily-insight/src/services/evaluation_service.py diff --git a/Python/daily-insight/README.md b/Python/daily-insight/README.md index 247fc62..a298039 100644 --- a/Python/daily-insight/README.md +++ b/Python/daily-insight/README.md @@ -26,6 +26,9 @@ curl http://127.0.0.1:8000/health # Generate a fresh insight curl -X POST http://127.0.0.1:8000/insight + +# Generate fresh insight(s) and evaluate them +curl -X POST http://127.0.0.1:8000/evaluate -H 'Content-Type: application/json' -d '{"sample_size": 1}' ``` `POST /insight` runs the Railtracks agent against your engine and returns: @@ -41,6 +44,20 @@ curl -X POST http://127.0.0.1:8000/insight Expect each call to take a few seconds — the agent makes one Anthropic call plus one Railengine GET. The exact latency depends on the model. +## Evaluation + +`POST /evaluate` is a self-contained smoke test: it triggers `sample_size` fresh `/insight` runs (default 1, capped at 5), then runs Railtracks evaluators against the resulting sessions and returns the scores. When `RAILTOWN_API_KEY` is set, each result also uploads to Conductr via `railtownai.upload_agent_evaluation`. + +Three evaluators run per call (see [`agents/evaluations.py`](src/agents/evaluations.py)): + +- **`ToolUseEvaluator`** (free, local) — checks the agent's tool call pattern. The system prompt says "use AT MOST 1 tool call to `get_recent_metrics`"; this catches regressions where the model skips it or over-calls. +- **`LLMInferenceEvaluator`** (free, local) — checks the LLM call pattern (latency, tokens, errors). +- **`JudgeEvaluator`** with two custom metrics: + - **`FormatCompliance`** (Compliant / MinorDeviation / MajorDeviation) — does the response follow the strict one-line-per-metric format that the C# card expects to render unchanged? + - **`FactualGrounding`** (FullyGrounded / PartiallyGrounded / Hallucinated) — does every value and trend in the response trace back to the tool output, or is the model inventing things? + +Cost per call ≈ `sample_size · 3` LLM calls (one to generate, two for the judge metrics). Override the judge model via `EVAL_JUDGE_MODEL`; defaults to `LLM_MODEL` or `claude-haiku-4-5-20251001`. + ## Environment variables | Variable | Required | Purpose | diff --git a/Python/daily-insight/pyproject.toml b/Python/daily-insight/pyproject.toml index 2ec33d0..71a2d80 100644 --- a/Python/daily-insight/pyproject.toml +++ b/Python/daily-insight/pyproject.toml @@ -10,7 +10,7 @@ readme = "README.md" requires-python = ">=3.10" dependencies = [ "rail-engine>=0.2.1", - "railtracks[visual]>=1.3.0", + "railtracks[visual]>=1.4.0", "railtownai>=2.0.13", "pydantic>=2.0", "python-dotenv>=1.0.0", diff --git a/Python/daily-insight/src/agents/evaluations.py b/Python/daily-insight/src/agents/evaluations.py new file mode 100644 index 0000000..5760cb6 --- /dev/null +++ b/Python/daily-insight/src/agents/evaluations.py @@ -0,0 +1,69 @@ +"""Railtracks evaluation setup for the daily insight agent. + +Metrics target the agent's specific failure modes (strict per-line output +format, factual grounding against the tool output) rather than the generic +helpfulness/efficiency that the railtracks quickstart shows. +""" + +from __future__ import annotations + +import os + +import railtracks as rt +from railtracks import evaluations as evals + + +# Same default as the agent — when LLM_MODEL/EVAL_JUDGE_MODEL is set, override. +_DEFAULT_JUDGE_MODEL = "claude-haiku-4-5-20251001" + + +FORMAT_COMPLIANCE = evals.metrics.Categorical( + name="FormatCompliance", + description=( + "Does the agent's final response follow the strict format: one line per " + "metric, no markdown, no preamble, format `{metric}: {one-sentence " + "insight with latest value and trend}`? " + "Compliant = every line conforms. " + "MinorDeviation = mostly conforms but one stray formatting element " + "(extra blank line, single missing colon, etc.). " + "MajorDeviation = bullets, headers, prose paragraphs, multi-line " + "insights per metric." + ), + categories=["Compliant", "MinorDeviation", "MajorDeviation"], +) + + +FACTUAL_GROUNDING = evals.metrics.Categorical( + name="FactualGrounding", + description=( + "Is every numeric value and trend statement in the agent's final " + "response actually present in the get_recent_metrics tool output? " + "Inspect the tool calls and tool outputs in the data. " + "FullyGrounded = every claim traceable to the tool data. " + "PartiallyGrounded = some values right, some invented or wrong. " + "Hallucinated = at least one fabricated value or trend." + ), + categories=["FullyGrounded", "PartiallyGrounded", "Hallucinated"], +) + + +def build_evaluators() -> list: + """Return the evaluator list to run against each daily-insight session. + + The judge model defaults to the agent's own model; override via + EVAL_JUDGE_MODEL (otherwise LLM_MODEL, otherwise the constant default). + """ + judge_model = ( + os.environ.get("EVAL_JUDGE_MODEL", "").strip() + or os.environ.get("LLM_MODEL", "").strip() + or _DEFAULT_JUDGE_MODEL + ) + return [ + evals.ToolUseEvaluator(), + evals.LLMInferenceEvaluator(), + evals.JudgeEvaluator( + llm=rt.llm.AnthropicLLM(judge_model), + metrics=[FORMAT_COMPLIANCE, FACTUAL_GROUNDING], + reasoning=True, + ), + ] diff --git a/Python/daily-insight/src/controllers/api.py b/Python/daily-insight/src/controllers/api.py index 4f799f6..19276af 100644 --- a/Python/daily-insight/src/controllers/api.py +++ b/Python/daily-insight/src/controllers/api.py @@ -5,19 +5,23 @@ uv run uvicorn daily_insight.controllers.api:app --reload --port 8000 Endpoints: -- GET /health — liveness probe -- POST /insight — run the Railtracks agent against Railengine and return a DailyInsight +- GET /health — liveness probe +- POST /insight — run the Railtracks agent against Railengine and return a DailyInsight +- POST /evaluate — run the agent N times, evaluate the sessions, return scores """ from __future__ import annotations import logging import os +from typing import Optional +from uuid import UUID import railtownai from fastapi import FastAPI, Request from fastapi.responses import JSONResponse from contextlib import asynccontextmanager +from pydantic import BaseModel, Field from daily_insight.config import ( MissingEnvVarsError, @@ -25,8 +29,25 @@ ensure_dotenv_loaded, validate_required_env, ) -from daily_insight.models import DailyInsight -from daily_insight.services import InsightService +from daily_insight.models import DailyInsight, EvaluationRun +from daily_insight.services import EvaluationService, InsightService + + +class EvaluateRequest(BaseModel): + sample_size: int = Field( + default=1, + ge=1, + le=5, + description="Number of fresh insight runs to generate and evaluate (capped at 5).", + ) + # Forward-compat: when populated, future versions will fetch the named + # historical run from Conductr and evaluate that instead of generating + # fresh runs. Currently accepted (so existing clients don't need updating + # later) but logged + ignored — `sample_size` still drives behaviour. + agent_run_id: Optional[UUID] = Field( + default=None, + description="Optional. Reserved for future use — will identify a specific historical agent run to evaluate. Currently logged and ignored; sample_size still drives behaviour.", + ) logger = logging.getLogger(__name__) @@ -93,3 +114,19 @@ async def health() -> dict[str, str]: @app.post("/insight", response_model=DailyInsight) async def generate_insight() -> DailyInsight: return await InsightService().run() + + +@app.post("/evaluate", response_model=EvaluationRun) +async def evaluate(req: EvaluateRequest = EvaluateRequest()) -> EvaluationRun: + """Run the agent N times and evaluate the resulting sessions. + + Cost per call ≈ ``sample_size · 3`` LLM calls — one to generate the insight, + two for the judge metrics (FormatCompliance + FactualGrounding). The + ToolUseEvaluator and LLMInferenceEvaluator are cost-free local checks. + """ + if req.agent_run_id is not None: + logger.info( + "agent_run_id=%s provided but not yet wired — generating fresh runs", + req.agent_run_id, + ) + return await EvaluationService().run(sample_size=req.sample_size) diff --git a/Python/daily-insight/src/models/__init__.py b/Python/daily-insight/src/models/__init__.py index 4ab9b67..8113d1b 100644 --- a/Python/daily-insight/src/models/__init__.py +++ b/Python/daily-insight/src/models/__init__.py @@ -1,4 +1,5 @@ +from daily_insight.models.evaluation import EvaluationRun from daily_insight.models.insight import DailyInsight from daily_insight.models.metric import MetricRecord -__all__ = ["DailyInsight", "MetricRecord"] +__all__ = ["DailyInsight", "EvaluationRun", "MetricRecord"] diff --git a/Python/daily-insight/src/models/evaluation.py b/Python/daily-insight/src/models/evaluation.py new file mode 100644 index 0000000..a1bef48 --- /dev/null +++ b/Python/daily-insight/src/models/evaluation.py @@ -0,0 +1,20 @@ +"""Evaluation run result returned by POST /evaluate.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from pydantic import BaseModel + +from daily_insight.models.insight import DailyInsight + + +class EvaluationRun(BaseModel): + """One end-to-end evaluation: N fresh insight runs plus their scores.""" + + sample_size: int + started_at: datetime + completed_at: datetime + insights: list[DailyInsight] + evaluation_results: list[dict[str, Any]] diff --git a/Python/daily-insight/src/services/__init__.py b/Python/daily-insight/src/services/__init__.py index a5e2cbc..f4d4b38 100644 --- a/Python/daily-insight/src/services/__init__.py +++ b/Python/daily-insight/src/services/__init__.py @@ -1,3 +1,4 @@ +from daily_insight.services.evaluation_service import EvaluationService from daily_insight.services.insight_service import InsightService -__all__ = ["InsightService"] +__all__ = ["EvaluationService", "InsightService"] diff --git a/Python/daily-insight/src/services/evaluation_service.py b/Python/daily-insight/src/services/evaluation_service.py new file mode 100644 index 0000000..ccef8b1 --- /dev/null +++ b/Python/daily-insight/src/services/evaluation_service.py @@ -0,0 +1,88 @@ +"""Generate fresh insight runs and evaluate them with the configured evaluators.""" + +from __future__ import annotations + +import json +import logging +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import railtownai +from railtracks import evaluations as evals + +from daily_insight.agents.evaluations import build_evaluators +from daily_insight.models import DailyInsight, EvaluationRun +from daily_insight.services.insight_service import InsightService + + +logger = logging.getLogger(__name__) + +_AGENT_NAME = "Daily Insight Agent" +_MAX_SAMPLE_SIZE = 5 + + +def _upload_evaluation(payload: dict[str, Any]) -> None: + """Callback handed to evals.evaluate — uploads each EvaluationResult to Conductr.""" + if railtownai.get_railtown_handler() is None: + return + try: + railtownai.upload_agent_evaluation(payload) + logger.info("Evaluation result uploaded to Conductr") + except Exception: + logger.exception("Failed to upload evaluation result to Railtown") + + +class EvaluationService: + """Run N fresh insight invocations and evaluate them with the agent's evaluators.""" + + async def run(self, sample_size: int = 1) -> EvaluationRun: + sample_size = max(1, min(int(sample_size), _MAX_SAMPLE_SIZE)) + started_at = datetime.now(timezone.utc) + + insight_service = InsightService() + insights: list[DailyInsight] = [] + session_payloads: list[dict[str, Any]] = [] + + for i in range(sample_size): + logger.info("Generating insight run %s/%s", i + 1, sample_size) + insight, payload = await insight_service.run_and_capture_session() + insights.append(insight) + session_payloads.append(payload) + + # extract_agent_data_points reads JSON session files from disk. Stage the + # captured payloads in a request-scoped tempdir so each /evaluate call + # sees exactly its own sessions — no leakage between concurrent requests. + with tempfile.TemporaryDirectory(prefix="daily-insight-eval-") as tmpdir: + tmp = Path(tmpdir) + session_files: list[str] = [] + for idx, payload in enumerate(session_payloads): + fp = tmp / f"session_{idx}.json" + fp.write_text(json.dumps(payload), encoding="utf-8") + session_files.append(str(fp)) + + data = evals.extract_agent_data_points(session_files) + logger.info("Extracted %s agent data points from %s sessions", len(data), len(session_files)) + + evaluation_name = f"daily-insight-{started_at.strftime('%Y%m%dT%H%M%SZ')}" + + # agent_selection=False + agents=[...] keeps evaluate() headless. + # payload_callback fires once per EvaluationResult as it completes. + evaluation_results = evals.evaluate( + data=data, + evaluators=build_evaluators(), + agents=[_AGENT_NAME], + agent_selection=False, + name=evaluation_name, + payload_callback=_upload_evaluation, + ) + + completed_at = datetime.now(timezone.utc) + return EvaluationRun( + sample_size=sample_size, + started_at=started_at, + completed_at=completed_at, + insights=insights, + evaluation_results=[r.model_dump(mode="json") for r in evaluation_results], + ) diff --git a/Python/daily-insight/src/services/insight_service.py b/Python/daily-insight/src/services/insight_service.py index 62afa91..1fb1607 100644 --- a/Python/daily-insight/src/services/insight_service.py +++ b/Python/daily-insight/src/services/insight_service.py @@ -4,6 +4,7 @@ import logging from datetime import datetime, timezone +from typing import Any import railtracks as rt import railtownai @@ -49,10 +50,29 @@ def _upload_session(session: "rt.Session") -> None: logger.exception("Unexpected error uploading agent run to Railtown") +async def _count_metrics() -> int: + """Cheap second read for the API response — does not block the agent.""" + try: + records = await MetricRepository().list_recent(limit=50) + return len({r.metric for r in records}) + except Exception: + return 0 + + class InsightService: """Run the daily insight agent and return a DailyInsight.""" async def run(self) -> DailyInsight: + insight, _payload = await self.run_and_capture_session() + return insight + + async def run_and_capture_session(self) -> tuple[DailyInsight, dict[str, Any]]: + """Run the agent and return both the DailyInsight and the session payload. + + The payload is the same structure passed to ``railtownai.upload_agent_run`` + — used by the evaluation service to feed sessions into + ``evals.extract_agent_data_points`` without re-running the agent. + """ agent_cls = build_insight_agent() message_history = rt.llm.MessageHistory([rt.llm.UserMessage(PROMPT)]) @@ -64,20 +84,14 @@ async def run(self) -> DailyInsight: result = await rt.call(agent_cls, message_history) _upload_session(session) + session_payload = session.payload() + text = _result_text(result).strip() metric_count = await _count_metrics() - return DailyInsight( + insight = DailyInsight( text=text, generated_at=datetime.now(timezone.utc), metric_count=metric_count, ) - - -async def _count_metrics() -> int: - """Cheap second read for the API response — does not block the agent.""" - try: - records = await MetricRepository().list_recent(limit=50) - return len({r.metric for r in records}) - except Exception: - return 0 + return insight, session_payload From 95bd382eb3439b067f733dbdffc635f925315d15 Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 12 Jun 2026 15:08:03 -0700 Subject: [PATCH 08/19] pin railtracks to commit 4e4ed57 for ToolUseEvaluator single-session fix The 1.4.0 release of railtracks raises ValueError in ToolUseEvaluator when fewer than 2 aggregate nodes exist per tool, which kills /evaluate calls with sample_size=1 (our default). The fix landed on main 2026-06-11 but hasn't shipped to PyPI yet, so we pin to the specific commit via PEP 508's git URL syntax. Swap back to a version range once 1.4.1+ releases. Dockerfile gains a minimal `apt-get install git` step because python:3.10-slim lacks git and pip needs it to clone the pinned commit at build time. Fix being pinned: https://github.com/RailtownAI/railtracks/commit/4e4ed572d76f946908bb3fd441443be8903ae8de Co-Authored-By: Claude Opus 4.7 (1M context) --- Python/daily-insight/Dockerfile | 6 ++++++ Python/daily-insight/pyproject.toml | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Python/daily-insight/Dockerfile b/Python/daily-insight/Dockerfile index 1eae936..5e1b714 100644 --- a/Python/daily-insight/Dockerfile +++ b/Python/daily-insight/Dockerfile @@ -2,6 +2,12 @@ FROM python:3.10-slim WORKDIR /app +# git is needed to pip-install railtracks from its pinned commit (see +# pyproject.toml). Remove once we move back to a PyPI-released version. +RUN apt-get update \ + && apt-get install -y --no-install-recommends git \ + && rm -rf /var/lib/apt/lists/* + # Install the package + deps. Copying pyproject.toml + src separately lets # Docker cache the dependency install across source-only changes. COPY pyproject.toml ./ diff --git a/Python/daily-insight/pyproject.toml b/Python/daily-insight/pyproject.toml index 71a2d80..4cc6f97 100644 --- a/Python/daily-insight/pyproject.toml +++ b/Python/daily-insight/pyproject.toml @@ -10,7 +10,12 @@ readme = "README.md" requires-python = ">=3.10" dependencies = [ "rail-engine>=0.2.1", - "railtracks[visual]>=1.4.0", + # Pinned to commit 4e4ed57 for the ToolUseEvaluator single-session fix + # (https://github.com/RailtownAI/railtracks/commit/4e4ed572d76f946908bb3fd441443be8903ae8de + # — needed because /evaluate's sample_size=1 default produces one session + # and the released 1.4.0 ToolUseEvaluator raises ValueError on fewer than 2). + # Swap back to "railtracks[visual]>=1.4.1" (or higher) once that release ships. + "railtracks[visual] @ git+https://github.com/RailtownAI/railtracks.git@4e4ed572d76f946908bb3fd441443be8903ae8de", "railtownai>=2.0.13", "pydantic>=2.0", "python-dotenv>=1.0.0", From 19ec35dc8495b4f3c42a083adceecc85f30f8f43 Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 12 Jun 2026 15:14:47 -0700 Subject: [PATCH 09/19] add subdirectory=packages/railtracks hint to the railtracks git pin The pinned commit installation failed during the ACR build: error: Multiple top-level packages discovered in a flat-layout: ['pdoc', 'packages']. The railtracks repo is a monorepo (packages/, pdoc/, docs/, etc. at the root), so setuptools' flat-layout auto-discovery refuses to guess. The actual package lives at packages/railtracks/ with its own pyproject.toml + src/ layout. PEP 508's #subdirectory= URL fragment tells pip to enter that subdirectory before running the build. The PyPI wheel sidesteps this entirely (it's pre-built), so this hint will go away when we revert to a >=1.4.1 version range. Co-Authored-By: Claude Opus 4.7 (1M context) --- Python/daily-insight/pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Python/daily-insight/pyproject.toml b/Python/daily-insight/pyproject.toml index 4cc6f97..ebb2631 100644 --- a/Python/daily-insight/pyproject.toml +++ b/Python/daily-insight/pyproject.toml @@ -15,7 +15,11 @@ dependencies = [ # — needed because /evaluate's sample_size=1 default produces one session # and the released 1.4.0 ToolUseEvaluator raises ValueError on fewer than 2). # Swap back to "railtracks[visual]>=1.4.1" (or higher) once that release ships. - "railtracks[visual] @ git+https://github.com/RailtownAI/railtracks.git@4e4ed572d76f946908bb3fd441443be8903ae8de", + # The #subdirectory hint is required: the railtracks repo is a monorepo + # (packages/, pdoc/, docs/ at the root), so pip's flat-layout discovery + # fails when pointed at the root. The actual package lives at + # packages/railtracks/ with its own pyproject.toml + src/ layout. + "railtracks[visual] @ git+https://github.com/RailtownAI/railtracks.git@4e4ed572d76f946908bb3fd441443be8903ae8de#subdirectory=packages/railtracks", "railtownai>=2.0.13", "pydantic>=2.0", "python-dotenv>=1.0.0", From f778dc9f284d6f0d91b00b92d0628ad4684fc0c1 Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 12 Jun 2026 15:30:28 -0700 Subject: [PATCH 10/19] wire agent_run_id: evaluate historical sessions fetched from Conductr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /evaluate gains a second mode. When the body sets agent_run_id, the service fetches that single session from Conductr's platform API via railtownai.get_agent_runs([str(agent_run_id)]) (new in railtownai 2.0.14), stages the returned payload in the same request-scoped tempdir the fresh mode uses, and runs evaluators against just that session. sample_size is ignored in this mode and insights=[] in the response since no fresh generation happens. Fresh-mode behaviour is unchanged. AgentRunsNotInitializedError / AgentRunFetchError bubble up to the global exception handler and surface in the 500 JSON body — clean signal for the operator when CONDUCTR_PROJECT_PAT or CONDUCTR_PROJECT_ID is missing. Co-Authored-By: Claude Opus 4.7 (1M context) --- Python/daily-insight/README.md | 25 ++++++- Python/daily-insight/pyproject.toml | 2 +- Python/daily-insight/src/controllers/api.py | 33 ++++---- .../src/services/evaluation_service.py | 75 +++++++++++++++---- 4 files changed, 100 insertions(+), 35 deletions(-) diff --git a/Python/daily-insight/README.md b/Python/daily-insight/README.md index a298039..388bffe 100644 --- a/Python/daily-insight/README.md +++ b/Python/daily-insight/README.md @@ -46,7 +46,23 @@ Expect each call to take a few seconds — the agent makes one Anthropic call pl ## Evaluation -`POST /evaluate` is a self-contained smoke test: it triggers `sample_size` fresh `/insight` runs (default 1, capped at 5), then runs Railtracks evaluators against the resulting sessions and returns the scores. When `RAILTOWN_API_KEY` is set, each result also uploads to Conductr via `railtownai.upload_agent_evaluation`. +`POST /evaluate` runs Railtracks evaluators against agent sessions and returns the scores. Two modes: + +| Mode | Trigger | Cost | What it evaluates | +|---|---|---|---| +| **Fresh** (default) | body omits `agent_run_id` | ≈ `sample_size · 3` LLM calls | Generates `sample_size` brand-new `/insight` runs (default 1, capped at 5) and scores them. Self-contained smoke test. | +| **Historical** | body sets `agent_run_id` | ≈ 2 LLM calls (judge only) | Fetches the named past run from Conductr via [`railtownai.get_agent_runs`](https://pypi.org/project/railtownai/) and scores that single session. Replays a real production interaction without re-spending generation cost. | + +```bash +# Fresh (default) +curl -X POST .../evaluate -H 'Content-Type: application/json' -d '{"sample_size": 1}' + +# Historical +curl -X POST .../evaluate -H 'Content-Type: application/json' \ + -d '{"agent_run_id": "0466964a-1234-5678-9abc-def012345678"}' +``` + +When `RAILTOWN_API_KEY` is set, each `EvaluationResult` also uploads to Conductr via `railtownai.upload_agent_evaluation`. Three evaluators run per call (see [`agents/evaluations.py`](src/agents/evaluations.py)): @@ -56,7 +72,12 @@ Three evaluators run per call (see [`agents/evaluations.py`](src/agents/evaluati - **`FormatCompliance`** (Compliant / MinorDeviation / MajorDeviation) — does the response follow the strict one-line-per-metric format that the C# card expects to render unchanged? - **`FactualGrounding`** (FullyGrounded / PartiallyGrounded / Hallucinated) — does every value and trend in the response trace back to the tool output, or is the model inventing things? -Cost per call ≈ `sample_size · 3` LLM calls (one to generate, two for the judge metrics). Override the judge model via `EVAL_JUDGE_MODEL`; defaults to `LLM_MODEL` or `claude-haiku-4-5-20251001`. +Override the judge model via `EVAL_JUDGE_MODEL`; defaults to `LLM_MODEL` or `claude-haiku-4-5-20251001`. + +**Historical-mode prerequisites.** The agent fetches sessions from Conductr's platform API, so it needs: + +- `CONDUCTR_PROJECT_ID` — already present in deployed environments (the deploy tooling seeds it). +- `CONDUCTR_PROJECT_PAT` — a project-level access token generated in the Conductr UI under *project → Secret Tokens*. Not yet auto-seeded by the deploy tooling; for a deployed agent, set it once with `az keyvault secret set --vault-name --name CONDUCTR-PROJECT-PAT --value ''` and roll a config-only revision to wire it onto the container. ## Environment variables diff --git a/Python/daily-insight/pyproject.toml b/Python/daily-insight/pyproject.toml index ebb2631..1419095 100644 --- a/Python/daily-insight/pyproject.toml +++ b/Python/daily-insight/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ # fails when pointed at the root. The actual package lives at # packages/railtracks/ with its own pyproject.toml + src/ layout. "railtracks[visual] @ git+https://github.com/RailtownAI/railtracks.git@4e4ed572d76f946908bb3fd441443be8903ae8de#subdirectory=packages/railtracks", - "railtownai>=2.0.13", + "railtownai>=2.0.14", "pydantic>=2.0", "python-dotenv>=1.0.0", "fastapi>=0.110.0", diff --git a/Python/daily-insight/src/controllers/api.py b/Python/daily-insight/src/controllers/api.py index 19276af..337bb4c 100644 --- a/Python/daily-insight/src/controllers/api.py +++ b/Python/daily-insight/src/controllers/api.py @@ -38,15 +38,11 @@ class EvaluateRequest(BaseModel): default=1, ge=1, le=5, - description="Number of fresh insight runs to generate and evaluate (capped at 5).", + description="Number of fresh insight runs to generate and evaluate (capped at 5). Ignored when agent_run_id is provided.", ) - # Forward-compat: when populated, future versions will fetch the named - # historical run from Conductr and evaluate that instead of generating - # fresh runs. Currently accepted (so existing clients don't need updating - # later) but logged + ignored — `sample_size` still drives behaviour. agent_run_id: Optional[UUID] = Field( default=None, - description="Optional. Reserved for future use — will identify a specific historical agent run to evaluate. Currently logged and ignored; sample_size still drives behaviour.", + description="Optional. When set, fetches the named historical agent run from Conductr (via railtownai.get_agent_runs) and evaluates that single session instead of generating fresh ones. Requires CONDUCTR_PROJECT_PAT and CONDUCTR_PROJECT_ID on the agent.", ) @@ -118,15 +114,20 @@ async def generate_insight() -> DailyInsight: @app.post("/evaluate", response_model=EvaluationRun) async def evaluate(req: EvaluateRequest = EvaluateRequest()) -> EvaluationRun: - """Run the agent N times and evaluate the resulting sessions. + """Evaluate either fresh insight runs or a named historical run. - Cost per call ≈ ``sample_size · 3`` LLM calls — one to generate the insight, - two for the judge metrics (FormatCompliance + FactualGrounding). The - ToolUseEvaluator and LLMInferenceEvaluator are cost-free local checks. + Fresh mode (``agent_run_id`` absent): generates ``sample_size`` insight + runs and scores them. Cost ≈ ``sample_size · 3`` LLM calls (1 to generate, + 2 for the FormatCompliance + FactualGrounding judge metrics). + + Historical mode (``agent_run_id`` set): fetches the named session from + Conductr and scores it. Cost ≈ 2 LLM calls (judge only). Requires + ``CONDUCTR_PROJECT_PAT`` and ``CONDUCTR_PROJECT_ID`` on the agent. + + ``ToolUseEvaluator`` and ``LLMInferenceEvaluator`` are free local checks in + either mode. """ - if req.agent_run_id is not None: - logger.info( - "agent_run_id=%s provided but not yet wired — generating fresh runs", - req.agent_run_id, - ) - return await EvaluationService().run(sample_size=req.sample_size) + return await EvaluationService().run( + sample_size=req.sample_size, + agent_run_id=req.agent_run_id, + ) diff --git a/Python/daily-insight/src/services/evaluation_service.py b/Python/daily-insight/src/services/evaluation_service.py index ccef8b1..6da5cf8 100644 --- a/Python/daily-insight/src/services/evaluation_service.py +++ b/Python/daily-insight/src/services/evaluation_service.py @@ -1,4 +1,4 @@ -"""Generate fresh insight runs and evaluate them with the configured evaluators.""" +"""Generate fresh insight runs (or fetch a historical run) and evaluate them.""" from __future__ import annotations @@ -8,6 +8,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +from uuid import UUID import railtownai from railtracks import evaluations as evals @@ -34,22 +35,60 @@ def _upload_evaluation(payload: dict[str, Any]) -> None: logger.exception("Failed to upload evaluation result to Railtown") -class EvaluationService: - """Run N fresh insight invocations and evaluate them with the agent's evaluators.""" +def _fetch_historical_session(agent_run_id: UUID) -> dict[str, Any]: + """Fetch a session payload from Conductr via railtownai.get_agent_runs. - async def run(self, sample_size: int = 1) -> EvaluationRun: - sample_size = max(1, min(int(sample_size), _MAX_SAMPLE_SIZE)) - started_at = datetime.now(timezone.utc) + Returns the nested session shape (session_id, runs: [...]) that + extract_agent_data_points consumes. Raises whatever the SDK raises — + AgentRunsNotInitializedError when CONDUCTR_PROJECT_PAT/CONDUCTR_PROJECT_ID + aren't set, AgentRunFetchError on HTTP/parse failures — and the global + exception handler shapes those into a 500 JSON body. + """ + payloads = railtownai.get_agent_runs([str(agent_run_id)]) + if not payloads: + raise RuntimeError( + f"Conductr returned no payload for agent_run_id={agent_run_id}" + ) + return payloads[0] - insight_service = InsightService() - insights: list[DailyInsight] = [] - session_payloads: list[dict[str, Any]] = [] - for i in range(sample_size): - logger.info("Generating insight run %s/%s", i + 1, sample_size) - insight, payload = await insight_service.run_and_capture_session() - insights.append(insight) - session_payloads.append(payload) +class EvaluationService: + """Generate or fetch insight session(s) and evaluate them. + + Two modes: + - **Fresh** (default) — generate ``sample_size`` insight runs via + ``InsightService.run_and_capture_session`` and evaluate them. + - **Historical** — when ``agent_run_id`` is provided, fetch that specific + run from Conductr via ``railtownai.get_agent_runs`` and evaluate only + that session. ``sample_size`` is ignored in this mode. + """ + + async def run( + self, + sample_size: int = 1, + agent_run_id: UUID | None = None, + ) -> EvaluationRun: + started_at = datetime.now(timezone.utc) + + if agent_run_id is not None: + logger.info("Evaluating historical session agent_run_id=%s", agent_run_id) + session_payloads: list[dict[str, Any]] = [ + _fetch_historical_session(agent_run_id) + ] + insights: list[DailyInsight] = [] + effective_sample_size = 1 + else: + effective_sample_size = max(1, min(int(sample_size), _MAX_SAMPLE_SIZE)) + insight_service = InsightService() + insights = [] + session_payloads = [] + for i in range(effective_sample_size): + logger.info( + "Generating insight run %s/%s", i + 1, effective_sample_size + ) + insight, payload = await insight_service.run_and_capture_session() + insights.append(insight) + session_payloads.append(payload) # extract_agent_data_points reads JSON session files from disk. Stage the # captured payloads in a request-scoped tempdir so each /evaluate call @@ -63,7 +102,11 @@ async def run(self, sample_size: int = 1) -> EvaluationRun: session_files.append(str(fp)) data = evals.extract_agent_data_points(session_files) - logger.info("Extracted %s agent data points from %s sessions", len(data), len(session_files)) + logger.info( + "Extracted %s agent data points from %s session(s)", + len(data), + len(session_files), + ) evaluation_name = f"daily-insight-{started_at.strftime('%Y%m%dT%H%M%SZ')}" @@ -80,7 +123,7 @@ async def run(self, sample_size: int = 1) -> EvaluationRun: completed_at = datetime.now(timezone.utc) return EvaluationRun( - sample_size=sample_size, + sample_size=effective_sample_size, started_at=started_at, completed_at=completed_at, insights=insights, From f68b33094fe44d1d0b2c0db7971c39213261867e Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 12 Jun 2026 15:52:37 -0700 Subject: [PATCH 11/19] rename /evaluate endpoint to /evals/run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /evals/ namespace leaves room for future eval-related endpoints (list, retrieve a past evaluation by id, etc.) without crowding the top-level route table. /evals/run is the verb-y entry point that triggers an evaluation; siblings under /evals/ would be CRUD-y reads against persisted results. Wire-level breaking change for the endpoint URL only — request and response shapes are unchanged. The renamed handler is now run_evaluation (the previous `evaluate` name shadowed the imported evals function in some contexts). Co-Authored-By: Claude Opus 4.7 (1M context) --- Python/daily-insight/README.md | 8 ++++---- Python/daily-insight/pyproject.toml | 2 +- Python/daily-insight/src/controllers/api.py | 6 +++--- Python/daily-insight/src/models/evaluation.py | 2 +- Python/daily-insight/src/services/evaluation_service.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Python/daily-insight/README.md b/Python/daily-insight/README.md index 388bffe..53296f0 100644 --- a/Python/daily-insight/README.md +++ b/Python/daily-insight/README.md @@ -28,7 +28,7 @@ curl http://127.0.0.1:8000/health curl -X POST http://127.0.0.1:8000/insight # Generate fresh insight(s) and evaluate them -curl -X POST http://127.0.0.1:8000/evaluate -H 'Content-Type: application/json' -d '{"sample_size": 1}' +curl -X POST http://127.0.0.1:8000/evals/run -H 'Content-Type: application/json' -d '{"sample_size": 1}' ``` `POST /insight` runs the Railtracks agent against your engine and returns: @@ -46,7 +46,7 @@ Expect each call to take a few seconds — the agent makes one Anthropic call pl ## Evaluation -`POST /evaluate` runs Railtracks evaluators against agent sessions and returns the scores. Two modes: +`POST /evals/run` runs Railtracks evaluators against agent sessions and returns the scores. Two modes: | Mode | Trigger | Cost | What it evaluates | |---|---|---|---| @@ -55,10 +55,10 @@ Expect each call to take a few seconds — the agent makes one Anthropic call pl ```bash # Fresh (default) -curl -X POST .../evaluate -H 'Content-Type: application/json' -d '{"sample_size": 1}' +curl -X POST .../evals/run -H 'Content-Type: application/json' -d '{"sample_size": 1}' # Historical -curl -X POST .../evaluate -H 'Content-Type: application/json' \ +curl -X POST .../evals/run -H 'Content-Type: application/json' \ -d '{"agent_run_id": "0466964a-1234-5678-9abc-def012345678"}' ``` diff --git a/Python/daily-insight/pyproject.toml b/Python/daily-insight/pyproject.toml index 1419095..ba7dacf 100644 --- a/Python/daily-insight/pyproject.toml +++ b/Python/daily-insight/pyproject.toml @@ -12,7 +12,7 @@ dependencies = [ "rail-engine>=0.2.1", # Pinned to commit 4e4ed57 for the ToolUseEvaluator single-session fix # (https://github.com/RailtownAI/railtracks/commit/4e4ed572d76f946908bb3fd441443be8903ae8de - # — needed because /evaluate's sample_size=1 default produces one session + # — needed because /evals/run's sample_size=1 default produces one session # and the released 1.4.0 ToolUseEvaluator raises ValueError on fewer than 2). # Swap back to "railtracks[visual]>=1.4.1" (or higher) once that release ships. # The #subdirectory hint is required: the railtracks repo is a monorepo diff --git a/Python/daily-insight/src/controllers/api.py b/Python/daily-insight/src/controllers/api.py index 337bb4c..278f681 100644 --- a/Python/daily-insight/src/controllers/api.py +++ b/Python/daily-insight/src/controllers/api.py @@ -7,7 +7,7 @@ Endpoints: - GET /health — liveness probe - POST /insight — run the Railtracks agent against Railengine and return a DailyInsight -- POST /evaluate — run the agent N times, evaluate the sessions, return scores +- POST /evals/run — run the agent N times (or fetch a historical session), evaluate, return scores """ from __future__ import annotations @@ -112,8 +112,8 @@ async def generate_insight() -> DailyInsight: return await InsightService().run() -@app.post("/evaluate", response_model=EvaluationRun) -async def evaluate(req: EvaluateRequest = EvaluateRequest()) -> EvaluationRun: +@app.post("/evals/run", response_model=EvaluationRun) +async def run_evaluation(req: EvaluateRequest = EvaluateRequest()) -> EvaluationRun: """Evaluate either fresh insight runs or a named historical run. Fresh mode (``agent_run_id`` absent): generates ``sample_size`` insight diff --git a/Python/daily-insight/src/models/evaluation.py b/Python/daily-insight/src/models/evaluation.py index a1bef48..8b45a2d 100644 --- a/Python/daily-insight/src/models/evaluation.py +++ b/Python/daily-insight/src/models/evaluation.py @@ -1,4 +1,4 @@ -"""Evaluation run result returned by POST /evaluate.""" +"""Evaluation run result returned by POST /evals/run.""" from __future__ import annotations diff --git a/Python/daily-insight/src/services/evaluation_service.py b/Python/daily-insight/src/services/evaluation_service.py index 6da5cf8..7384120 100644 --- a/Python/daily-insight/src/services/evaluation_service.py +++ b/Python/daily-insight/src/services/evaluation_service.py @@ -91,7 +91,7 @@ async def run( session_payloads.append(payload) # extract_agent_data_points reads JSON session files from disk. Stage the - # captured payloads in a request-scoped tempdir so each /evaluate call + # captured payloads in a request-scoped tempdir so each /evals/run call # sees exactly its own sessions — no leakage between concurrent requests. with tempfile.TemporaryDirectory(prefix="daily-insight-eval-") as tmpdir: tmp = Path(tmpdir) From fe9f6fa590aadf19af2aa08784257769d8971cd2 Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 12 Jun 2026 16:02:47 -0700 Subject: [PATCH 12/19] surface railtownai.upload_agent_evaluation's silent-False failure path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit railtownai.upload_agent_evaluation catches all non-config exceptions and returns False without raising or logging — HTTP-layer failures (token rejected by Conductr, ingestion endpoint unreachable, rail-engine-ingest errors) look indistinguishable from success at the call site. The SDK also explicitly suppresses rail-engine-ingest INFO logs (so the underlying HTTP response never reaches our root logger). Three changes: 1. Gate on EVALUATIONS_API_TOKEN (the actual prerequisite) instead of railtownai.get_railtown_handler() (which only signals RAILTOWN_API_KEY init — a different feature). 2. Check the return value of upload_agent_evaluation. The previous code logged "uploaded to Conductr" on every call regardless of outcome. 3. Log loudly when the SDK returns False so silent failures become visible in container logs, with diagnostic guidance pointing at the suppressed rail-engine-ingest logger. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/services/evaluation_service.py | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/Python/daily-insight/src/services/evaluation_service.py b/Python/daily-insight/src/services/evaluation_service.py index 7384120..717a37f 100644 --- a/Python/daily-insight/src/services/evaluation_service.py +++ b/Python/daily-insight/src/services/evaluation_service.py @@ -4,6 +4,7 @@ import json import logging +import os import tempfile from datetime import datetime, timezone from pathlib import Path @@ -25,14 +26,37 @@ def _upload_evaluation(payload: dict[str, Any]) -> None: - """Callback handed to evals.evaluate — uploads each EvaluationResult to Conductr.""" - if railtownai.get_railtown_handler() is None: + """Callback handed to evals.evaluate — uploads each EvaluationResult to Conductr. + + Skips silently when EVALUATIONS_API_TOKEN is unset (local dev without the + upload token). Otherwise logs success, the SDK's silent-False generic-error + path, or any raised exception (EvaluationsNotInitializedError / + EvaluationsValidationError). + + Why explicit-False logging: railtownai.upload_agent_evaluation catches all + non-config exceptions and returns False without raising or logging. Without + surfacing that path explicitly, HTTP-layer failures (auth rejection by + Conductr, ingestion endpoint unreachable, rail-engine-ingest errors) look + indistinguishable from success — the previous wrapper logged + "uploaded to Conductr" on every call regardless of outcome. + """ + if not os.environ.get("EVALUATIONS_API_TOKEN", "").strip(): return try: - railtownai.upload_agent_evaluation(payload) - logger.info("Evaluation result uploaded to Conductr") + success = railtownai.upload_agent_evaluation(payload) + if success: + logger.info("Evaluation result uploaded to Conductr") + else: + logger.error( + "railtownai.upload_agent_evaluation returned False — upload " + "failed without raising. Likely causes: token rejected by " + "Conductr, ingestion endpoint unreachable, or rail-engine-ingest " + "HTTP error. The SDK suppresses details; bump the " + "`railtown.engine.ingest` logger to DEBUG to see the HTTP " + "exchange." + ) except Exception: - logger.exception("Failed to upload evaluation result to Railtown") + logger.exception("Evaluation upload raised") def _fetch_historical_session(agent_run_id: UUID) -> dict[str, Any]: From 623a806a6be85e5e3d6c82cc67734b84a2f95863 Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 12 Jun 2026 16:16:15 -0700 Subject: [PATCH 13/19] upload evaluations via asyncio.to_thread, not the sync payload_callback hook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit railtownai.upload_agent_evaluation is a sync function that drives its async implementation via asyncio.run(). When invoked from inside FastAPI's running event loop (which is where evals.evaluate's payload_callback fires from), asyncio.run() raises RuntimeError — the SDK catches that as a generic Exception and returns False without surfacing the cause. The unscheduled coroutine leaks a "coroutine was never awaited" warning to stderr. Confirmed by container logs after the previous diagnostic-logging commit: RuntimeWarning: coroutine '_upload_agent_evaluation_async' was never awaited return False Fix: drop the payload_callback hook, iterate evaluation_results after evals.evaluate() returns, and run each upload via asyncio.to_thread so the SDK gets the clean thread-local state (no running loop) it expects. Batches the whole list into a single SDK call for efficiency — one ingest session instead of one per result. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/services/evaluation_service.py | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/Python/daily-insight/src/services/evaluation_service.py b/Python/daily-insight/src/services/evaluation_service.py index 717a37f..6d6d207 100644 --- a/Python/daily-insight/src/services/evaluation_service.py +++ b/Python/daily-insight/src/services/evaluation_service.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import json import logging import os @@ -25,27 +26,39 @@ _MAX_SAMPLE_SIZE = 5 -def _upload_evaluation(payload: dict[str, Any]) -> None: - """Callback handed to evals.evaluate — uploads each EvaluationResult to Conductr. +async def _upload_evaluations(results: list[Any]) -> None: + """Upload completed EvaluationResults to Conductr after evals.evaluate finishes. Skips silently when EVALUATIONS_API_TOKEN is unset (local dev without the - upload token). Otherwise logs success, the SDK's silent-False generic-error - path, or any raised exception (EvaluationsNotInitializedError / + upload token). Logs success, the SDK's silent-False generic-error path, or + any raised exception (EvaluationsNotInitializedError / EvaluationsValidationError). - Why explicit-False logging: railtownai.upload_agent_evaluation catches all - non-config exceptions and returns False without raising or logging. Without - surfacing that path explicitly, HTTP-layer failures (auth rejection by - Conductr, ingestion endpoint unreachable, rail-engine-ingest errors) look - indistinguishable from success — the previous wrapper logged - "uploaded to Conductr" on every call regardless of outcome. + Why this is async + offloaded to a worker thread instead of using + evals.evaluate's payload_callback hook: + + railtownai.upload_agent_evaluation is a *sync* function that drives its + async implementation via asyncio.run(). asyncio.run() raises RuntimeError + when invoked from within an already-running event loop — exactly what + happens when FastAPI calls into this service. The SDK catches that as a + generic Exception and returns False, leaving an unawaited coroutine + warning in stderr. asyncio.to_thread runs the call in a worker thread + with clean thread-local state where asyncio.run() works as designed. """ if not os.environ.get("EVALUATIONS_API_TOKEN", "").strip(): return + if not results: + return + + payloads = [r.model_dump(mode="json") for r in results] try: - success = railtownai.upload_agent_evaluation(payload) + success = await asyncio.to_thread( + railtownai.upload_agent_evaluation, payloads + ) if success: - logger.info("Evaluation result uploaded to Conductr") + logger.info( + "Uploaded %s evaluation result(s) to Conductr", len(payloads) + ) else: logger.error( "railtownai.upload_agent_evaluation returned False — upload " @@ -135,16 +148,20 @@ async def run( evaluation_name = f"daily-insight-{started_at.strftime('%Y%m%dT%H%M%SZ')}" # agent_selection=False + agents=[...] keeps evaluate() headless. - # payload_callback fires once per EvaluationResult as it completes. + # No payload_callback: the SDK's sync upload helper calls + # asyncio.run() under the hood which fails from inside FastAPI's + # running event loop. Upload via asyncio.to_thread after evaluate + # returns instead — see _upload_evaluations. evaluation_results = evals.evaluate( data=data, evaluators=build_evaluators(), agents=[_AGENT_NAME], agent_selection=False, name=evaluation_name, - payload_callback=_upload_evaluation, ) + await _upload_evaluations(evaluation_results) + completed_at = datetime.now(timezone.utc) return EvaluationRun( sample_size=effective_sample_size, From dce0a65f2ccc771a6d8a9051f01e0b814e837167 Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 12 Jun 2026 16:54:04 -0700 Subject: [PATCH 14/19] include agent_run_id in evaluation_name for historical-mode runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Historical mode now produces names like: daily-insight--20260612T233526Z Fresh mode keeps the original timestamp-only shape since there's no single run id to attach (each of the N generated sessions has its own). Makes it trivial to grep Conductr's evaluation list for "did I evaluate this specific session" — previously the only way to correlate was via timestamp. Co-Authored-By: Claude Opus 4.7 (1M context) --- Python/daily-insight/src/services/evaluation_service.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Python/daily-insight/src/services/evaluation_service.py b/Python/daily-insight/src/services/evaluation_service.py index 6d6d207..d5908df 100644 --- a/Python/daily-insight/src/services/evaluation_service.py +++ b/Python/daily-insight/src/services/evaluation_service.py @@ -145,7 +145,11 @@ async def run( len(session_files), ) - evaluation_name = f"daily-insight-{started_at.strftime('%Y%m%dT%H%M%SZ')}" + timestamp = started_at.strftime("%Y%m%dT%H%M%SZ") + if agent_run_id is not None: + evaluation_name = f"daily-insight-{agent_run_id}-{timestamp}" + else: + evaluation_name = f"daily-insight-{timestamp}" # agent_selection=False + agents=[...] keeps evaluate() headless. # No payload_callback: the SDK's sync upload helper calls From e966d5f88b9716948625740daeb7b4e339a8ee05 Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Mon, 15 Jun 2026 08:56:18 -0700 Subject: [PATCH 15/19] fix linter errors and supress linter verbose output - ascii art and successes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - hadolint DL3008 on Dockerfile: add ignore directive for the apt-version pin rule. Pinning git to a specific apt version forces a Dockerfile edit on every base-image refresh, which is brittle for a transient dependency that only exists until we move back to a PyPI railtracks release. - black on src/config/env.py: wrap the over-88-char REQUIRED_ENV_VARS list comprehension across multiple lines as black prefers. - black on src/services/evaluation_service.py: the inverse — collapse the asyncio.to_thread call and the logger.info call back to single lines now that they fit under 88 chars. - flake8 E501 on src/agents/insight_agent.py: split the 163-char prompt line at sentence-ish boundaries. Newlines inside a paragraph are semantically equivalent to spaces for the LLM. - flake8 E501 on src/controllers/api.py: wrap the long pydantic Field description= strings using Python implicit string concatenation inside the description=(...) parens. Co-Authored-By: Claude Opus 4.7 (1M context) --- Python/daily-insight/Dockerfile | 3 +++ Python/daily-insight/src/agents/insight_agent.py | 4 +++- Python/daily-insight/src/config/env.py | 4 +++- Python/daily-insight/src/controllers/api.py | 12 ++++++++++-- .../daily-insight/src/services/evaluation_service.py | 8 ++------ 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/Python/daily-insight/Dockerfile b/Python/daily-insight/Dockerfile index 5e1b714..f27c198 100644 --- a/Python/daily-insight/Dockerfile +++ b/Python/daily-insight/Dockerfile @@ -4,6 +4,9 @@ WORKDIR /app # git is needed to pip-install railtracks from its pinned commit (see # pyproject.toml). Remove once we move back to a PyPI-released version. +# hadolint ignore=DL3008 +# Pinning git's apt version would force a Dockerfile edit on every base-image +# refresh; ignore the apt-version-pin rule for this transient dependency. RUN apt-get update \ && apt-get install -y --no-install-recommends git \ && rm -rf /var/lib/apt/lists/* diff --git a/Python/daily-insight/src/agents/insight_agent.py b/Python/daily-insight/src/agents/insight_agent.py index 6ed4064..c40cfc1 100644 --- a/Python/daily-insight/src/agents/insight_agent.py +++ b/Python/daily-insight/src/agents/insight_agent.py @@ -13,7 +13,9 @@ SYSTEM_PROMPT = """You are an automated reviewer for a status page dashboard. -Use AT MOST 1 tool call. Call `get_recent_metrics` with limit=50 to fetch the most recent metric records, then summarize. Do not make additional exploratory calls. +Use AT MOST 1 tool call. Call `get_recent_metrics` with limit=50 to fetch +the most recent metric records, then summarize. Do not make additional +exploratory calls. Output format (strict): - One single line per metric, nothing else. diff --git a/Python/daily-insight/src/config/env.py b/Python/daily-insight/src/config/env.py index 3083b93..0b7a158 100644 --- a/Python/daily-insight/src/config/env.py +++ b/Python/daily-insight/src/config/env.py @@ -47,6 +47,8 @@ def validate_required_env() -> None: Run ``configure_runtime_env`` first so the LLM_API_KEY alias has a chance to populate ANTHROPIC_API_KEY before this check. """ - missing = [name for name in REQUIRED_ENV_VARS if not os.environ.get(name, "").strip()] + missing = [ + name for name in REQUIRED_ENV_VARS if not os.environ.get(name, "").strip() + ] if missing: raise MissingEnvVarsError(missing) diff --git a/Python/daily-insight/src/controllers/api.py b/Python/daily-insight/src/controllers/api.py index 278f681..d99b97b 100644 --- a/Python/daily-insight/src/controllers/api.py +++ b/Python/daily-insight/src/controllers/api.py @@ -38,11 +38,19 @@ class EvaluateRequest(BaseModel): default=1, ge=1, le=5, - description="Number of fresh insight runs to generate and evaluate (capped at 5). Ignored when agent_run_id is provided.", + description=( + "Number of fresh insight runs to generate and evaluate (capped at 5). " + "Ignored when agent_run_id is provided." + ), ) agent_run_id: Optional[UUID] = Field( default=None, - description="Optional. When set, fetches the named historical agent run from Conductr (via railtownai.get_agent_runs) and evaluates that single session instead of generating fresh ones. Requires CONDUCTR_PROJECT_PAT and CONDUCTR_PROJECT_ID on the agent.", + description=( + "Optional. When set, fetches the named historical agent run from " + "Conductr (via railtownai.get_agent_runs) and evaluates that single " + "session instead of generating fresh ones. Requires " + "CONDUCTR_PROJECT_PAT and CONDUCTR_PROJECT_ID on the agent." + ), ) diff --git a/Python/daily-insight/src/services/evaluation_service.py b/Python/daily-insight/src/services/evaluation_service.py index d5908df..f49a1c7 100644 --- a/Python/daily-insight/src/services/evaluation_service.py +++ b/Python/daily-insight/src/services/evaluation_service.py @@ -52,13 +52,9 @@ async def _upload_evaluations(results: list[Any]) -> None: payloads = [r.model_dump(mode="json") for r in results] try: - success = await asyncio.to_thread( - railtownai.upload_agent_evaluation, payloads - ) + success = await asyncio.to_thread(railtownai.upload_agent_evaluation, payloads) if success: - logger.info( - "Uploaded %s evaluation result(s) to Conductr", len(payloads) - ) + logger.info("Uploaded %s evaluation result(s) to Conductr", len(payloads)) else: logger.error( "railtownai.upload_agent_evaluation returned False — upload " From fbe9d69be8700655565af0501f87eccba09160f6 Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Mon, 15 Jun 2026 09:07:08 -0700 Subject: [PATCH 16/19] move hadolint DL3008 ignore directive directly above RUN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous attempt placed the directive earlier in the comment block, with two more explanatory comment lines between it and the RUN instruction. Hadolint binds the ignore comment to the *next instruction* — intervening comments break that binding, so the warning kept firing in CI. Reorder so the ignore comment is the final comment before RUN; the explanation stays above it in the same block. Co-Authored-By: Claude Opus 4.7 (1M context) --- Python/daily-insight/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/daily-insight/Dockerfile b/Python/daily-insight/Dockerfile index f27c198..93ca095 100644 --- a/Python/daily-insight/Dockerfile +++ b/Python/daily-insight/Dockerfile @@ -4,9 +4,10 @@ WORKDIR /app # git is needed to pip-install railtracks from its pinned commit (see # pyproject.toml). Remove once we move back to a PyPI-released version. -# hadolint ignore=DL3008 # Pinning git's apt version would force a Dockerfile edit on every base-image -# refresh; ignore the apt-version-pin rule for this transient dependency. +# refresh; the DL3008 ignore directive must sit directly above the RUN to +# bind to it (intervening comments break the association). +# hadolint ignore=DL3008 RUN apt-get update \ && apt-get install -y --no-install-recommends git \ && rm -rf /var/lib/apt/lists/* From 36374e284a823f84fbd4445992d09ade71c61e87 Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Mon, 15 Jun 2026 10:56:31 -0700 Subject: [PATCH 17/19] unpin railtracks: revert to >=1.4.1 now that the fix shipped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit railtracks 1.4.1 (PyPI, 2026-06-15) includes the ToolUseEvaluator single-session fix from commit 4e4ed57 (2026-06-11) that we'd been pinning. Reverting to a version range: - pyproject.toml: drop the git+url with #subdirectory hint, restore the simple `railtracks[visual]>=1.4.1` line. - Dockerfile: drop the apt-get install git layer and its hadolint DL3008 ignore directive — git was only there to clone railtracks from source during the pin, no longer needed for a PyPI wheel. Smaller image, faster builds, simpler dep graph. Co-Authored-By: Claude Opus 4.7 (1M context) --- Python/daily-insight/Dockerfile | 10 ---------- Python/daily-insight/pyproject.toml | 11 +---------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/Python/daily-insight/Dockerfile b/Python/daily-insight/Dockerfile index 93ca095..1eae936 100644 --- a/Python/daily-insight/Dockerfile +++ b/Python/daily-insight/Dockerfile @@ -2,16 +2,6 @@ FROM python:3.10-slim WORKDIR /app -# git is needed to pip-install railtracks from its pinned commit (see -# pyproject.toml). Remove once we move back to a PyPI-released version. -# Pinning git's apt version would force a Dockerfile edit on every base-image -# refresh; the DL3008 ignore directive must sit directly above the RUN to -# bind to it (intervening comments break the association). -# hadolint ignore=DL3008 -RUN apt-get update \ - && apt-get install -y --no-install-recommends git \ - && rm -rf /var/lib/apt/lists/* - # Install the package + deps. Copying pyproject.toml + src separately lets # Docker cache the dependency install across source-only changes. COPY pyproject.toml ./ diff --git a/Python/daily-insight/pyproject.toml b/Python/daily-insight/pyproject.toml index ba7dacf..02574f6 100644 --- a/Python/daily-insight/pyproject.toml +++ b/Python/daily-insight/pyproject.toml @@ -10,16 +10,7 @@ readme = "README.md" requires-python = ">=3.10" dependencies = [ "rail-engine>=0.2.1", - # Pinned to commit 4e4ed57 for the ToolUseEvaluator single-session fix - # (https://github.com/RailtownAI/railtracks/commit/4e4ed572d76f946908bb3fd441443be8903ae8de - # — needed because /evals/run's sample_size=1 default produces one session - # and the released 1.4.0 ToolUseEvaluator raises ValueError on fewer than 2). - # Swap back to "railtracks[visual]>=1.4.1" (or higher) once that release ships. - # The #subdirectory hint is required: the railtracks repo is a monorepo - # (packages/, pdoc/, docs/ at the root), so pip's flat-layout discovery - # fails when pointed at the root. The actual package lives at - # packages/railtracks/ with its own pyproject.toml + src/ layout. - "railtracks[visual] @ git+https://github.com/RailtownAI/railtracks.git@4e4ed572d76f946908bb3fd441443be8903ae8de#subdirectory=packages/railtracks", + "railtracks[visual]>=1.4.1", "railtownai>=2.0.14", "pydantic>=2.0", "python-dotenv>=1.0.0", From e2163ab0bc9ce4f6022ac2aeee7facb14b13c7b2 Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 19 Jun 2026 11:49:20 -0700 Subject: [PATCH 18/19] accept agent_run_ids list on /evals/run for batch eval The Conductr Hosted endpoint now POSTs { agent_run_ids: [...] } so the agent has to accept a list. Historical mode now fetches all named runs in one get_agent_runs call and scores them under a single evaluation_name; the batch name is daily-insight-batch- when there's more than one id (single-id keeps the existing id-in-name form). --- Python/daily-insight/README.md | 8 +-- Python/daily-insight/src/controllers/api.py | 26 ++++----- .../src/services/evaluation_service.py | 53 ++++++++++++------- 3 files changed, 52 insertions(+), 35 deletions(-) diff --git a/Python/daily-insight/README.md b/Python/daily-insight/README.md index 53296f0..6884e68 100644 --- a/Python/daily-insight/README.md +++ b/Python/daily-insight/README.md @@ -50,16 +50,16 @@ Expect each call to take a few seconds — the agent makes one Anthropic call pl | Mode | Trigger | Cost | What it evaluates | |---|---|---|---| -| **Fresh** (default) | body omits `agent_run_id` | ≈ `sample_size · 3` LLM calls | Generates `sample_size` brand-new `/insight` runs (default 1, capped at 5) and scores them. Self-contained smoke test. | -| **Historical** | body sets `agent_run_id` | ≈ 2 LLM calls (judge only) | Fetches the named past run from Conductr via [`railtownai.get_agent_runs`](https://pypi.org/project/railtownai/) and scores that single session. Replays a real production interaction without re-spending generation cost. | +| **Fresh** (default) | body omits `agent_run_ids` | ≈ `sample_size · 3` LLM calls | Generates `sample_size` brand-new `/insight` runs (default 1, capped at 5) and scores them. Self-contained smoke test. | +| **Historical** | body sets `agent_run_ids` | ≈ `N · 2` LLM calls (judge only) | Fetches the named past runs from Conductr via [`railtownai.get_agent_runs`](https://pypi.org/project/railtownai/) and scores them as one batch. Replays real production interactions without re-spending generation cost. | ```bash # Fresh (default) curl -X POST .../evals/run -H 'Content-Type: application/json' -d '{"sample_size": 1}' -# Historical +# Historical (one or more runs) curl -X POST .../evals/run -H 'Content-Type: application/json' \ - -d '{"agent_run_id": "0466964a-1234-5678-9abc-def012345678"}' + -d '{"agent_run_ids": ["0466964a-1234-5678-9abc-def012345678"]}' ``` When `RAILTOWN_API_KEY` is set, each `EvaluationResult` also uploads to Conductr via `railtownai.upload_agent_evaluation`. diff --git a/Python/daily-insight/src/controllers/api.py b/Python/daily-insight/src/controllers/api.py index d99b97b..b3b2f5e 100644 --- a/Python/daily-insight/src/controllers/api.py +++ b/Python/daily-insight/src/controllers/api.py @@ -40,16 +40,17 @@ class EvaluateRequest(BaseModel): le=5, description=( "Number of fresh insight runs to generate and evaluate (capped at 5). " - "Ignored when agent_run_id is provided." + "Ignored when agent_run_ids is provided." ), ) - agent_run_id: Optional[UUID] = Field( + agent_run_ids: Optional[list[UUID]] = Field( default=None, description=( - "Optional. When set, fetches the named historical agent run from " - "Conductr (via railtownai.get_agent_runs) and evaluates that single " - "session instead of generating fresh ones. Requires " - "CONDUCTR_PROJECT_PAT and CONDUCTR_PROJECT_ID on the agent." + "Optional. When set, fetches the named historical agent runs from " + "Conductr (via railtownai.get_agent_runs) and evaluates those " + "sessions as a single batch instead of generating fresh ones. " + "Requires CONDUCTR_PROJECT_PAT and CONDUCTR_PROJECT_ID on the " + "agent." ), ) @@ -122,20 +123,21 @@ async def generate_insight() -> DailyInsight: @app.post("/evals/run", response_model=EvaluationRun) async def run_evaluation(req: EvaluateRequest = EvaluateRequest()) -> EvaluationRun: - """Evaluate either fresh insight runs or a named historical run. + """Evaluate either fresh insight runs or one or more named historical runs. - Fresh mode (``agent_run_id`` absent): generates ``sample_size`` insight + Fresh mode (``agent_run_ids`` absent): generates ``sample_size`` insight runs and scores them. Cost ≈ ``sample_size · 3`` LLM calls (1 to generate, 2 for the FormatCompliance + FactualGrounding judge metrics). - Historical mode (``agent_run_id`` set): fetches the named session from - Conductr and scores it. Cost ≈ 2 LLM calls (judge only). Requires - ``CONDUCTR_PROJECT_PAT`` and ``CONDUCTR_PROJECT_ID`` on the agent. + Historical mode (``agent_run_ids`` set): fetches the named sessions from + Conductr and scores them as a single batch. Cost ≈ ``N · 2`` LLM calls + (judge only). Requires ``CONDUCTR_PROJECT_PAT`` and ``CONDUCTR_PROJECT_ID`` + on the agent. ``ToolUseEvaluator`` and ``LLMInferenceEvaluator`` are free local checks in either mode. """ return await EvaluationService().run( sample_size=req.sample_size, - agent_run_id=req.agent_run_id, + agent_run_ids=req.agent_run_ids, ) diff --git a/Python/daily-insight/src/services/evaluation_service.py b/Python/daily-insight/src/services/evaluation_service.py index f49a1c7..e903ed8 100644 --- a/Python/daily-insight/src/services/evaluation_service.py +++ b/Python/daily-insight/src/services/evaluation_service.py @@ -68,21 +68,23 @@ async def _upload_evaluations(results: list[Any]) -> None: logger.exception("Evaluation upload raised") -def _fetch_historical_session(agent_run_id: UUID) -> dict[str, Any]: - """Fetch a session payload from Conductr via railtownai.get_agent_runs. +def _fetch_historical_sessions(agent_run_ids: list[UUID]) -> list[dict[str, Any]]: + """Fetch session payloads from Conductr via railtownai.get_agent_runs. - Returns the nested session shape (session_id, runs: [...]) that + Returns the nested session shapes (session_id, runs: [...]) that extract_agent_data_points consumes. Raises whatever the SDK raises — AgentRunsNotInitializedError when CONDUCTR_PROJECT_PAT/CONDUCTR_PROJECT_ID aren't set, AgentRunFetchError on HTTP/parse failures — and the global exception handler shapes those into a 500 JSON body. """ - payloads = railtownai.get_agent_runs([str(agent_run_id)]) - if not payloads: + payloads = railtownai.get_agent_runs([str(rid) for rid in agent_run_ids]) + if len(payloads) != len(agent_run_ids): raise RuntimeError( - f"Conductr returned no payload for agent_run_id={agent_run_id}" + f"Conductr returned {len(payloads)} payload(s) for " + f"{len(agent_run_ids)} requested agent_run_ids " + f"({[str(r) for r in agent_run_ids]})" ) - return payloads[0] + return payloads class EvaluationService: @@ -91,25 +93,29 @@ class EvaluationService: Two modes: - **Fresh** (default) — generate ``sample_size`` insight runs via ``InsightService.run_and_capture_session`` and evaluate them. - - **Historical** — when ``agent_run_id`` is provided, fetch that specific - run from Conductr via ``railtownai.get_agent_runs`` and evaluate only - that session. ``sample_size`` is ignored in this mode. + - **Historical** — when ``agent_run_ids`` is provided, fetch those runs + from Conductr via ``railtownai.get_agent_runs`` and evaluate them as a + single batch. ``sample_size`` is ignored in this mode. """ async def run( self, sample_size: int = 1, - agent_run_id: UUID | None = None, + agent_run_ids: list[UUID] | None = None, ) -> EvaluationRun: started_at = datetime.now(timezone.utc) - if agent_run_id is not None: - logger.info("Evaluating historical session agent_run_id=%s", agent_run_id) - session_payloads: list[dict[str, Any]] = [ - _fetch_historical_session(agent_run_id) - ] + if agent_run_ids: + logger.info( + "Evaluating %s historical session(s): %s", + len(agent_run_ids), + [str(r) for r in agent_run_ids], + ) + session_payloads: list[dict[str, Any]] = _fetch_historical_sessions( + agent_run_ids + ) insights: list[DailyInsight] = [] - effective_sample_size = 1 + effective_sample_size = len(agent_run_ids) else: effective_sample_size = max(1, min(int(sample_size), _MAX_SAMPLE_SIZE)) insight_service = InsightService() @@ -142,8 +148,17 @@ async def run( ) timestamp = started_at.strftime("%Y%m%dT%H%M%SZ") - if agent_run_id is not None: - evaluation_name = f"daily-insight-{agent_run_id}-{timestamp}" + if agent_run_ids: + # Keep the name human-readable when there's only one run; for a + # batch, just embed the count so the name stays bounded. + if len(agent_run_ids) == 1: + evaluation_name = ( + f"daily-insight-{agent_run_ids[0]}-{timestamp}" + ) + else: + evaluation_name = ( + f"daily-insight-batch{len(agent_run_ids)}-{timestamp}" + ) else: evaluation_name = f"daily-insight-{timestamp}" From 2ebc1bc607a465fd99e0fccb5deab4a6c5cf946a Mon Sep 17 00:00:00 2001 From: Matthew Boulton Date: Fri, 19 Jun 2026 13:31:05 -0700 Subject: [PATCH 19/19] forbid extras and enforce fresh/historical mode XOR on /evals/run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds ConfigDict(extra=forbid) so misspelled keys (e.g. the old singular agent_run_id) return 422 instead of silently dropping into Fresh mode. Caps agent_run_ids at 1..10 ids and adds a model_validator that rejects bodies setting both sample_size and agent_run_ids — uses model_fields_set so the default sample_size=1 doesn't trip the XOR when only agent_run_ids is provided. Plus a black reformat in evaluation_service.py: one stray blank line removed from the import block, one f-string assignment unwrapped from parens. --- Python/daily-insight/src/controllers/api.py | 40 +++++++++++++++---- .../src/services/evaluation_service.py | 5 +-- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/Python/daily-insight/src/controllers/api.py b/Python/daily-insight/src/controllers/api.py index b3b2f5e..da94468 100644 --- a/Python/daily-insight/src/controllers/api.py +++ b/Python/daily-insight/src/controllers/api.py @@ -21,7 +21,7 @@ from fastapi import FastAPI, Request from fastapi.responses import JSONResponse from contextlib import asynccontextmanager -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator from daily_insight.config import ( MissingEnvVarsError, @@ -34,26 +34,50 @@ class EvaluateRequest(BaseModel): + # extra="forbid" rejects typo'd keys at 422 instead of silently dropping + # them — the old singular agent_run_id would otherwise slip through and + # quietly trigger a Fresh-mode generation run. + model_config = ConfigDict(extra="forbid") + sample_size: int = Field( default=1, ge=1, le=5, description=( - "Number of fresh insight runs to generate and evaluate (capped at 5). " - "Ignored when agent_run_ids is provided." + "Fresh-mode only. Number of fresh insight runs to generate and " + "evaluate (capped at 5). Mutually exclusive with agent_run_ids." ), ) agent_run_ids: Optional[list[UUID]] = Field( default=None, + min_length=1, + max_length=10, description=( - "Optional. When set, fetches the named historical agent runs from " - "Conductr (via railtownai.get_agent_runs) and evaluates those " - "sessions as a single batch instead of generating fresh ones. " - "Requires CONDUCTR_PROJECT_PAT and CONDUCTR_PROJECT_ID on the " - "agent." + "Historical-mode only. When set, fetches the named historical " + "agent runs from Conductr (via railtownai.get_agent_runs) and " + "evaluates those sessions as a single batch instead of generating " + "fresh ones. Mutually exclusive with sample_size. Requires " + "CONDUCTR_PROJECT_PAT and CONDUCTR_PROJECT_ID on the agent." ), ) + @model_validator(mode="after") + def _enforce_mode_xor(self) -> "EvaluateRequest": + """Reject bodies that try to drive both modes at once. + + Uses model_fields_set so the default sample_size=1 doesn't count as + "set" — a caller passing only agent_run_ids still validates cleanly. + """ + if ( + "sample_size" in self.model_fields_set + and "agent_run_ids" in self.model_fields_set + ): + raise ValueError( + "sample_size and agent_run_ids are mutually exclusive — " + "omit sample_size when targeting historical runs." + ) + return self + logger = logging.getLogger(__name__) diff --git a/Python/daily-insight/src/services/evaluation_service.py b/Python/daily-insight/src/services/evaluation_service.py index e903ed8..1258472 100644 --- a/Python/daily-insight/src/services/evaluation_service.py +++ b/Python/daily-insight/src/services/evaluation_service.py @@ -19,7 +19,6 @@ from daily_insight.models import DailyInsight, EvaluationRun from daily_insight.services.insight_service import InsightService - logger = logging.getLogger(__name__) _AGENT_NAME = "Daily Insight Agent" @@ -152,9 +151,7 @@ async def run( # Keep the name human-readable when there's only one run; for a # batch, just embed the count so the name stays bounded. if len(agent_run_ids) == 1: - evaluation_name = ( - f"daily-insight-{agent_run_ids[0]}-{timestamp}" - ) + evaluation_name = f"daily-insight-{agent_run_ids[0]}-{timestamp}" else: evaluation_name = ( f"daily-insight-batch{len(agent_run_ids)}-{timestamp}"