diff --git a/CSharp/Examples/RailenginePoweredStatusPage/README.md b/CSharp/Examples/RailenginePoweredStatusPage/README.md index b5beb85..ebacdb5 100644 --- a/CSharp/Examples/RailenginePoweredStatusPage/README.md +++ b/CSharp/Examples/RailenginePoweredStatusPage/README.md @@ -98,6 +98,14 @@ The prompt in `Services/DailyInsightService.cs` is deliberately tuned for short, If `Anthropic:ApiKey` is not set, the service logs a notice and exits cleanly; the insight card simply doesn't appear. +### Offloading to a Railtracks agent (optional) + +Set `DailyInsight:AgentUrl` to the base URL of a service that exposes `POST /insight` returning `{ text, generated_at, error? }` — for example the sibling [`Python/daily-insight`](../../../Python/daily-insight/) project, which runs the same prompt through a Railtracks agent backed by the Railengine Python SDK instead of MCP. + +When `AgentUrl` is set, `DailyInsightService` POSTs to `{AgentUrl}/insight` every 24 hours instead of calling Anthropic + MCP inline. `Anthropic:ApiKey` is then optional — the agent owns the LLM call. Leave `AgentUrl` empty (or unset) to keep the inline path. + +If the agent endpoint requires bearer authentication, set `DailyInsight:AgentBearerToken` to the token — `DailyInsightService` attaches it as `Authorization: Bearer ` on every POST. Leave it blank for an unauthenticated endpoint (e.g. local development on `:8000`). + > **Heads-up for local development:** the 24-hour timer is in-memory only, so each app restart triggers a fresh generation and a corresponding Anthropic API call. If you're iterating on the app you may want to comment out `AddHostedService()` in `Program.cs` until you're ready to test it. Once deployed to a long-running host, restarts are rare and this isn't a concern. ## Configuration diff --git a/CSharp/Examples/RailenginePoweredStatusPage/Services/DailyInsightService.cs b/CSharp/Examples/RailenginePoweredStatusPage/Services/DailyInsightService.cs index 9e6fb83..dee9e65 100644 --- a/CSharp/Examples/RailenginePoweredStatusPage/Services/DailyInsightService.cs +++ b/CSharp/Examples/RailenginePoweredStatusPage/Services/DailyInsightService.cs @@ -16,6 +16,8 @@ public class DailyInsightService : BackgroundService private readonly string pat; private readonly string mcpServerBaseUrl; private readonly string mcpServerName; + private readonly string agentUrl; + private readonly string agentBearerToken; private static readonly TimeSpan Interval = TimeSpan.FromHours(24); private static readonly TimeSpan RequestTimeout = TimeSpan.FromMinutes(10); @@ -47,13 +49,16 @@ public DailyInsightService( pat = configuration["RailEngine:PAT"]!; mcpServerBaseUrl = configuration["RailEngine:McpServerBaseUrl"]!.TrimEnd('/'); mcpServerName = configuration["RailEngine:McpServerName"]!; + agentUrl = (configuration["DailyInsight:AgentUrl"] ?? "").TrimEnd('/'); + agentBearerToken = configuration["DailyInsight:AgentBearerToken"] ?? ""; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - if (string.IsNullOrEmpty(apiKey)) + var useAgent = !string.IsNullOrEmpty(agentUrl); + if (!useAgent && string.IsNullOrEmpty(apiKey)) { - logger.LogInformation("Anthropic:ApiKey not configured; daily insight disabled."); + logger.LogInformation("Neither DailyInsight:AgentUrl nor Anthropic:ApiKey configured; daily insight disabled."); return; } @@ -61,7 +66,14 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { - await GenerateAsync(stoppingToken); + if (useAgent) + { + await GenerateFromAgentAsync(stoppingToken); + } + else + { + await GenerateAsync(stoppingToken); + } state.Error = null; } catch (Exception ex) @@ -75,6 +87,52 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } } + private async Task GenerateFromAgentAsync(CancellationToken ct) + { + logger.LogInformation("Starting daily insight generation via agent at {Url}…", agentUrl); + + using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + cts.CancelAfter(RequestTimeout); + + using var client = httpClientFactory.CreateClient(); + client.Timeout = Timeout.InfiniteTimeSpan; + + using var request = new HttpRequestMessage(HttpMethod.Post, $"{agentUrl}/insight"); + request.Content = new StringContent("{}", Encoding.UTF8, "application/json"); + if (!string.IsNullOrEmpty(agentBearerToken)) + { + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", agentBearerToken); + } + + using var response = await client.SendAsync(request, cts.Token); + + if (!response.IsSuccessStatusCode) + { + var errBody = await response.Content.ReadAsStringAsync(cts.Token); + throw new InvalidOperationException($"Agent returned {(int)response.StatusCode}: {errBody}"); + } + + var body = await response.Content.ReadAsStringAsync(cts.Token); + using var doc = JsonDocument.Parse(body); + var root = doc.RootElement; + + var text = root.TryGetProperty("text", out var t) ? t.GetString() ?? "" : ""; + var generatedAt = root.TryGetProperty("generated_at", out var ga) && ga.ValueKind == JsonValueKind.String + ? DateTimeOffset.Parse(ga.GetString()!) + : DateTimeOffset.UtcNow; + var agentError = root.TryGetProperty("error", out var err) && err.ValueKind == JsonValueKind.String + ? err.GetString() + : null; + + state.Text = text; + state.GeneratedAt = generatedAt; + if (!string.IsNullOrEmpty(agentError)) + { + state.Error = agentError; + } + logger.LogInformation("Daily insight generated via agent ({Length} chars)", text.Length); + } + private async Task GenerateAsync(CancellationToken ct) { logger.LogInformation("Starting daily insight generation…"); diff --git a/CSharp/Examples/RailenginePoweredStatusPage/appsettings.Development.sample.json b/CSharp/Examples/RailenginePoweredStatusPage/appsettings.Development.sample.json index 4d1fdca..0cafdf2 100644 --- a/CSharp/Examples/RailenginePoweredStatusPage/appsettings.Development.sample.json +++ b/CSharp/Examples/RailenginePoweredStatusPage/appsettings.Development.sample.json @@ -12,5 +12,11 @@ "Anthropic": { "ApiKey": "[your-anthropic-api-key-here]" }, + "DailyInsight": { + "// AgentUrl": "Optional — when set, posts to {AgentUrl}/insight (e.g. the Python/daily-insight service) instead of calling Anthropic + MCP inline. Leave empty/unset to use the inline path.", + "AgentUrl": "", + "// AgentBearerToken": "Optional — when set, sent as Authorization: Bearer on every POST to {AgentUrl}/insight. Set this when the agent endpoint requires bearer authentication. Leave empty for an unauthenticated endpoint (e.g. local development).", + "AgentBearerToken": "" + }, "AllowedIPs": [ "127.0.0.1", "::1" ] } diff --git a/CSharp/Examples/RailenginePoweredStatusPage/appsettings.json b/CSharp/Examples/RailenginePoweredStatusPage/appsettings.json index 9254a89..646e75b 100644 --- a/CSharp/Examples/RailenginePoweredStatusPage/appsettings.json +++ b/CSharp/Examples/RailenginePoweredStatusPage/appsettings.json @@ -17,5 +17,9 @@ "Beta": "mcp-client-2025-04-04", "Model": "claude-haiku-4-5-20251001" }, + "DailyInsight": { + "AgentUrl": "", + "AgentBearerToken": "" + }, "AllowedIPs": [ "127.0.0.1", "::1" ] } diff --git a/Python/daily-insight/.env.example b/Python/daily-insight/.env.example new file mode 100644 index 0000000..3ec1c9c --- /dev/null +++ b/Python/daily-insight/.env.example @@ -0,0 +1,20 @@ +# Retrieval (PAT + Engine ID for the same engine the status page reads from) +ENGINE_ID="[Your Engine ID]" +ENGINE_PAT="[Your Engine PAT]" + +# Optional: override API host (must match the stack ENGINE_PAT targets) +# RAILTOWN_API_URL="https://cndr.railtown.ai/api" + +# LLM provider key. The agent uses the provider-neutral name LLM_API_KEY and +# copies it into ANTHROPIC_API_KEY on startup so the Anthropic SDK picks it up. +# (You can also set ANTHROPIC_API_KEY directly if you prefer the SDK-native +# name — the bridge runs in both directions.) +LLM_API_KEY="[Your Anthropic API key]" +# ANTHROPIC_API_KEY="[same key — SDK-native name]" + +# Optional: override the default Claude model (defaults to claude-haiku-4-5-20251001 in code) +# LLM_MODEL="claude-haiku-4-5-20251001" + +# Optional: Railtown observability. When set, the agent's logs + any unhandled +# exception tracebacks ship to Railtown via the railtownai logging handler. +# RAILTOWN_API_KEY="[Your Railtown API key]" diff --git a/Python/daily-insight/Dockerfile b/Python/daily-insight/Dockerfile new file mode 100644 index 0000000..1eae936 --- /dev/null +++ b/Python/daily-insight/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.10-slim + +WORKDIR /app + +# Install the package + deps. Copying pyproject.toml + src separately lets +# Docker cache the dependency install across source-only changes. +COPY pyproject.toml ./ +COPY src/ ./src/ +RUN pip install --no-cache-dir . + +EXPOSE 8000 + +CMD ["uvicorn", "daily_insight.controllers.api:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/Python/daily-insight/README.md b/Python/daily-insight/README.md new file mode 100644 index 0000000..6884e68 --- /dev/null +++ b/Python/daily-insight/README.md @@ -0,0 +1,147 @@ +# Daily Insight (Railtracks) + +A small Python service that mirrors the **Daily Insight** feature from [`CSharp/Examples/RailenginePoweredStatusPage`](../../CSharp/Examples/RailenginePoweredStatusPage/): given a Railengine of metric records, ask an LLM to produce a one-line-per-metric plain-text summary of recent values. + +Where the C# version drives the LLM through an Anthropic MCP server attached as a tool source, this version drives it through [Railtracks](https://github.com/RailtownAI/railtracks) with a single `@rt.function_node` tool that calls the [Railengine Python SDK](https://pypi.org/project/rail-engine/) directly. + +## Before you start + +- A [Railengine](https://railengine.ai/) engine populated with `MetricRecord`-shaped documents (`metric`, `timestamp`, `value` — see [`MetricRecord`](src/models/metric.py)). The C# status page example produces records in exactly this shape. +- An Anthropic API key with access to `claude-haiku-4-5-20251001` (or another model you set via `LLM_MODEL`). + +## Quick start + +```bash +cd Python/daily-insight +cp .env.example .env # fill ENGINE_ID, ENGINE_PAT, LLM_API_KEY +uv sync +uv run uvicorn daily_insight.controllers.api:app --reload --port 8000 +``` + +Then: + +```bash +# Liveness +curl http://127.0.0.1:8000/health + +# Generate a fresh insight +curl -X POST http://127.0.0.1:8000/insight + +# Generate fresh insight(s) and evaluate them +curl -X POST http://127.0.0.1:8000/evals/run -H 'Content-Type: application/json' -d '{"sample_size": 1}' +``` + +`POST /insight` runs the Railtracks agent against your engine and returns: + +```json +{ + "text": "latency-p95: Latest reading 87.3 ms, trending down over the last 50 samples.\nerror-rate: Holding flat near 2.4 errors/min.\n...", + "generated_at": "2026-06-12T18:42:11.123456+00:00", + "metric_count": 4, + "error": null +} +``` + +Expect each call to take a few seconds — the agent makes one Anthropic call plus one Railengine GET. The exact latency depends on the model. + +## Evaluation + +`POST /evals/run` runs Railtracks evaluators against agent sessions and returns the scores. Two modes: + +| Mode | Trigger | Cost | What it evaluates | +|---|---|---|---| +| **Fresh** (default) | body omits `agent_run_ids` | ≈ `sample_size · 3` LLM calls | Generates `sample_size` brand-new `/insight` runs (default 1, capped at 5) and scores them. Self-contained smoke test. | +| **Historical** | body sets `agent_run_ids` | ≈ `N · 2` LLM calls (judge only) | Fetches the named past runs from Conductr via [`railtownai.get_agent_runs`](https://pypi.org/project/railtownai/) and scores them as one batch. Replays real production interactions without re-spending generation cost. | + +```bash +# Fresh (default) +curl -X POST .../evals/run -H 'Content-Type: application/json' -d '{"sample_size": 1}' + +# Historical (one or more runs) +curl -X POST .../evals/run -H 'Content-Type: application/json' \ + -d '{"agent_run_ids": ["0466964a-1234-5678-9abc-def012345678"]}' +``` + +When `RAILTOWN_API_KEY` is set, each `EvaluationResult` also uploads to Conductr via `railtownai.upload_agent_evaluation`. + +Three evaluators run per call (see [`agents/evaluations.py`](src/agents/evaluations.py)): + +- **`ToolUseEvaluator`** (free, local) — checks the agent's tool call pattern. The system prompt says "use AT MOST 1 tool call to `get_recent_metrics`"; this catches regressions where the model skips it or over-calls. +- **`LLMInferenceEvaluator`** (free, local) — checks the LLM call pattern (latency, tokens, errors). +- **`JudgeEvaluator`** with two custom metrics: + - **`FormatCompliance`** (Compliant / MinorDeviation / MajorDeviation) — does the response follow the strict one-line-per-metric format that the C# card expects to render unchanged? + - **`FactualGrounding`** (FullyGrounded / PartiallyGrounded / Hallucinated) — does every value and trend in the response trace back to the tool output, or is the model inventing things? + +Override the judge model via `EVAL_JUDGE_MODEL`; defaults to `LLM_MODEL` or `claude-haiku-4-5-20251001`. + +**Historical-mode prerequisites.** The agent fetches sessions from Conductr's platform API, so it needs: + +- `CONDUCTR_PROJECT_ID` — already present in deployed environments (the deploy tooling seeds it). +- `CONDUCTR_PROJECT_PAT` — a project-level access token generated in the Conductr UI under *project → Secret Tokens*. Not yet auto-seeded by the deploy tooling; for a deployed agent, set it once with `az keyvault secret set --vault-name --name CONDUCTR-PROJECT-PAT --value ''` and roll a config-only revision to wire it onto the container. + +## Environment variables + +| Variable | Required | Purpose | +|---|---|---| +| `ENGINE_ID` | Yes | Railengine engine GUID — same one the C# status page reads from | +| `ENGINE_PAT` | Yes | Railengine PAT used for retrieval | +| `LLM_API_KEY` | Yes | Anthropic API key. Provider-neutral name so the same setting works across agents; on startup the value is copied into `ANTHROPIC_API_KEY` for the Anthropic SDK to pick up. `ANTHROPIC_API_KEY` is also accepted directly if you prefer the SDK-native name. | +| `LLM_MODEL` | No | Override the default Claude model (`claude-haiku-4-5-20251001`) | +| `RAILTOWN_API_KEY` | No | Enables Railtown observability. When set, the agent ships its logs (and any unhandled-exception tracebacks from the global handler) to Railtown via the [`railtownai`](https://pypi.org/project/railtownai/) logging handler. Unset → no observability, agent runs normally. | +| `RAILTOWN_API_URL` | No | Override the Railengine API host (defaults to production) | + +Variables are read from `.env` next to `pyproject.toml`, then from the process environment. + +## Project layout + +```text +controllers (FastAPI) → services → agents (Railtracks) → repositories → rail-engine + ↓ + models +``` + +| Path | Role | +|---|---| +| [`src/models/`](src/models/) | Pydantic types — `MetricRecord`, `DailyInsight` | +| [`src/repositories/`](src/repositories/) | [`MetricRepository`](src/repositories/metric_repository.py) — wraps `Railengine.list_storage_documents` | +| [`src/agents/`](src/agents/) | [`tools.py`](src/agents/tools.py) defines the `get_recent_metrics` function node; [`insight_agent.py`](src/agents/insight_agent.py) wires it to `rt.llm.AnthropicLLM` | +| [`src/services/`](src/services/) | [`InsightService`](src/services/insight_service.py) — runs the `rt.Flow` and shapes the response | +| [`src/controllers/`](src/controllers/) | [`api.py`](src/controllers/api.py) — FastAPI app with `/health` and `/insight` | +| [`src/config/`](src/config/) | `.env` loading and required-env validation | + +## How this differs from the C# version + +The C# [`DailyInsightService`](../../CSharp/Examples/RailenginePoweredStatusPage/Services/DailyInsightService.cs) is a `BackgroundService` that wakes once every 24 hours and posts to `/v1/messages` with the Railengine MCP server (`mcp_servers`) attached, then streams the response. The same prompt and output format are used here. + +This Python version: + +- Replaces the MCP attachment with a Railtracks `@rt.function_node` tool (`get_recent_metrics`) that calls the Railengine Python SDK directly. The LLM still chooses when to call it, but the schema and execution are local. +- Replaces the 24h `BackgroundService` with an on-demand HTTP endpoint. A scheduler (cron, GitHub Actions, Azure Logic Apps, etc.) can POST `/insight` daily if you want the same cadence. +- Keeps the strict plain-text output rules so the existing C# `Daily Insight` card can render the result unchanged. + +## Containerization + +A [`Dockerfile`](Dockerfile) is included so the service can be containerized for deployment. The image installs the package via `pyproject.toml` and runs `uvicorn daily_insight.controllers.api:app` on port 8000. Build and run locally with: + +```bash +docker build -t daily-insight . +docker run --rm -p 8000:8000 --env-file .env daily-insight +``` + +If you put the agent behind a reverse proxy that requires bearer auth, the C# caller in [`RailenginePoweredStatusPage`](../../CSharp/Examples/RailenginePoweredStatusPage/) can attach the token via the `DailyInsight:AgentBearerToken` setting — `DailyInsightService.GenerateFromAgentAsync` adds it as `Authorization: Bearer ` when non-empty. + +## Debug and visualize the agent (optional) + +After at least one insight run: + +```bash +cd Python/daily-insight +railtracks update +railtracks viz +``` + +(`railtracks[visual]` is in `pyproject.toml`; run `uv sync` if you have not already.) Opens the local visualization app for inspecting tool calls and prompts. + +## Local only + +Don't expose this service on the public internet without authentication — `/insight` triggers a paid LLM call and a Railengine read on every invocation. diff --git a/Python/daily-insight/pyproject.toml b/Python/daily-insight/pyproject.toml new file mode 100644 index 0000000..02574f6 --- /dev/null +++ b/Python/daily-insight/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "daily-insight" +version = "0.1.0" +description = "Daily status-page insight demo: Railengine retrieval + Railtracks agent + FastAPI" +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "rail-engine>=0.2.1", + "railtracks[visual]>=1.4.1", + "railtownai>=2.0.14", + "pydantic>=2.0", + "python-dotenv>=1.0.0", + "fastapi>=0.110.0", + "uvicorn[standard]>=0.27.0", +] + +[tool.setuptools] +package-dir = { "daily_insight" = "src" } +packages = [ + "daily_insight", + "daily_insight.agents", + "daily_insight.config", + "daily_insight.controllers", + "daily_insight.models", + "daily_insight.repositories", + "daily_insight.services", +] diff --git a/Python/daily-insight/src/__init__.py b/Python/daily-insight/src/__init__.py new file mode 100644 index 0000000..81c697c --- /dev/null +++ b/Python/daily-insight/src/__init__.py @@ -0,0 +1,3 @@ +"""Daily status-page insight demo: Railengine + Railtracks + FastAPI.""" + +__version__ = "0.1.0" diff --git a/Python/daily-insight/src/agents/__init__.py b/Python/daily-insight/src/agents/__init__.py new file mode 100644 index 0000000..5919831 --- /dev/null +++ b/Python/daily-insight/src/agents/__init__.py @@ -0,0 +1,3 @@ +from daily_insight.agents.insight_agent import build_insight_agent + +__all__ = ["build_insight_agent"] diff --git a/Python/daily-insight/src/agents/evaluations.py b/Python/daily-insight/src/agents/evaluations.py new file mode 100644 index 0000000..5760cb6 --- /dev/null +++ b/Python/daily-insight/src/agents/evaluations.py @@ -0,0 +1,69 @@ +"""Railtracks evaluation setup for the daily insight agent. + +Metrics target the agent's specific failure modes (strict per-line output +format, factual grounding against the tool output) rather than the generic +helpfulness/efficiency that the railtracks quickstart shows. +""" + +from __future__ import annotations + +import os + +import railtracks as rt +from railtracks import evaluations as evals + + +# Same default as the agent — when LLM_MODEL/EVAL_JUDGE_MODEL is set, override. +_DEFAULT_JUDGE_MODEL = "claude-haiku-4-5-20251001" + + +FORMAT_COMPLIANCE = evals.metrics.Categorical( + name="FormatCompliance", + description=( + "Does the agent's final response follow the strict format: one line per " + "metric, no markdown, no preamble, format `{metric}: {one-sentence " + "insight with latest value and trend}`? " + "Compliant = every line conforms. " + "MinorDeviation = mostly conforms but one stray formatting element " + "(extra blank line, single missing colon, etc.). " + "MajorDeviation = bullets, headers, prose paragraphs, multi-line " + "insights per metric." + ), + categories=["Compliant", "MinorDeviation", "MajorDeviation"], +) + + +FACTUAL_GROUNDING = evals.metrics.Categorical( + name="FactualGrounding", + description=( + "Is every numeric value and trend statement in the agent's final " + "response actually present in the get_recent_metrics tool output? " + "Inspect the tool calls and tool outputs in the data. " + "FullyGrounded = every claim traceable to the tool data. " + "PartiallyGrounded = some values right, some invented or wrong. " + "Hallucinated = at least one fabricated value or trend." + ), + categories=["FullyGrounded", "PartiallyGrounded", "Hallucinated"], +) + + +def build_evaluators() -> list: + """Return the evaluator list to run against each daily-insight session. + + The judge model defaults to the agent's own model; override via + EVAL_JUDGE_MODEL (otherwise LLM_MODEL, otherwise the constant default). + """ + judge_model = ( + os.environ.get("EVAL_JUDGE_MODEL", "").strip() + or os.environ.get("LLM_MODEL", "").strip() + or _DEFAULT_JUDGE_MODEL + ) + return [ + evals.ToolUseEvaluator(), + evals.LLMInferenceEvaluator(), + evals.JudgeEvaluator( + llm=rt.llm.AnthropicLLM(judge_model), + metrics=[FORMAT_COMPLIANCE, FACTUAL_GROUNDING], + reasoning=True, + ), + ] diff --git a/Python/daily-insight/src/agents/insight_agent.py b/Python/daily-insight/src/agents/insight_agent.py new file mode 100644 index 0000000..c40cfc1 --- /dev/null +++ b/Python/daily-insight/src/agents/insight_agent.py @@ -0,0 +1,44 @@ +"""Railtracks insight agent definition.""" + +from __future__ import annotations + +import os + +import railtracks as rt + +from daily_insight.agents.tools import get_recent_metrics + + +DEFAULT_MODEL = "claude-haiku-4-5-20251001" + +SYSTEM_PROMPT = """You are an automated reviewer for a status page dashboard. + +Use AT MOST 1 tool call. Call `get_recent_metrics` with limit=50 to fetch +the most recent metric records, then summarize. Do not make additional +exploratory calls. + +Output format (strict): +- One single line per metric, nothing else. +- Format: "{Metric name}: {one-sentence insight including the latest value and any notable trend}" +- Plain text only. No markdown, no headers, no bullets, no emoji, no bold. +- No preamble, no commentary about your process, no closing remarks. +- If a metric is flat, say so concisely.""" + + +def build_insight_agent(): + """Create the Railtracks insight agent backed by AnthropicLLM.""" + api_key = os.environ.get("ANTHROPIC_API_KEY", "").strip() + if not api_key: + raise RuntimeError( + "ANTHROPIC_API_KEY is not set. Add it to .env or export it before running." + ) + + model = os.environ.get("LLM_MODEL", "").strip() or DEFAULT_MODEL + llm = rt.llm.AnthropicLLM(model) + + return rt.agent_node( + "Daily Insight Agent", + tool_nodes=(get_recent_metrics,), + llm=llm, + system_message=SYSTEM_PROMPT, + ) diff --git a/Python/daily-insight/src/agents/tools.py b/Python/daily-insight/src/agents/tools.py new file mode 100644 index 0000000..805d1a7 --- /dev/null +++ b/Python/daily-insight/src/agents/tools.py @@ -0,0 +1,22 @@ +"""Railtracks tool nodes backed by MetricRepository.""" + +from __future__ import annotations + +import json + +import railtracks as rt + +from daily_insight.repositories import MetricRepository + + +@rt.function_node +async def get_recent_metrics(limit: int = 50) -> str: + """ + Fetch the most recent metric records from Railengine. + + Args: + limit: Maximum number of records to return (capped at 100 by the SDK). + """ + repo = MetricRepository() + records = await repo.list_recent(limit=limit) + return json.dumps([r.model_dump() for r in records], ensure_ascii=False) diff --git a/Python/daily-insight/src/config/__init__.py b/Python/daily-insight/src/config/__init__.py new file mode 100644 index 0000000..03eea61 --- /dev/null +++ b/Python/daily-insight/src/config/__init__.py @@ -0,0 +1,13 @@ +from daily_insight.config.env import ( + MissingEnvVarsError, + configure_runtime_env, + ensure_dotenv_loaded, + validate_required_env, +) + +__all__ = [ + "MissingEnvVarsError", + "configure_runtime_env", + "ensure_dotenv_loaded", + "validate_required_env", +] diff --git a/Python/daily-insight/src/config/env.py b/Python/daily-insight/src/config/env.py new file mode 100644 index 0000000..0b7a158 --- /dev/null +++ b/Python/daily-insight/src/config/env.py @@ -0,0 +1,54 @@ +"""Load `.env`, bridge env var aliases, and validate required environment variables.""" + +from __future__ import annotations + +import os +from pathlib import Path + +from dotenv import load_dotenv + + +REQUIRED_ENV_VARS = ("ENGINE_PAT", "ENGINE_ID", "ANTHROPIC_API_KEY") + + +class MissingEnvVarsError(RuntimeError): + def __init__(self, missing: list[str]) -> None: + self.missing = missing + names = ", ".join(missing) + super().__init__( + f"Missing required environment variables: {names}. " + f"Set them in .env (see .env.example). " + f"ANTHROPIC_API_KEY also accepts the provider-neutral alias LLM_API_KEY." + ) + + +def ensure_dotenv_loaded() -> None: + """Load env from `Python/daily-insight/.env`, then cwd `.env` (override).""" + project_root = Path(__file__).resolve().parents[2] + load_dotenv(project_root / ".env") + load_dotenv() + + +def configure_runtime_env() -> None: + """Bridge LLM_API_KEY ↔ ANTHROPIC_API_KEY so deploy tooling that stores + keys under a provider-neutral name still satisfies the AnthropicLLM + SDK's lookup. Whichever is set on startup is mirrored to the other.""" + if not os.environ.get("LLM_API_KEY", "").strip(): + if anthropic := os.environ.get("ANTHROPIC_API_KEY", "").strip(): + os.environ["LLM_API_KEY"] = anthropic + + if llm_key := os.environ.get("LLM_API_KEY", "").strip(): + os.environ["ANTHROPIC_API_KEY"] = llm_key + + +def validate_required_env() -> None: + """Raise MissingEnvVarsError if any required variable is unset or blank. + + Run ``configure_runtime_env`` first so the LLM_API_KEY alias has a chance + to populate ANTHROPIC_API_KEY before this check. + """ + missing = [ + name for name in REQUIRED_ENV_VARS if not os.environ.get(name, "").strip() + ] + if missing: + raise MissingEnvVarsError(missing) diff --git a/Python/daily-insight/src/controllers/__init__.py b/Python/daily-insight/src/controllers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Python/daily-insight/src/controllers/api.py b/Python/daily-insight/src/controllers/api.py new file mode 100644 index 0000000..da94468 --- /dev/null +++ b/Python/daily-insight/src/controllers/api.py @@ -0,0 +1,167 @@ +"""FastAPI service that runs the daily insight agent on demand. + +Run:: + + uv run uvicorn daily_insight.controllers.api:app --reload --port 8000 + +Endpoints: +- GET /health — liveness probe +- POST /insight — run the Railtracks agent against Railengine and return a DailyInsight +- POST /evals/run — run the agent N times (or fetch a historical session), evaluate, return scores +""" + +from __future__ import annotations + +import logging +import os +from typing import Optional +from uuid import UUID + +import railtownai +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse +from contextlib import asynccontextmanager +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from daily_insight.config import ( + MissingEnvVarsError, + configure_runtime_env, + ensure_dotenv_loaded, + validate_required_env, +) +from daily_insight.models import DailyInsight, EvaluationRun +from daily_insight.services import EvaluationService, InsightService + + +class EvaluateRequest(BaseModel): + # extra="forbid" rejects typo'd keys at 422 instead of silently dropping + # them — the old singular agent_run_id would otherwise slip through and + # quietly trigger a Fresh-mode generation run. + model_config = ConfigDict(extra="forbid") + + sample_size: int = Field( + default=1, + ge=1, + le=5, + description=( + "Fresh-mode only. Number of fresh insight runs to generate and " + "evaluate (capped at 5). Mutually exclusive with agent_run_ids." + ), + ) + agent_run_ids: Optional[list[UUID]] = Field( + default=None, + min_length=1, + max_length=10, + description=( + "Historical-mode only. When set, fetches the named historical " + "agent runs from Conductr (via railtownai.get_agent_runs) and " + "evaluates those sessions as a single batch instead of generating " + "fresh ones. Mutually exclusive with sample_size. Requires " + "CONDUCTR_PROJECT_PAT and CONDUCTR_PROJECT_ID on the agent." + ), + ) + + @model_validator(mode="after") + def _enforce_mode_xor(self) -> "EvaluateRequest": + """Reject bodies that try to drive both modes at once. + + Uses model_fields_set so the default sample_size=1 doesn't count as + "set" — a caller passing only agent_run_ids still validates cleanly. + """ + if ( + "sample_size" in self.model_fields_set + and "agent_run_ids" in self.model_fields_set + ): + raise ValueError( + "sample_size and agent_run_ids are mutually exclusive — " + "omit sample_size when targeting historical runs." + ) + return self + + +logger = logging.getLogger(__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + ensure_dotenv_loaded() + configure_runtime_env() + + # Wire Railtown observability when configured. railtownai.init attaches a + # logging handler to the root logger, so any logger.exception/error call + # downstream ships to Railtown automatically. + if railtown_key := os.environ.get("RAILTOWN_API_KEY", "").strip(): + railtownai.init(railtown_key) + logger.info("Railtown observability enabled") + else: + logger.info("RAILTOWN_API_KEY not set — Railtown observability disabled") + + try: + validate_required_env() + except MissingEnvVarsError as exc: + logger.error(str(exc)) + raise + yield + + +app = FastAPI( + title="Daily Insight", + description="Railtracks-powered status-page insight over Railengine metrics.", + version="0.1.0", + lifespan=lifespan, +) + + +@app.exception_handler(Exception) +async def global_exception_handler(request: Request, exc: Exception) -> JSONResponse: + """Catch-all for unhandled exceptions. + + Logs the full traceback via Python's logging (Railtown's handler ships + it onward when initialized) and returns a JSON 500 body with the + exception type and message — far more useful than FastAPI's default + plain-text "Internal Server Error" for diagnosing live failures. + """ + logger.exception( + "Unhandled exception in %s %s", + request.method, + request.url.path, + ) + return JSONResponse( + status_code=500, + content={ + "type": type(exc).__name__, + "detail": str(exc), + }, + ) + + +@app.get("/health") +async def health() -> dict[str, str]: + return {"status": "ok"} + + +@app.post("/insight", response_model=DailyInsight) +async def generate_insight() -> DailyInsight: + return await InsightService().run() + + +@app.post("/evals/run", response_model=EvaluationRun) +async def run_evaluation(req: EvaluateRequest = EvaluateRequest()) -> EvaluationRun: + """Evaluate either fresh insight runs or one or more named historical runs. + + Fresh mode (``agent_run_ids`` absent): generates ``sample_size`` insight + runs and scores them. Cost ≈ ``sample_size · 3`` LLM calls (1 to generate, + 2 for the FormatCompliance + FactualGrounding judge metrics). + + Historical mode (``agent_run_ids`` set): fetches the named sessions from + Conductr and scores them as a single batch. Cost ≈ ``N · 2`` LLM calls + (judge only). Requires ``CONDUCTR_PROJECT_PAT`` and ``CONDUCTR_PROJECT_ID`` + on the agent. + + ``ToolUseEvaluator`` and ``LLMInferenceEvaluator`` are free local checks in + either mode. + """ + return await EvaluationService().run( + sample_size=req.sample_size, + agent_run_ids=req.agent_run_ids, + ) diff --git a/Python/daily-insight/src/models/__init__.py b/Python/daily-insight/src/models/__init__.py new file mode 100644 index 0000000..8113d1b --- /dev/null +++ b/Python/daily-insight/src/models/__init__.py @@ -0,0 +1,5 @@ +from daily_insight.models.evaluation import EvaluationRun +from daily_insight.models.insight import DailyInsight +from daily_insight.models.metric import MetricRecord + +__all__ = ["DailyInsight", "EvaluationRun", "MetricRecord"] diff --git a/Python/daily-insight/src/models/evaluation.py b/Python/daily-insight/src/models/evaluation.py new file mode 100644 index 0000000..8b45a2d --- /dev/null +++ b/Python/daily-insight/src/models/evaluation.py @@ -0,0 +1,20 @@ +"""Evaluation run result returned by POST /evals/run.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from pydantic import BaseModel + +from daily_insight.models.insight import DailyInsight + + +class EvaluationRun(BaseModel): + """One end-to-end evaluation: N fresh insight runs plus their scores.""" + + sample_size: int + started_at: datetime + completed_at: datetime + insights: list[DailyInsight] + evaluation_results: list[dict[str, Any]] diff --git a/Python/daily-insight/src/models/insight.py b/Python/daily-insight/src/models/insight.py new file mode 100644 index 0000000..76ad479 --- /dev/null +++ b/Python/daily-insight/src/models/insight.py @@ -0,0 +1,17 @@ +"""Daily insight result shape returned to API callers.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel + + +class DailyInsight(BaseModel): + """One-line-per-metric plain-text summary of recent engine data.""" + + text: str + generated_at: datetime + metric_count: int + error: Optional[str] = None diff --git a/Python/daily-insight/src/models/metric.py b/Python/daily-insight/src/models/metric.py new file mode 100644 index 0000000..ba73368 --- /dev/null +++ b/Python/daily-insight/src/models/metric.py @@ -0,0 +1,17 @@ +"""Metric record schema — matches the C# RailenginePoweredStatusPage example.""" + +from __future__ import annotations + +from pydantic import BaseModel + + +class MetricRecord(BaseModel): + """One metric reading as stored in Railengine. + + Fields mirror the camelCase JSON keys produced by the status-page example + (`metric`, `timestamp`, `value`), so no field aliases are needed. + """ + + metric: str + timestamp: int + value: float diff --git a/Python/daily-insight/src/repositories/__init__.py b/Python/daily-insight/src/repositories/__init__.py new file mode 100644 index 0000000..040b481 --- /dev/null +++ b/Python/daily-insight/src/repositories/__init__.py @@ -0,0 +1,3 @@ +from daily_insight.repositories.metric_repository import MetricRepository + +__all__ = ["MetricRepository"] diff --git a/Python/daily-insight/src/repositories/metric_repository.py b/Python/daily-insight/src/repositories/metric_repository.py new file mode 100644 index 0000000..3278e44 --- /dev/null +++ b/Python/daily-insight/src/repositories/metric_repository.py @@ -0,0 +1,44 @@ +"""Railengine retrieval for MetricRecord documents.""" + +from __future__ import annotations + +from typing import Any + +from railtown.engine import Railengine + +from daily_insight.models import MetricRecord + + +def _as_metric(item: Any) -> MetricRecord | None: + if isinstance(item, MetricRecord): + return item + if isinstance(item, dict): + try: + return MetricRecord.model_validate(item) + except Exception: + return None + return None + + +class MetricRepository: + """Single-page reads against the Railengine storage API.""" + + async def list_recent(self, limit: int = 50) -> list[MetricRecord]: + """Return up to ``limit`` metric records from the most recent page. + + ``page_size`` is capped at 100 by the SDK; callers wanting more should + page through ``list_storage_documents`` directly. + """ + page_size = max(1, min(int(limit), 100)) + async with Railengine(model=MetricRecord) as client: + page = await client.list_storage_documents( + page_number=1, + page_size=page_size, + ) + + out: list[MetricRecord] = [] + for item in page.items: + metric = _as_metric(item) + if metric is not None: + out.append(metric) + return out diff --git a/Python/daily-insight/src/services/__init__.py b/Python/daily-insight/src/services/__init__.py new file mode 100644 index 0000000..f4d4b38 --- /dev/null +++ b/Python/daily-insight/src/services/__init__.py @@ -0,0 +1,4 @@ +from daily_insight.services.evaluation_service import EvaluationService +from daily_insight.services.insight_service import InsightService + +__all__ = ["EvaluationService", "InsightService"] diff --git a/Python/daily-insight/src/services/evaluation_service.py b/Python/daily-insight/src/services/evaluation_service.py new file mode 100644 index 0000000..1258472 --- /dev/null +++ b/Python/daily-insight/src/services/evaluation_service.py @@ -0,0 +1,184 @@ +"""Generate fresh insight runs (or fetch a historical run) and evaluate them.""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from uuid import UUID + +import railtownai +from railtracks import evaluations as evals + +from daily_insight.agents.evaluations import build_evaluators +from daily_insight.models import DailyInsight, EvaluationRun +from daily_insight.services.insight_service import InsightService + +logger = logging.getLogger(__name__) + +_AGENT_NAME = "Daily Insight Agent" +_MAX_SAMPLE_SIZE = 5 + + +async def _upload_evaluations(results: list[Any]) -> None: + """Upload completed EvaluationResults to Conductr after evals.evaluate finishes. + + Skips silently when EVALUATIONS_API_TOKEN is unset (local dev without the + upload token). Logs success, the SDK's silent-False generic-error path, or + any raised exception (EvaluationsNotInitializedError / + EvaluationsValidationError). + + Why this is async + offloaded to a worker thread instead of using + evals.evaluate's payload_callback hook: + + railtownai.upload_agent_evaluation is a *sync* function that drives its + async implementation via asyncio.run(). asyncio.run() raises RuntimeError + when invoked from within an already-running event loop — exactly what + happens when FastAPI calls into this service. The SDK catches that as a + generic Exception and returns False, leaving an unawaited coroutine + warning in stderr. asyncio.to_thread runs the call in a worker thread + with clean thread-local state where asyncio.run() works as designed. + """ + if not os.environ.get("EVALUATIONS_API_TOKEN", "").strip(): + return + if not results: + return + + payloads = [r.model_dump(mode="json") for r in results] + try: + success = await asyncio.to_thread(railtownai.upload_agent_evaluation, payloads) + if success: + logger.info("Uploaded %s evaluation result(s) to Conductr", len(payloads)) + else: + logger.error( + "railtownai.upload_agent_evaluation returned False — upload " + "failed without raising. Likely causes: token rejected by " + "Conductr, ingestion endpoint unreachable, or rail-engine-ingest " + "HTTP error. The SDK suppresses details; bump the " + "`railtown.engine.ingest` logger to DEBUG to see the HTTP " + "exchange." + ) + except Exception: + logger.exception("Evaluation upload raised") + + +def _fetch_historical_sessions(agent_run_ids: list[UUID]) -> list[dict[str, Any]]: + """Fetch session payloads from Conductr via railtownai.get_agent_runs. + + Returns the nested session shapes (session_id, runs: [...]) that + extract_agent_data_points consumes. Raises whatever the SDK raises — + AgentRunsNotInitializedError when CONDUCTR_PROJECT_PAT/CONDUCTR_PROJECT_ID + aren't set, AgentRunFetchError on HTTP/parse failures — and the global + exception handler shapes those into a 500 JSON body. + """ + payloads = railtownai.get_agent_runs([str(rid) for rid in agent_run_ids]) + if len(payloads) != len(agent_run_ids): + raise RuntimeError( + f"Conductr returned {len(payloads)} payload(s) for " + f"{len(agent_run_ids)} requested agent_run_ids " + f"({[str(r) for r in agent_run_ids]})" + ) + return payloads + + +class EvaluationService: + """Generate or fetch insight session(s) and evaluate them. + + Two modes: + - **Fresh** (default) — generate ``sample_size`` insight runs via + ``InsightService.run_and_capture_session`` and evaluate them. + - **Historical** — when ``agent_run_ids`` is provided, fetch those runs + from Conductr via ``railtownai.get_agent_runs`` and evaluate them as a + single batch. ``sample_size`` is ignored in this mode. + """ + + async def run( + self, + sample_size: int = 1, + agent_run_ids: list[UUID] | None = None, + ) -> EvaluationRun: + started_at = datetime.now(timezone.utc) + + if agent_run_ids: + logger.info( + "Evaluating %s historical session(s): %s", + len(agent_run_ids), + [str(r) for r in agent_run_ids], + ) + session_payloads: list[dict[str, Any]] = _fetch_historical_sessions( + agent_run_ids + ) + insights: list[DailyInsight] = [] + effective_sample_size = len(agent_run_ids) + else: + effective_sample_size = max(1, min(int(sample_size), _MAX_SAMPLE_SIZE)) + insight_service = InsightService() + insights = [] + session_payloads = [] + for i in range(effective_sample_size): + logger.info( + "Generating insight run %s/%s", i + 1, effective_sample_size + ) + insight, payload = await insight_service.run_and_capture_session() + insights.append(insight) + session_payloads.append(payload) + + # extract_agent_data_points reads JSON session files from disk. Stage the + # captured payloads in a request-scoped tempdir so each /evals/run call + # sees exactly its own sessions — no leakage between concurrent requests. + with tempfile.TemporaryDirectory(prefix="daily-insight-eval-") as tmpdir: + tmp = Path(tmpdir) + session_files: list[str] = [] + for idx, payload in enumerate(session_payloads): + fp = tmp / f"session_{idx}.json" + fp.write_text(json.dumps(payload), encoding="utf-8") + session_files.append(str(fp)) + + data = evals.extract_agent_data_points(session_files) + logger.info( + "Extracted %s agent data points from %s session(s)", + len(data), + len(session_files), + ) + + timestamp = started_at.strftime("%Y%m%dT%H%M%SZ") + if agent_run_ids: + # Keep the name human-readable when there's only one run; for a + # batch, just embed the count so the name stays bounded. + if len(agent_run_ids) == 1: + evaluation_name = f"daily-insight-{agent_run_ids[0]}-{timestamp}" + else: + evaluation_name = ( + f"daily-insight-batch{len(agent_run_ids)}-{timestamp}" + ) + else: + evaluation_name = f"daily-insight-{timestamp}" + + # agent_selection=False + agents=[...] keeps evaluate() headless. + # No payload_callback: the SDK's sync upload helper calls + # asyncio.run() under the hood which fails from inside FastAPI's + # running event loop. Upload via asyncio.to_thread after evaluate + # returns instead — see _upload_evaluations. + evaluation_results = evals.evaluate( + data=data, + evaluators=build_evaluators(), + agents=[_AGENT_NAME], + agent_selection=False, + name=evaluation_name, + ) + + await _upload_evaluations(evaluation_results) + + completed_at = datetime.now(timezone.utc) + return EvaluationRun( + sample_size=effective_sample_size, + started_at=started_at, + completed_at=completed_at, + insights=insights, + evaluation_results=[r.model_dump(mode="json") for r in evaluation_results], + ) diff --git a/Python/daily-insight/src/services/insight_service.py b/Python/daily-insight/src/services/insight_service.py new file mode 100644 index 0000000..1fb1607 --- /dev/null +++ b/Python/daily-insight/src/services/insight_service.py @@ -0,0 +1,97 @@ +"""Orchestrate the insight Railtracks session and shape the result.""" + +from __future__ import annotations + +import logging +from datetime import datetime, timezone +from typing import Any + +import railtracks as rt +import railtownai + +from daily_insight.agents import build_insight_agent +from daily_insight.models import DailyInsight +from daily_insight.repositories import MetricRepository + + +PROMPT = ( + "Summarize the current metric data. Call get_recent_metrics(limit=50), " + "then produce the strict per-metric output described in your system prompt." +) + +_SESSION_NAME = "daily-insight-session" +_FLOW_NAME = "daily-insight" +_FLOW_ID = "daily-insight-agent" + + +logger = logging.getLogger(__name__) + + +def _result_text(result: object) -> str: + if hasattr(result, "content"): + return str(result.content) + return str(result) + + +def _upload_session(session: "rt.Session") -> None: + """Upload the Railtracks session payload to Conductr when configured. + + Skips silently when ``railtownai.init`` hasn't run (no RAILTOWN_API_KEY), + so a local invocation without observability still succeeds. + """ + if railtownai.get_railtown_handler() is None: + return + try: + success = railtownai.upload_agent_run(session.payload()) + logger.info("Agent run uploaded to Conductr: %s", success) + except RuntimeError as exc: + logger.warning("Skipping Railtown upload: %s", exc) + except Exception: + logger.exception("Unexpected error uploading agent run to Railtown") + + +async def _count_metrics() -> int: + """Cheap second read for the API response — does not block the agent.""" + try: + records = await MetricRepository().list_recent(limit=50) + return len({r.metric for r in records}) + except Exception: + return 0 + + +class InsightService: + """Run the daily insight agent and return a DailyInsight.""" + + async def run(self) -> DailyInsight: + insight, _payload = await self.run_and_capture_session() + return insight + + async def run_and_capture_session(self) -> tuple[DailyInsight, dict[str, Any]]: + """Run the agent and return both the DailyInsight and the session payload. + + The payload is the same structure passed to ``railtownai.upload_agent_run`` + — used by the evaluation service to feed sessions into + ``evals.extract_agent_data_points`` without re-running the agent. + """ + agent_cls = build_insight_agent() + message_history = rt.llm.MessageHistory([rt.llm.UserMessage(PROMPT)]) + + with rt.Session( + name=_SESSION_NAME, + flow_name=_FLOW_NAME, + flow_id=_FLOW_ID, + ) as session: + result = await rt.call(agent_cls, message_history) + _upload_session(session) + + session_payload = session.payload() + + text = _result_text(result).strip() + metric_count = await _count_metrics() + + insight = DailyInsight( + text=text, + generated_at=datetime.now(timezone.utc), + metric_count=metric_count, + ) + return insight, session_payload