diff --git a/.github/workflows/daily-report.yml b/.github/workflows/daily-report.yml index a9b9960..6d6bf8f 100644 --- a/.github/workflows/daily-report.yml +++ b/.github/workflows/daily-report.yml @@ -25,20 +25,41 @@ jobs: run: | pip install -r reports/requirements.txt - - name: Download database from Fly.io + - name: Fetch data from Render API env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + RENDER_SERVICE_URL: ${{ secrets.RENDER_SERVICE_URL }} run: | - # Install flyctl - curl -L https://fly.io/install.sh | sh - export FLYCTL_INSTALL="/home/runner/.fly" - export PATH="$FLYCTL_INSTALL/bin:$PATH" + # Use secret URL or fallback to default service name + SERVICE_URL="${RENDER_SERVICE_URL:-https://orp-flow-trading.onrender.com}" - # Download database file + echo "πŸ“Š Fetching trading data from: $SERVICE_URL" mkdir -p data - flyctl ssh console -C "cat /data/trades.db" > data/trades.db 2>/dev/null || echo "No database found, using empty" + + # Fetch metrics + echo "Fetching /metrics..." + curl -s "$SERVICE_URL/metrics" --connect-timeout 30 --max-time 60 > data/metrics.json || echo '{}' > data/metrics.json + + # Fetch trades history + echo "Fetching /trades..." + curl -s "$SERVICE_URL/trades" --connect-timeout 30 --max-time 60 > data/trades.json || echo '[]' > data/trades.json + + # Fetch account status + echo "Fetching /account..." + curl -s "$SERVICE_URL/account" --connect-timeout 30 --max-time 60 > data/account.json || echo '{}' > data/account.json + + # Show fetched data summary + echo "" + echo "πŸ“ˆ Fetched data summary:" + echo "Metrics:" + cat data/metrics.json | jq . 2>/dev/null || cat data/metrics.json + echo "" + echo "Trades count: $(cat data/trades.json | jq 'length' 2>/dev/null || echo 'N/A')" + echo "Account:" + cat data/account.json | jq . 2>/dev/null || cat data/account.json - name: Generate report + env: + DATA_SOURCE: api run: | python reports/generate.py @@ -59,7 +80,19 @@ jobs: run: | if [ -n "$TELEGRAM_BOT_TOKEN" ] && [ -n "$TELEGRAM_CHAT_ID" ]; then # Extract key metrics for notification - MESSAGE="πŸ“Š QuantumFlow Daily Report Generated\n\nCheck the updated README for performance metrics." + TOTAL_TRADES=$(cat data/metrics.json | jq -r '.total_trades // 0' 2>/dev/null || echo "0") + WIN_RATE=$(cat data/metrics.json | jq -r '.win_rate // 0' 2>/dev/null || echo "0") + TOTAL_PNL=$(cat data/metrics.json | jq -r '.total_pnl // 0' 2>/dev/null || echo "0") + + MESSAGE="πŸ“Š ORPFlow Daily Report Generated + +πŸ“ˆ Performance Summary: +β€’ Total Trades: $TOTAL_TRADES +β€’ Win Rate: ${WIN_RATE}% +β€’ Total P&L: \$${TOTAL_PNL} + +Check the updated README for full metrics and charts." + curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ -d "chat_id=${TELEGRAM_CHAT_ID}" \ -d "text=${MESSAGE}" \ diff --git a/.github/workflows/health-check.yml b/.github/workflows/health-check.yml index 761ae83..1049814 100644 --- a/.github/workflows/health-check.yml +++ b/.github/workflows/health-check.yml @@ -14,40 +14,58 @@ jobs: steps: - name: Check API Health id: health + env: + RENDER_SERVICE_URL: ${{ secrets.RENDER_SERVICE_URL }} run: | - response=$(curl -s -o response.json -w "%{http_code}" https://quantumflow-hft.fly.dev/health || echo "000") + # Use secret URL or fallback to default service name + SERVICE_URL="${RENDER_SERVICE_URL:-https://orp-flow-trading.onrender.com}" + + echo "Checking health at: $SERVICE_URL/health" + response=$(curl -s -o response.json -w "%{http_code}" "$SERVICE_URL/health" --connect-timeout 30 --max-time 60 || echo "000") if [ "$response" = "200" ]; then echo "status=healthy" >> $GITHUB_OUTPUT - echo "API is healthy" - cat response.json + echo "βœ… API is healthy" + cat response.json | jq . 2>/dev/null || cat response.json else echo "status=unhealthy" >> $GITHUB_OUTPUT - echo "API returned status: $response" + echo "❌ API returned status: $response" + cat response.json 2>/dev/null || echo "No response body" fi - name: Check if Shabbat pause id: shabbat if: steps.health.outputs.status == 'healthy' run: | - status=$(cat response.json | jq -r '.details.shabbat_pause // false') + status=$(cat response.json | jq -r '.details.shabbat_pause // false' 2>/dev/null || echo "false") echo "shabbat_pause=$status" >> $GITHUB_OUTPUT - echo "Shabbat pause: $status" + echo "πŸ•―οΈ Shabbat pause: $status" + + - name: Get system metrics + if: steps.health.outputs.status == 'healthy' + env: + RENDER_SERVICE_URL: ${{ secrets.RENDER_SERVICE_URL }} + run: | + SERVICE_URL="${RENDER_SERVICE_URL:-https://orp-flow-trading.onrender.com}" + + echo "πŸ“Š Fetching metrics..." + curl -s "$SERVICE_URL/metrics" --connect-timeout 10 | jq . 2>/dev/null || echo "Metrics not available" - name: Restart if unhealthy if: steps.health.outputs.status == 'unhealthy' env: - FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} + RENDER_DEPLOY_HOOK_URL: ${{ secrets.RENDER_DEPLOY_HOOK_URL }} run: | - echo "System unhealthy, attempting restart..." - - # Install flyctl - curl -L https://fly.io/install.sh | sh - export FLYCTL_INSTALL="/home/runner/.fly" - export PATH="$FLYCTL_INSTALL/bin:$PATH" + echo "⚠️ System unhealthy, attempting restart..." - # Restart the app - flyctl apps restart quantumflow-hft + if [ -n "$RENDER_DEPLOY_HOOK_URL" ]; then + # Trigger redeploy via Render deploy hook + curl -X POST "$RENDER_DEPLOY_HOOK_URL" + echo "πŸ”„ Redeploy triggered via Render deploy hook" + else + echo "⚠️ RENDER_DEPLOY_HOOK_URL not configured. Cannot trigger restart." + echo "Set up a deploy hook in Render Dashboard β†’ Settings β†’ Deploy Hook" + fi - name: Send alert on failure if: failure() @@ -56,7 +74,11 @@ jobs: TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }} run: | if [ -n "$TELEGRAM_BOT_TOKEN" ] && [ -n "$TELEGRAM_CHAT_ID" ]; then - MESSAGE="⚠️ QuantumFlow Health Check Failed\n\nThe system may be down. Automatic restart attempted." + MESSAGE="⚠️ ORPFlow Health Check Failed + +The system may be down. Automatic restart attempted. + +Check: https://dashboard.render.com" curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ -d "chat_id=${TELEGRAM_CHAT_ID}" \ -d "text=${MESSAGE}" \ diff --git a/README.md b/README.md index 7466fee..2dc11e2 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,19 @@ # ORPflow - HFT Paper Trading -> **O**Caml + **R**ust + **P**ython Flow +> **O**Caml + **R**ust + **P**erformance Flow [![CI](https://github.com/SamoraDC/ORPflow/actions/workflows/ci.yml/badge.svg)](https://github.com/SamoraDC/ORPflow/actions/workflows/ci.yml) +[![Health Check](https://github.com/SamoraDC/ORPFlow/actions/workflows/health-check.yml/badge.svg)](https://github.com/SamoraDC/ORPFlow/actions/workflows/health-check.yml) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Render](https://img.shields.io/badge/Deploy-Render-46E3B7)](https://render.com) -A high-frequency paper trading system demonstrating multi-language systems engineering with **OCaml**, **Rust**, and **Python**. Designed as a portfolio project showcasing low-latency market data processing, type-safe risk management, and quantitative trading strategies. +A high-frequency paper trading system demonstrating multi-language systems engineering with **OCaml** and **Rust**. Built following **Jane Street architecture principles**: no interpreted languages in the hot path. Uses ONNX Runtime for ML inference directly in Rust. ## Features -- **Real-time Market Data** (Rust): WebSocket connection to Binance with order book reconstruction -- **Type-Safe Risk Engine** (OCaml): Position limits, drawdown circuit breakers, P&L calculation -- **Trading Strategy** (Python): Order flow imbalance strategy with volatility adjustment +- **Unified Trading Engine** (Rust): WebSocket market data, order book, strategy, paper broker, REST API - all in one binary +- **ML Inference in Hot Path** (Rust + ONNX): LightGBM, XGBoost, CNN, LSTM models loaded via ONNX Runtime +- **Type-Safe Risk Gateway** (OCaml): Position limits, drawdown circuit breakers, P&L validation - **Paper Trading**: Realistic execution simulation with slippage, market impact, and fees - **Shabbat Pause**: Automatic trading pause from Friday to Saturday sunset - **Live Dashboard**: Auto-updating README with performance charts via GitHub Actions @@ -26,41 +27,47 @@ A high-frequency paper trading system demonstrating multi-language systems engin ## Architecture ``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Binance │────▢│ Rust │────▢│ OCaml β”‚ -β”‚ WebSocket β”‚ β”‚ Market Data β”‚ β”‚ Risk Engine β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β–Ό - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ Python β”‚ - β”‚ Strategy Engine β”‚ - β”‚ + REST API β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - β”‚ - β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β” - β”‚ SQLite β”‚ - β”‚ Database β”‚ - β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Binance │────▢│ Rust (Unified) β”‚ +β”‚ WebSocket β”‚ β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β” β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ β”‚ WebSocketβ”‚ β”‚ Strategy β”‚ β”‚ REST β”‚ β”‚ + β”‚ β”‚ + Order │──▢│ + Paper │──▢│ API β”‚ β”‚ + β”‚ β”‚ Book β”‚ β”‚ Broker β”‚ β”‚:8000 β”‚ β”‚ + β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”˜ β”‚ + β”‚ β”‚ β”‚ β”‚ + β”‚ β–Ό β–Ό β”‚ + β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ + β”‚ β”‚ ONNX Runtime (ML) β”‚ β”‚ + β”‚ β”‚ LightGBM XGBoost CNN LSTM β”‚ β”‚ + β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ IPC + β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ OCaml β”‚ + β”‚ Risk Gateway β”‚ + β”‚ (Validation) β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ ``` -### Why This Tech Stack? +### Why This Tech Stack? (Jane Street Style) | Component | Language | Reason | |-----------|----------|--------| -| Market Data | **Rust** | Zero-cost abstractions, memory safety, excellent async runtime | -| Risk Engine | **OCaml** | Algebraic data types prevent invalid states, exhaustive pattern matching | -| Strategy | **Python** | Rapid prototyping, rich data science ecosystem, easy modification | +| Unified Engine | **Rust** | Zero-cost abstractions, async I/O, ONNX bindings, no GC pauses | +| ML Inference | **ONNX** | Models trained in Python, deployed in Rust at microsecond latency | +| Risk Gateway | **OCaml** | Algebraic types prevent invalid states, exhaustive pattern matching | + +**No Python in Hot Path**: All ML models are exported to ONNX format during training (offline), then loaded by Rust for real-time inference. This eliminates Python interpreter overhead in the trading loop. ## Quick Start ### Prerequisites -- Rust 1.75+ +- Rust 1.85+ (for edition2024 support) - OCaml 5.1+ with opam -- Python 3.11+ -- Docker (optional) +- ONNX Runtime 1.19+ +- Docker (recommended for deployment) ### Setup @@ -86,41 +93,42 @@ docker-compose up --build ``` ORPflow/ -β”œβ”€β”€ market-data/ # Rust - WebSocket & order book +β”œβ”€β”€ market-data/ # Rust - Unified trading engine β”‚ └── src/ -β”‚ β”œβ”€β”€ websocket/ # Connection management -β”‚ β”œβ”€β”€ parser/ # Message deserialization -β”‚ β”œβ”€β”€ orderbook/ # Order book data structure -β”‚ └── publisher/ # IPC to other components +β”‚ β”œβ”€β”€ main.rs # Entry point + REST API (port 8000) +β”‚ β”œβ”€β”€ websocket/ # Binance WebSocket connection +β”‚ β”œβ”€β”€ orderbook/ # Order book reconstruction +β”‚ β”œβ”€β”€ strategy/ # Trading strategy logic +β”‚ β”œβ”€β”€ broker/ # Paper trading execution +β”‚ └── ml/ # ONNX model loading & inference β”‚ -β”œβ”€β”€ core/ # OCaml - Risk engine +β”œβ”€β”€ core/ # OCaml - Risk gateway β”‚ └── lib/ β”‚ β”œβ”€β”€ types/ # Domain types (Order, Trade, Position) -β”‚ β”œβ”€β”€ orderbook/ # Type-safe order book β”‚ β”œβ”€β”€ risk/ # Risk validation & limits β”‚ └── pnl/ # P&L calculation β”‚ -β”œβ”€β”€ strategy/ # Python - Trading strategy -β”‚ └── src/ -β”‚ β”œβ”€β”€ signals/ # Trading strategies -β”‚ β”œβ”€β”€ features/ # Microstructure features -β”‚ β”œβ”€β”€ broker/ # Paper broker -β”‚ β”œβ”€β”€ api/ # REST API (FastAPI) -β”‚ └── storage/ # SQLite persistence +β”œβ”€β”€ models/ # ML model training (offline, Python) +β”‚ β”œβ”€β”€ ml/ # LightGBM, XGBoost training +β”‚ β”œβ”€β”€ dl/ # CNN, LSTM training +β”‚ β”œβ”€β”€ export/ # ONNX export utilities +β”‚ └── training/ # Training scripts β”‚ -β”œβ”€β”€ reports/ # Report generator -β”‚ β”œβ”€β”€ generate.py # Chart generation -β”‚ └── templates/ # README templates +β”œβ”€β”€ trained/ # Trained model artifacts +β”‚ └── onnx/ # ONNX models for Rust runtime β”‚ -β”œβ”€β”€ docs/ # Documentation -β”‚ β”œβ”€β”€ architecture.md # System design -β”‚ β”œβ”€β”€ strategies.md # Trading logic -β”‚ └── deployment.md # Deploy guide +β”œβ”€β”€ reports/ # Report generator (GitHub Actions) +β”‚ β”œβ”€β”€ generate.py # Chart generation from API data +β”‚ └── assets/ # Generated charts +β”‚ +β”œβ”€β”€ deploy/ # Deployment configuration +β”‚ β”œβ”€β”€ supervisord.conf # Process management +β”‚ └── entrypoint.sh # Container startup β”‚ └── .github/workflows/ # CI/CD β”œβ”€β”€ ci.yml # Tests on every push β”œβ”€β”€ deploy-render.yml # Deploy to Render - β”œβ”€β”€ daily-report.yml # Update charts daily + β”œβ”€β”€ daily-report.yml # Fetch metrics & update README └── health-check.yml # Monitor every 15min ``` @@ -226,10 +234,23 @@ Track and compare strategy variations: ### Environment Variables (set in Render Dashboard) -| Variable | Required | Description | -|----------|----------|-------------| -| `TELEGRAM_BOT_TOKEN` | No | For trade notifications | -| `TELEGRAM_CHAT_ID` | No | Your Telegram chat ID | +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `SYMBOLS` | No | `BTCUSDT,ETHUSDT` | Trading symbols | +| `INITIAL_BALANCE` | No | `10000` | Starting paper balance | +| `IMBALANCE_THRESHOLD` | No | `0.3` | Order book imbalance threshold | +| `MIN_CONFIDENCE` | No | `0.6` | Minimum ML confidence for trades | +| `RISK_MAX_POSITION` | No | `1.0` | Maximum position size per symbol | +| `RISK_MAX_DRAWDOWN` | No | `0.05` | Maximum drawdown before circuit breaker | +| `TELEGRAM_BOT_TOKEN` | No | - | For trade notifications | +| `TELEGRAM_CHAT_ID` | No | - | Your Telegram chat ID | + +### GitHub Secrets (for workflows) + +| Secret | Description | +|--------|-------------| +| `RENDER_SERVICE_URL` | Your Render service URL (e.g., `https://orp-flow-trading.onrender.com`) | +| `RENDER_DEPLOY_HOOK_URL` | Deploy hook URL from Render Dashboard β†’ Settings β†’ Deploy Hook | ### Telegram Setup (Optional) @@ -257,16 +278,24 @@ Track and compare strategy variations: # All tests make test -# Individual components +# Rust tests (trading engine) cd market-data && cargo test + +# OCaml tests (risk gateway) cd core && dune test -cd strategy && pytest + +# Python tests (model training only - offline) +cd models && pytest ``` -### Linting +### Building ```bash -make lint +# Build Rust binary with ML support +cd market-data && cargo build --release --features ml + +# Build OCaml risk gateway +cd core && dune build --release ``` ### Benchmarks diff --git a/render.yaml b/render.yaml index b203ffa..a878b34 100644 --- a/render.yaml +++ b/render.yaml @@ -3,8 +3,8 @@ # https://render.com/docs/blueprint-spec services: - # Main Trading Service (Background Worker - never sleeps) - - type: worker + # Main Trading Service (Web Service for API access) + - type: web name: orp-flow-trading runtime: docker dockerfilePath: ./Dockerfile @@ -12,6 +12,9 @@ services: region: oregon # Closest to major exchanges plan: starter # $7/month - always on, no spin-down + # Health check configuration + healthCheckPath: /health + # Environment Variables envVars: # Logging diff --git a/reports/generate.py b/reports/generate.py index 2c92995..78313a6 100755 --- a/reports/generate.py +++ b/reports/generate.py @@ -1,32 +1,38 @@ #!/usr/bin/env python3 -"""Report generator for QuantumFlow HFT Paper Trading. +"""Report generator for ORPFlow HFT Paper Trading. Generates performance charts and updates the README with latest metrics. Designed to run daily via GitHub Actions. + +Supports two data sources: +1. SQLite database (local development) +2. API JSON files (production - fetched from Render API) """ +import json import os import sqlite3 import sys -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path -from typing import Any, Optional +from typing import Any import matplotlib.pyplot as plt import matplotlib.dates as mdates import numpy as np import pandas as pd -from jinja2 import Environment, FileSystemLoader # Configuration DB_PATH = os.environ.get("DATABASE_URL", "sqlite:///data/trades.db").replace("sqlite:///", "") +DATA_SOURCE = os.environ.get("DATA_SOURCE", "auto") # "auto", "api", or "sqlite" +DATA_DIR = Path(__file__).parent.parent / "data" OUTPUT_DIR = Path(__file__).parent / "assets" TEMPLATE_DIR = Path(__file__).parent / "templates" README_PATH = Path(__file__).parent.parent / "README.md" -def load_trades(db_path: str) -> pd.DataFrame: +def load_trades_from_sqlite(db_path: str) -> pd.DataFrame: """Load trades from SQLite database.""" if not Path(db_path).exists(): print(f"Database not found: {db_path}") @@ -40,7 +46,11 @@ def load_trades(db_path: str) -> pd.DataFrame: FROM trades ORDER BY timestamp ASC """ - df = pd.read_sql_query(query, conn) + try: + df = pd.read_sql_query(query, conn) + except Exception as e: + print(f"Error reading from SQLite: {e}") + df = pd.DataFrame() conn.close() if not df.empty: @@ -53,6 +63,113 @@ def load_trades(db_path: str) -> pd.DataFrame: return df +def load_trades_from_api_json(data_dir: Path) -> pd.DataFrame: + """Load trades from API JSON files (fetched by GitHub Actions).""" + trades_file = data_dir / "trades.json" + + if not trades_file.exists(): + print(f"Trades JSON not found: {trades_file}") + return pd.DataFrame() + + try: + with open(trades_file) as f: + trades_data = json.load(f) + + if not trades_data: + print("No trades in JSON file") + return pd.DataFrame() + + # Handle both array and object with trades key + if isinstance(trades_data, dict): + trades_data = trades_data.get("trades", []) + + df = pd.DataFrame(trades_data) + + if df.empty: + return df + + # Normalize column names (API might use camelCase) + column_mapping = { + "orderId": "order_id", + "orderid": "order_id", + "timestamp": "timestamp", + "created_at": "timestamp", + "createdAt": "timestamp", + } + df = df.rename(columns=column_mapping) + + # Ensure required columns exist + for col in ["symbol", "side", "price", "quantity", "fee", "pnl", "timestamp"]: + if col not in df.columns: + if col == "fee": + df[col] = 0.0 + elif col == "pnl": + df[col] = 0.0 + else: + df[col] = None + + df["timestamp"] = pd.to_datetime(df["timestamp"]) + df["price"] = pd.to_numeric(df["price"], errors="coerce").fillna(0) + df["quantity"] = pd.to_numeric(df["quantity"], errors="coerce").fillna(0) + df["fee"] = pd.to_numeric(df["fee"], errors="coerce").fillna(0) + df["pnl"] = pd.to_numeric(df["pnl"], errors="coerce").fillna(0) + + return df.sort_values("timestamp").reset_index(drop=True) + + except json.JSONDecodeError as e: + print(f"Error parsing trades JSON: {e}") + return pd.DataFrame() + except Exception as e: + print(f"Error loading trades from JSON: {e}") + return pd.DataFrame() + + +def load_metrics_from_api_json(data_dir: Path) -> dict[str, Any] | None: + """Load pre-calculated metrics from API JSON (if available).""" + metrics_file = data_dir / "metrics.json" + + if not metrics_file.exists(): + return None + + try: + with open(metrics_file) as f: + metrics = json.load(f) + + # Return None if empty or error response + if not metrics or "error" in metrics: + return None + + return metrics + + except (json.JSONDecodeError, Exception) as e: + print(f"Error loading metrics JSON: {e}") + return None + + +def load_trades() -> pd.DataFrame: + """Load trades from the best available source.""" + source = DATA_SOURCE.lower() + + if source == "api": + print("Loading trades from API JSON...") + return load_trades_from_api_json(DATA_DIR) + + elif source == "sqlite": + print(f"Loading trades from SQLite: {DB_PATH}") + return load_trades_from_sqlite(DB_PATH) + + else: # auto + # Try API JSON first (from GitHub Actions), then SQLite + if (DATA_DIR / "trades.json").exists(): + print("Auto-detected: Loading trades from API JSON...") + df = load_trades_from_api_json(DATA_DIR) + if not df.empty: + return df + + print(f"Auto-detected: Loading trades from SQLite: {DB_PATH}") + return load_trades_from_sqlite(DB_PATH) + + def calculate_metrics(trades: pd.DataFrame, initial_balance: float = 10000.0) -> dict[str, Any]: """Calculate performance metrics from trades.""" if trades.empty: @@ -87,6 +204,7 @@ def calculate_metrics(trades: pd.DataFrame, initial_balance: float = 10000.0) -> total_fees = trades["fee"].sum() # Equity curve + trades = trades.copy() trades["cumulative_pnl"] = trades["pnl"].cumsum() trades["equity"] = initial_balance + trades["cumulative_pnl"] @@ -142,6 +260,7 @@ def generate_equity_curve(trades: pd.DataFrame, output_path: Path, initial_balan if trades.empty: return + trades = trades.copy() trades["cumulative_pnl"] = trades["pnl"].cumsum() trades["equity"] = initial_balance + trades["cumulative_pnl"] @@ -190,6 +309,7 @@ def generate_drawdown_chart(trades: pd.DataFrame, output_path: Path, initial_bal if trades.empty: return + trades = trades.copy() trades["cumulative_pnl"] = trades["pnl"].cumsum() trades["equity"] = initial_balance + trades["cumulative_pnl"] trades["peak"] = trades["equity"].cummax() @@ -260,6 +380,7 @@ def generate_hourly_heatmap(trades: pd.DataFrame, output_path: Path) -> None: if trades.empty: return + trades = trades.copy() trades["hour"] = trades["timestamp"].dt.hour trades["dayofweek"] = trades["timestamp"].dt.dayofweek @@ -300,34 +421,32 @@ def generate_hourly_heatmap(trades: pd.DataFrame, output_path: Path) -> None: print(f"Generated: {output_path}") -def update_readme(metrics: dict[str, Any], readme_path: Path) -> None: +def update_readme(metrics: dict[str, Any], readme_path: Path, has_charts: bool = False) -> None: """Update README with latest metrics.""" - # Load template - env = Environment(loader=FileSystemLoader(str(TEMPLATE_DIR))) + updated_at = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") - try: - template = env.get_template("metrics_section.md.j2") - metrics_section = template.render( - metrics=metrics, - updated_at=datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC"), - ) - except Exception as e: - print(f"Template error: {e}") - # Fallback to simple format - metrics_section = f""" -## Live Performance + # Generate metrics section + if metrics["total_trades"] == 0: + metrics_section = f"""## Live Performance -*Last updated: {datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")}* +*Last updated: {updated_at}* + +*System starting - metrics will appear after first trades* | Metric | Value | |--------|-------| -| Total Trades | {metrics['total_trades']} | -| Win Rate | {metrics['win_rate']:.1f}% | -| Total P&L | ${metrics['total_pnl']:.2f} ({metrics['total_pnl_pct']:.2f}%) | -| Sharpe Ratio | {metrics['sharpe_ratio']:.2f} | -| Max Drawdown | {metrics['max_drawdown_pct']:.2f}% | -| Profit Factor | {metrics['profit_factor']:.2f} | +| Total Trades | 0 | +| Win Rate | 0% | +| Total P&L | $0.00 | +| Sharpe Ratio | - | +| Max Drawdown | 0% | +*Charts will be generated daily by GitHub Actions* +""" + else: + charts_section = "" + if has_charts: + charts_section = """ ### Equity Curve ![Equity Curve](reports/assets/equity_curve.png) @@ -341,6 +460,31 @@ def update_readme(metrics: dict[str, Any], readme_path: Path) -> None: ![Hourly Heatmap](reports/assets/hourly_heatmap.png) """ + metrics_section = f"""## Live Performance + +*Last updated: {updated_at}* + +| Metric | Value | +|--------|-------| +| Total Trades | {metrics['total_trades']} | +| Win Rate | {metrics['win_rate']:.1f}% | +| Total P&L | ${metrics['total_pnl']:.2f} ({metrics['total_pnl_pct']:+.2f}%) | +| Sharpe Ratio | {metrics['sharpe_ratio']:.2f} | +| Max Drawdown | {metrics['max_drawdown_pct']:.2f}% | +| Profit Factor | {metrics['profit_factor']:.2f} | +| Trading Days | {metrics['trading_days']} | +| Trades/Day | {metrics['trades_per_day']:.1f} | + +| Win/Loss Stats | Value | +|----------------|-------| +| Winning Trades | {metrics['winning_trades']} | +| Losing Trades | {metrics['losing_trades']} | +| Average Win | ${metrics['avg_win']:.2f} | +| Average Loss | ${metrics['avg_loss']:.2f} | +| Largest Win | ${metrics['largest_win']:.2f} | +| Largest Loss | ${metrics['largest_loss']:.2f} | +{charts_section}""" + # Read current README if readme_path.exists(): readme_content = readme_path.read_text() @@ -363,33 +507,47 @@ def update_readme(metrics: dict[str, Any], readme_path: Path) -> None: def main() -> int: """Main entry point.""" - print("QuantumFlow Report Generator") + print("ORPFlow Report Generator") print("=" * 40) + print(f"Data source: {DATA_SOURCE}") # Create output directory OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + # Try to load pre-calculated metrics from API (if available) + api_metrics = load_metrics_from_api_json(DATA_DIR) + # Load trades - print(f"Loading trades from: {DB_PATH}") - trades = load_trades(DB_PATH) + trades = load_trades() if trades.empty: print("No trades found. Generating placeholder report.") - # Calculate metrics - metrics = calculate_metrics(trades) - print(f"Total trades: {metrics['total_trades']}") - print(f"Total P&L: ${metrics['total_pnl']:.2f}") + # Calculate metrics (or use API metrics if available) + if api_metrics and api_metrics.get("total_trades", 0) > 0: + print("Using pre-calculated metrics from API") + metrics = api_metrics + # Add missing keys for compatibility + for key in ["equity_curve", "drawdown_curve"]: + if key not in metrics: + metrics[key] = [] + else: + metrics = calculate_metrics(trades) + + print(f"Total trades: {metrics.get('total_trades', 0)}") + print(f"Total P&L: ${metrics.get('total_pnl', 0):.2f}") # Generate charts + has_charts = False if not trades.empty: generate_equity_curve(trades, OUTPUT_DIR / "equity_curve.png") generate_drawdown_chart(trades, OUTPUT_DIR / "drawdown.png") generate_pnl_distribution(trades, OUTPUT_DIR / "pnl_distribution.png") generate_hourly_heatmap(trades, OUTPUT_DIR / "hourly_heatmap.png") + has_charts = True # Update README - update_readme(metrics, README_PATH) + update_readme(metrics, README_PATH, has_charts=has_charts) print("\nReport generation complete!") return 0