From 44a3bb756f855c9556f8e8a73ec66510bc75c5c0 Mon Sep 17 00:00:00 2001 From: onepunchmonk Date: Mon, 7 Sep 2026 20:53:55 +0530 Subject: [PATCH] docs: clarify experimental evolution claims --- HARNESS_EVOLUTION_RESULTS.md | 8 ++++---- README.md | 21 +++++++++++---------- docs/TOOL_INTEGRATION_GUIDE.md | 8 ++++---- scripts/README.md | 2 +- scripts/harness_evolution_6_epochs.py | 4 +++- src/agent/agent_graph.py | 9 +++++---- 6 files changed, 28 insertions(+), 24 deletions(-) diff --git a/HARNESS_EVOLUTION_RESULTS.md b/HARNESS_EVOLUTION_RESULTS.md index 9ac424c..3d17fee 100644 --- a/HARNESS_EVOLUTION_RESULTS.md +++ b/HARNESS_EVOLUTION_RESULTS.md @@ -2,7 +2,7 @@ ## Executive Summary -Completed multi-iteration harness evolution with **3 evolutionary algorithms** and **6 epochs** of manual progression. +This is an archived development report for a six-epoch manual progression and experimental optimizer comparisons. The GA/DE comparison uses a mock fitness function, and the recorded claim-accuracy values are placeholders rather than measured forecast accuracy. **Result: +37.4% Sharpe improvement** (0.452 → 0.621) @@ -31,7 +31,7 @@ Epoch 6: v6_research +37.4% 0.621 Gap 0.048 Tools: 8 ✓ | Max Drawdown | 0.185 | 0.168 | 0.161 | 0.152 | 0.145 | 0.138 | | Win Rate | 60% | 80% | 82% | 85% | 87% | 90% | | Tool Calls | 0 | 3 | 4 | 5 | 6 | 8 | -| Claim Accuracy | — | 75% | 78% | 81% | 83% | 86% | +| Claim tracking | — | recorded | recorded | recorded | recorded | recorded | --- @@ -176,7 +176,7 @@ Three optimized harnesses saved to `.harness/` directory: ## Falsifiable Claims Accuracy -Tracked how accurately agent predicted proposal success: +Claims were recorded alongside proposals. These values were illustrative placeholders, not measured numerical-Sharpe forecast accuracy: ``` v2_tool_aware: 75% accuracy @@ -186,7 +186,7 @@ v5_multi_agent: 83% accuracy v6_research: 86% accuracy ``` -**Insight:** Claim accuracy improved as harness improved — system learned which predictions matter +**Status:** Claim accuracy is not reported until forecasts are stored in a structured form and evaluated against realized outcomes. --- diff --git a/README.md b/README.md index 574a8d5..da40f95 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ Most trading agent frameworks are static parameter-tuning tools. **AgentQuant is - ✅ **Runs a real ReAct loop** — analyze → hypothesize → backtest → reflect → store → improve - ✅ **Remembers across runs** — Cross-session SQLite memory lets the agent learn what worked - ✅ **Measures generalization** — Tracks overfitting risk with explicit train/validation/test splits -- ✅ **Evolves itself** — Uses genetic algorithms and differential evolution to optimize harness parameters -- ✅ **Makes falsifiable claims** — Every proposal includes predicted Sharpe; accuracy is tracked +- 🧪 **Includes experimental optimizers** — Genetic algorithms and differential evolution can search harness parameters; their benchmark currently uses a mock fitness function +- ✅ **Records falsifiable claims** — Proposals can include confidence and written outcome claims for later analysis; no calibrated Sharpe-prediction-accuracy metric is reported - ✅ **Integrates web search** — Uses Tavily to find market sentiment and strategy research in real-time - ✅ **Production-grade**: 63 unit tests, CI/CD gates, security checks, look-ahead bias guards @@ -39,11 +39,11 @@ Starting from a baseline grid-search agent, we evolved the harness through 6 pro **Key validations:** - ✅ **Generalization gap reduced 61%** (0.124 → 0.048) — improvements are real, not artifacts - ✅ **Tool efficiency increased 8x** (0 → 8 calls/epoch) -- ✅ **Claim accuracy 86%** — falsifiable claims systematically validated +- ℹ️ **Claim accuracy is not reported** — the current harness records claims but does not yet evaluate numerical Sharpe forecasts against realized outcomes ### Algorithm Comparison -Compared manual evolution against evolutionary algorithms on the same fitness function: +Compared manual evolution against experimental evolutionary optimizers on the same mock fitness function. These figures are a development benchmark, not backtest results. ``` Manual Evolution (Hand-crafted) ⭐ 0.621 (+37.4%) Domain knowledge wins @@ -52,7 +52,7 @@ Differential Evolution (20×5) → 0.571 (+28.3%) Struggles with discret Random Baseline (Control) → 0.465 (+12.9%) All beat random 5-33x ``` -**Finding:** Manual strategy beats algorithms due to domain knowledge encoding discrete decisions (tools on/off). But GA finds near-optimal solutions 16% faster. +**Development observation:** In this mock-fitness benchmark, the hand-crafted configuration scored higher than the experimental optimizers. This is not evidence of live or historical trading performance. ### Evolution Visualization @@ -84,7 +84,7 @@ v6_research ⭐ (0.621) [+37.4% total] - 📈 **Sharpe Improvement:** +37.4% (0.452 → 0.621) - 🎯 **Generalization Gap:** -61% (0.124 → 0.048) - 🔧 **Tool Integration:** 8x increase in tool calls per epoch -- ✓ **Claim Accuracy:** 86% falsifiable predictions validated +- ℹ️ **Claim validation:** recorded for analysis; numerical forecast accuracy is not yet reported ### UI & Dashboards @@ -167,7 +167,7 @@ graph TD 4. REFLECT • Score results: is Sharpe ≥ threshold? - • Track falsifiable claims (predicted vs. realized) + • Record falsifiable claims for later analysis (numerical forecast accuracy is not yet calibrated) • If below threshold, retry up to max_iterations • Score proposals for generalization risk @@ -252,8 +252,8 @@ python scripts/harness_evolution_6_epochs.py \ python scripts/benchmark_harness_evolution.py \ --strategy momentum -# Compares: Manual vs GA vs DE vs Random -# Output: JSON report with algorithm comparison +# Compares: Manual vs experimental GA vs experimental DE vs Random +# Output: JSON report based on a mock fitness function (not backtests) ``` ### Run Agent (Streamlit UI) @@ -334,7 +334,7 @@ pytest tests/ ### What This Does ✅ Discovers regime-aware trading parameters -✅ Evolves itself through iterative improvement +🧪 Includes experimental iterative harness optimization ✅ Remembers across runs (SQLite memory) ✅ Backtests with realistic costs ✅ Integrates web search for context @@ -343,6 +343,7 @@ pytest tests/ ### What This Doesn't Do ❌ Predict future prices (impossible) ❌ Guarantee profit (backtest ≠ live trading) +❌ Report calibrated numerical Sharpe forecasts or use GA/DE benchmark output as backtest evidence ❌ Beat the market (we haven't shipped live yet) ❌ Work without data (needs 5y+ history minimum) ❌ Replace a professional researcher (it's a tool) diff --git a/docs/TOOL_INTEGRATION_GUIDE.md b/docs/TOOL_INTEGRATION_GUIDE.md index 3121820..11ea6d2 100644 --- a/docs/TOOL_INTEGRATION_GUIDE.md +++ b/docs/TOOL_INTEGRATION_GUIDE.md @@ -104,11 +104,11 @@ result = orchestrator.run_tool_loop( When a proposal is generated with a claim (e.g., "This window length will improve Sharpe by 15%"): 1. **Proposal generation** — claim is recorded in `Proposal.reasoning` -2. **Reflect node** — `_score_falsifiable_claims()` compares predicted vs. actual -3. **Memory storage** — claim accuracy tracked in `StrategyMemory` -4. **Harness eval** — eval suite scores "% of claims materialized" +2. **Reflect node** — `_score_falsifiable_claims()` currently records a confidence/outcome heuristic +3. **Memory storage** — proposal and backtest context can be retained for future evaluation +4. **Harness eval** — structured numerical forecasts and measured accuracy remain future work -This creates a feedback loop where the harness learns which proposal strategies work. +This provides inputs for a future feedback loop; it does not yet establish prediction accuracy. ## Extending the Tool System diff --git a/scripts/README.md b/scripts/README.md index 63a78bd..cde5525 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -106,7 +106,7 @@ python scripts/harness_evolution_poc.py --output my_results.json ### Short-term 1. Implement walk-forward validation (train/val/held-out split) -2. Track falsifiable claim accuracy across epochs +2. Implement structured falsifiable-claim evaluation across epochs 3. Extend evolution strategy (parameter grid adaptation) ### Medium-term diff --git a/scripts/harness_evolution_6_epochs.py b/scripts/harness_evolution_6_epochs.py index 26af500..54bb7a8 100644 --- a/scripts/harness_evolution_6_epochs.py +++ b/scripts/harness_evolution_6_epochs.py @@ -281,7 +281,9 @@ def _compute_metrics(self, state: Dict[str, Any], elapsed: float) -> EpochMetric proposals_generated=len(proposals), proposals_accepted=1, execution_time=elapsed, - claim_accuracy=0.8, # Placeholder + # Numerical claim accuracy is not available until proposals carry + # structured forecasts that can be evaluated against outcomes. + claim_accuracy=0.0, ) def _describe_harness_changes(self, harness_spec: Dict[str, Any]) -> List[str]: diff --git a/src/agent/agent_graph.py b/src/agent/agent_graph.py index 4265b26..43cf23c 100644 --- a/src/agent/agent_graph.py +++ b/src/agent/agent_graph.py @@ -244,10 +244,10 @@ def reflect_node(state: AgentState) -> AgentState: def _score_falsifiable_claims(state: AgentState, best_result: Dict[str, Any]) -> None: """ - Score falsifiable claims from proposals against realized outcomes. + Log a provisional confidence/outcome diagnostic for proposal claims. - This enables the harness to learn which proposal-generation strategies actually work. - Claims are scored by checking if predicted improvements materialized. + Proposals currently store free-form claim text rather than a structured + numerical Sharpe forecast, so this is not a prediction-accuracy score. """ proposals = state.get("proposals", []) if not proposals: @@ -259,7 +259,8 @@ def _score_falsifiable_claims(state: AgentState, best_result: Dict[str, Any]) -> confidence = proposal.confidence params = proposal.params - # Heuristic scoring: if claim predicted positive and realized is positive, mark accurate + # Diagnostic only: structured forecasts are required before accuracy can + # be computed against realized Sharpe. predicted_improvement = confidence > 0.5 and realized_sharpe > config.agent.min_acceptable_sharpe actual_improvement = realized_sharpe > config.agent.min_acceptable_sharpe