From 9636a4f567e2dd8d3cd96fe0dd4ddb1ff6c388b6 Mon Sep 17 00:00:00 2001 From: hsm207 Date: Sun, 23 Aug 2026 20:09:25 +0200 Subject: [PATCH 1/3] Generate daily option universe files in RandomDataGenerator (#8854) - Automatically generate daily option universe CSV files when generating synthetic options data - Add standalone data provider fallback to InterestRateProvider - Add OptionUniverseWriter unit test suite --- Common/Data/InterestRateProvider.cs | 3 +- .../OptionUniverseWriterTests.cs | 97 ++++++++++++ .../OptionUniverseWriter.cs | 149 ++++++++++++++++++ .../RandomDataGenerator.cs | 6 + 4 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 Tests/ToolBox/RandomDataGenerator/OptionUniverseWriterTests.cs create mode 100644 ToolBox/RandomDataGenerator/OptionUniverseWriter.cs diff --git a/Common/Data/InterestRateProvider.cs b/Common/Data/InterestRateProvider.cs index 5c6aad24455a..d55d6decc57d 100644 --- a/Common/Data/InterestRateProvider.cs +++ b/Common/Data/InterestRateProvider.cs @@ -118,7 +118,8 @@ public static Dictionary FromCsvFile(string file, out decimal // skip the first header line, also skip #'s as these are comment lines var interestRateProvider = new Dictionary(); - foreach (var line in dataProvider.ReadLines(file).Skip(1) + var lines = dataProvider != null ? dataProvider.ReadLines(file) : File.ReadLines(file); + foreach (var line in lines.Skip(1) .Where(x => !string.IsNullOrWhiteSpace(x))) { if (TryParse(line, out var date, out var interestRate)) diff --git a/Tests/ToolBox/RandomDataGenerator/OptionUniverseWriterTests.cs b/Tests/ToolBox/RandomDataGenerator/OptionUniverseWriterTests.cs new file mode 100644 index 000000000000..a8d2a1287be7 --- /dev/null +++ b/Tests/ToolBox/RandomDataGenerator/OptionUniverseWriterTests.cs @@ -0,0 +1,97 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.IO; +using NUnit.Framework; +using QuantConnect.Configuration; +using QuantConnect.Data; +using QuantConnect.Data.Market; +using QuantConnect.Securities; +using QuantConnect.ToolBox.RandomDataGenerator; + +namespace QuantConnect.Tests.ToolBox.RandomDataGenerator; + +/// +/// Unit tests verifying that generates valid daily option universe CSV files. +/// +[TestFixture] +internal class OptionUniverseWriterTests +{ + private string _tempDataFolder = null!; + private string _originalDataFolder = null!; + + [SetUp] + public void SetUp() + { + _tempDataFolder = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + Directory.CreateDirectory(_tempDataFolder); + + _originalDataFolder = Globals.DataFolder; + Config.Set("data-folder", _tempDataFolder); + Globals.Reset(); + } + + [TearDown] + public void TearDown() + { + Config.Set("data-folder", _originalDataFolder); + Globals.Reset(); + + if (Directory.Exists(_tempDataFolder)) + { + Directory.Delete(_tempDataFolder, true); + } + } + + /// + /// Verifies that WriteUniverseFiles aggregates intraday contract ticks and writes the expected daily universe CSV. + /// + [Test] + public void WriteUniverseFiles_GivenOptionTicks_GeneratesMatchingDailyUniverseCsv() + { + var (underlying, tickHistories) = CreateSampleOptionTickHistory(); + + OptionUniverseWriter.WriteUniverseFiles(underlying, tickHistories); + + var lines = ReadGeneratedUniverseFile("tsla", "20260205"); + Assert.That(lines.Length, Is.EqualTo(3)); + Assert.That(lines[0], Is.EqualTo("expiry,strike,right,open,high,low,close,volume,open_interest,implied_volatility,delta,gamma,vega,theta,rho")); + Assert.That(lines[1], Is.EqualTo(",,,100.0000,100.0000,100.0000,100.0000,0,,,,,,,")); + Assert.That(lines[2], Is.EqualTo("20260213,200,C,1.50,1.75,1.50,1.75,30,20,0.20,,,,,")); + } + + private static (Symbol Underlying, Dictionary> Histories) CreateSampleOptionTickHistory() + { + var underlying = Symbol.Create("TSLA", SecurityType.Equity, Market.USA); + var option = Symbol.CreateOption(underlying, Market.USA, OptionStyle.American, OptionRight.Call, 200m, new DateTime(2026, 2, 13)); + + var ticks = new List + { + new(new DateTime(2026, 2, 5, 9, 30, 0), option, 1.50m, 1.50m) { Quantity = 10 }, + new(new DateTime(2026, 2, 5, 16, 0, 0), option, 1.75m, 1.75m) { Quantity = 20 } + }; + + return (underlying, new Dictionary> { { option, ticks } }); + } + + private string[] ReadGeneratedUniverseFile(string ticker, string date) + { + var path = Path.Combine(_tempDataFolder, "option", "usa", "universes", ticker, $"{date}.csv"); + Assert.That(File.Exists(path), Is.True, $"Universe file was not found at expected path: {path}"); + return File.ReadAllLines(path); + } +} diff --git a/ToolBox/RandomDataGenerator/OptionUniverseWriter.cs b/ToolBox/RandomDataGenerator/OptionUniverseWriter.cs new file mode 100644 index 000000000000..4bee40f5701b --- /dev/null +++ b/ToolBox/RandomDataGenerator/OptionUniverseWriter.cs @@ -0,0 +1,149 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using QuantConnect.Data.Market; +using QuantConnect.Data.UniverseSelection; + +namespace QuantConnect.ToolBox.RandomDataGenerator; + +/// +/// Writes daily Option Universe CSV files for generated options data. +/// +internal static class OptionUniverseWriter +{ + private const decimal DefaultSyntheticImpliedVolatility = 0.20m; + private const decimal DefaultUnderlyingPrice = 100.00m; + + private readonly record struct OptionRow( + Symbol Symbol, + decimal Open, + decimal High, + decimal Low, + decimal Close, + decimal Volume, + decimal OpenInterest + ); + + /// + /// Processes generated tick histories and writes daily universe selection CSVs for the option. + /// + /// The underlying asset symbol. + /// The historical ticks generated for each option contract. + public static void WriteUniverseFiles(Symbol underlyingSymbol, Dictionary> tickHistories) + { + var dailyData = AggregateDailyOptionRows(tickHistories); + WriteAllDailyUniverseFiles(underlyingSymbol, dailyData); + } + + private static Dictionary> AggregateDailyOptionRows( + Dictionary> tickHistories) + { + var dailyContractData = new Dictionary>(); + + foreach (var (contract, ticks) in tickHistories) + { + if (ticks.Count == 0) + { + continue; + } + + AggregateContractTicks(contract, ticks, dailyContractData); + } + + return dailyContractData; + } + + private static void AggregateContractTicks( + Symbol contract, + List ticks, + Dictionary> dailyContractData) + { + foreach (var dayGroup in ticks.GroupBy(t => t.Time.Date)) + { + var dayTicks = dayGroup.ToList(); + var row = CreateOptionRow(contract, dayTicks); + + if (!dailyContractData.TryGetValue(dayGroup.Key, out var list)) + { + list = new List(); + dailyContractData[dayGroup.Key] = list; + } + + list.Add(row); + } + } + + private static OptionRow CreateOptionRow(Symbol contract, List dayTicks) + { + var open = dayTicks.First().Value; + var high = dayTicks.Max(t => t.Value); + var low = dayTicks.Min(t => t.Value); + var close = dayTicks.Last().Value; + var volume = dayTicks.Sum(t => t.Quantity); + var openInterest = dayTicks.Last().Quantity; + + return new OptionRow(contract, open, high, low, close, volume, openInterest); + } + + private static void WriteAllDailyUniverseFiles(Symbol underlyingSymbol, Dictionary> dailyData) + { + var canonicalSymbol = Symbol.CreateCanonicalOption(underlyingSymbol); + + foreach (var (date, rows) in dailyData) + { + WriteSingleDailyUniverseFile(canonicalSymbol, date, rows); + } + } + + private static void WriteSingleDailyUniverseFile(Symbol canonicalSymbol, DateTime date, List rows) + { + var universePath = BaseChainUniverseData.GetUniverseFullFilePath(canonicalSymbol, date); + Directory.CreateDirectory(Path.GetDirectoryName(universePath)!); + + var lines = GenerateCsvLines(rows); + File.WriteAllLines(universePath, lines); + } + + private static IEnumerable GenerateCsvLines(List rows) + { + yield return OptionUniverse.CsvHeader(SecurityType.Option); + yield return FormatUnderlyingRow(DefaultUnderlyingPrice); + + var sortedRows = rows.OrderBy(r => r.Symbol.ID.Date).ThenBy(r => r.Symbol.ID.StrikePrice); + foreach (var row in sortedRows) + { + yield return OptionUniverse.ToCsv( + row.Symbol, + row.Open, + row.High, + row.Low, + row.Close, + row.Volume, + row.OpenInterest, + DefaultSyntheticImpliedVolatility, + null + ); + } + } + + private static string FormatUnderlyingRow(decimal price) + { + return FormattableString.Invariant($",,,{price:F4},{price:F4},{price:F4},{price:F4},0,,,,,,,"); + } +} diff --git a/ToolBox/RandomDataGenerator/RandomDataGenerator.cs b/ToolBox/RandomDataGenerator/RandomDataGenerator.cs index da8e2a154d16..aacc42a65124 100644 --- a/ToolBox/RandomDataGenerator/RandomDataGenerator.cs +++ b/ToolBox/RandomDataGenerator/RandomDataGenerator.cs @@ -279,6 +279,12 @@ public void Run() currentCount++; } } + + // Generate daily option universe CSVs if generating options data + if (_settings.SecurityType == SecurityType.Option) + { + OptionUniverseWriter.WriteUniverseFiles(symbolRef, tickHistories); + } } Log.Trace("RandomDataGenerator.Run(): Random data generation has completed."); From 4f1c97ce2f9a0aba824e03697cdf8157309ef02f Mon Sep 17 00:00:00 2001 From: hsm207 Date: Sun, 23 Aug 2026 21:01:03 +0200 Subject: [PATCH 2/3] Filter option contracts in OptionUniverseWriter to handle mixed security dictionaries --- .../RandomDataGenerator/OptionUniverseWriterTests.cs | 11 ++++++++++- ToolBox/RandomDataGenerator/OptionUniverseWriter.cs | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Tests/ToolBox/RandomDataGenerator/OptionUniverseWriterTests.cs b/Tests/ToolBox/RandomDataGenerator/OptionUniverseWriterTests.cs index a8d2a1287be7..99ac3260ba6c 100644 --- a/Tests/ToolBox/RandomDataGenerator/OptionUniverseWriterTests.cs +++ b/Tests/ToolBox/RandomDataGenerator/OptionUniverseWriterTests.cs @@ -79,13 +79,22 @@ private static (Symbol Underlying, Dictionary> Histories) Cre var underlying = Symbol.Create("TSLA", SecurityType.Equity, Market.USA); var option = Symbol.CreateOption(underlying, Market.USA, OptionStyle.American, OptionRight.Call, 200m, new DateTime(2026, 2, 13)); + var equityTicks = new List + { + new(new DateTime(2026, 2, 5, 9, 30, 0), underlying, 200m, 200m) { Quantity = 100 } + }; + var ticks = new List { new(new DateTime(2026, 2, 5, 9, 30, 0), option, 1.50m, 1.50m) { Quantity = 10 }, new(new DateTime(2026, 2, 5, 16, 0, 0), option, 1.75m, 1.75m) { Quantity = 20 } }; - return (underlying, new Dictionary> { { option, ticks } }); + return (underlying, new Dictionary> + { + { underlying, equityTicks }, + { option, ticks } + }); } private string[] ReadGeneratedUniverseFile(string ticker, string date) diff --git a/ToolBox/RandomDataGenerator/OptionUniverseWriter.cs b/ToolBox/RandomDataGenerator/OptionUniverseWriter.cs index 4bee40f5701b..0ba1119b1f49 100644 --- a/ToolBox/RandomDataGenerator/OptionUniverseWriter.cs +++ b/ToolBox/RandomDataGenerator/OptionUniverseWriter.cs @@ -58,7 +58,7 @@ private static Dictionary> AggregateDailyOptionRows( foreach (var (contract, ticks) in tickHistories) { - if (ticks.Count == 0) + if (contract.SecurityType != SecurityType.Option || ticks.Count == 0) { continue; } From 657c2ea420aab486c1fb1dbf8f1b4ae6a96e7e35 Mon Sep 17 00:00:00 2001 From: hsm207 Date: Sun, 23 Aug 2026 21:07:21 +0200 Subject: [PATCH 3/3] Refactor OptionUniverseWriter to accept filtered option tick histories adhering to SRP --- .../OptionUniverseWriterTests.cs | 15 +++------------ .../RandomDataGenerator/OptionUniverseWriter.cs | 14 +++++++------- .../RandomDataGenerator/RandomDataGenerator.cs | 5 ++++- 3 files changed, 14 insertions(+), 20 deletions(-) diff --git a/Tests/ToolBox/RandomDataGenerator/OptionUniverseWriterTests.cs b/Tests/ToolBox/RandomDataGenerator/OptionUniverseWriterTests.cs index 99ac3260ba6c..65a3e10e8147 100644 --- a/Tests/ToolBox/RandomDataGenerator/OptionUniverseWriterTests.cs +++ b/Tests/ToolBox/RandomDataGenerator/OptionUniverseWriterTests.cs @@ -63,9 +63,9 @@ public void TearDown() [Test] public void WriteUniverseFiles_GivenOptionTicks_GeneratesMatchingDailyUniverseCsv() { - var (underlying, tickHistories) = CreateSampleOptionTickHistory(); + var (underlying, optionTickHistories) = CreateSampleOptionTickHistory(); - OptionUniverseWriter.WriteUniverseFiles(underlying, tickHistories); + OptionUniverseWriter.WriteUniverseFiles(underlying, optionTickHistories); var lines = ReadGeneratedUniverseFile("tsla", "20260205"); Assert.That(lines.Length, Is.EqualTo(3)); @@ -79,22 +79,13 @@ private static (Symbol Underlying, Dictionary> Histories) Cre var underlying = Symbol.Create("TSLA", SecurityType.Equity, Market.USA); var option = Symbol.CreateOption(underlying, Market.USA, OptionStyle.American, OptionRight.Call, 200m, new DateTime(2026, 2, 13)); - var equityTicks = new List - { - new(new DateTime(2026, 2, 5, 9, 30, 0), underlying, 200m, 200m) { Quantity = 100 } - }; - var ticks = new List { new(new DateTime(2026, 2, 5, 9, 30, 0), option, 1.50m, 1.50m) { Quantity = 10 }, new(new DateTime(2026, 2, 5, 16, 0, 0), option, 1.75m, 1.75m) { Quantity = 20 } }; - return (underlying, new Dictionary> - { - { underlying, equityTicks }, - { option, ticks } - }); + return (underlying, new Dictionary> { { option, ticks } }); } private string[] ReadGeneratedUniverseFile(string ticker, string date) diff --git a/ToolBox/RandomDataGenerator/OptionUniverseWriter.cs b/ToolBox/RandomDataGenerator/OptionUniverseWriter.cs index 0ba1119b1f49..e4db9d3acca3 100644 --- a/ToolBox/RandomDataGenerator/OptionUniverseWriter.cs +++ b/ToolBox/RandomDataGenerator/OptionUniverseWriter.cs @@ -41,24 +41,24 @@ decimal OpenInterest ); /// - /// Processes generated tick histories and writes daily universe selection CSVs for the option. + /// Processes generated option tick histories and writes daily universe selection CSVs. /// /// The underlying asset symbol. - /// The historical ticks generated for each option contract. - public static void WriteUniverseFiles(Symbol underlyingSymbol, Dictionary> tickHistories) + /// The historical ticks generated for each option contract. + public static void WriteUniverseFiles(Symbol underlyingSymbol, Dictionary> optionTickHistories) { - var dailyData = AggregateDailyOptionRows(tickHistories); + var dailyData = AggregateDailyOptionRows(optionTickHistories); WriteAllDailyUniverseFiles(underlyingSymbol, dailyData); } private static Dictionary> AggregateDailyOptionRows( - Dictionary> tickHistories) + Dictionary> optionTickHistories) { var dailyContractData = new Dictionary>(); - foreach (var (contract, ticks) in tickHistories) + foreach (var (contract, ticks) in optionTickHistories) { - if (contract.SecurityType != SecurityType.Option || ticks.Count == 0) + if (ticks.Count == 0) { continue; } diff --git a/ToolBox/RandomDataGenerator/RandomDataGenerator.cs b/ToolBox/RandomDataGenerator/RandomDataGenerator.cs index aacc42a65124..11f66731cb03 100644 --- a/ToolBox/RandomDataGenerator/RandomDataGenerator.cs +++ b/ToolBox/RandomDataGenerator/RandomDataGenerator.cs @@ -283,7 +283,10 @@ public void Run() // Generate daily option universe CSVs if generating options data if (_settings.SecurityType == SecurityType.Option) { - OptionUniverseWriter.WriteUniverseFiles(symbolRef, tickHistories); + var optionTickHistories = tickHistories + .Where(kvp => kvp.Key.SecurityType == SecurityType.Option) + .ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + OptionUniverseWriter.WriteUniverseFiles(symbolRef, optionTickHistories); } }