From 9259bc370ac66880379669c935d6632af3d1996f Mon Sep 17 00:00:00 2001 From: BuildTools Date: Thu, 16 Apr 2026 19:43:33 +0300 Subject: [PATCH] Added multi-threaded symbol loading, update bazel --- .bazelrc | 6 +- .gitignore | 19 +++++ BUILD | 2 + MODULE.bazel | 3 +- cpp/session.cpp | 37 ++++++++-- cpp/session_pool.cpp | 43 +++++++++++ cpp/symbols.cpp | 72 +++++++++++++++++++ demo/multi_threading_demo/.gitignore | 1 + demo/multi_threading_demo/README.md | 31 ++++++++ demo/multi_threading_demo/multi_threading.cpp | 48 +++++++++++++ demo/tsla_options_3w.cpp | 3 + hpp/benchmark.h | 1 + hpp/session.h | 5 +- hpp/session_pool.h | 36 ++++++++++ hpp/symbols.h | 26 +++++++ 15 files changed, 323 insertions(+), 10 deletions(-) create mode 100644 cpp/session_pool.cpp create mode 100644 cpp/symbols.cpp create mode 100644 demo/multi_threading_demo/.gitignore create mode 100644 demo/multi_threading_demo/README.md create mode 100644 demo/multi_threading_demo/multi_threading.cpp create mode 100644 hpp/session_pool.h create mode 100644 hpp/symbols.h diff --git a/.bazelrc b/.bazelrc index 037e0e1..68a8c0d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1 +1,5 @@ -build --cxxopt='-std=c++17' +# Use this if MSVC +# build --cxxopt=/std:c++20 + +build --cxxopt=-std=c++20 + diff --git a/.gitignore b/.gitignore index 0d4fed2..6b200f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,21 @@ +# Bazel bazel-* MODULE.bazel.lock + +# Build folders, libs, includes +include/ +lib/ +x64/ +yfinance-cpp/ + +# Visual Studio +.vs/ +*.sln +*.vcxproj +*.vcxproj.user +*.vcxproj.filters + +# Other extensions +*.dll +*.pdb +*.exp diff --git a/BUILD b/BUILD index 87754f4..5a8b4c1 100644 --- a/BUILD +++ b/BUILD @@ -1,3 +1,5 @@ +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_binary", "cc_test") + cc_library( name = "yfinance", srcs = glob(["cpp/*.cpp"]), diff --git a/MODULE.bazel b/MODULE.bazel index 3354631..a35141c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -3,7 +3,8 @@ module( version = "0.0.1", ) -bazel_dep(name = "nlohmann_json", version = "3.11.3") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "nlohmann_json", version = "3.12.0.bcr.1") bazel_dep(name = "cpr", version = "1.14.1") bazel_dep(name = "curl", version = "8.8.0") diff --git a/cpp/session.cpp b/cpp/session.cpp index f7d8613..54af15e 100644 --- a/cpp/session.cpp +++ b/cpp/session.cpp @@ -15,10 +15,23 @@ namespace yfinance { refreshCredentials(); } + // Create a copy constructor + Session::Session(const Session& session) { + m_user_agent = USER_AGENT; + m_crumb = session.m_crumb; + m_cookie = session.m_cookie; + } + void Session::refreshCredentials() { // 1. Get cookie from fc.yahoo.com m_session.SetUrl(cpr::Url{"https://fc.yahoo.com"}); - m_session.Get(); + cpr::Response r1 = m_session.Get(); + + // Save the cookie to std::string + for (const auto& cookie : r1.cookies) { + if (!m_cookie.empty()) m_cookie += "; "; + m_cookie += cookie.GetName() + "=" + cookie.GetValue(); + } // 2. Get crumb from getcrumb m_session.SetUrl(cpr::Url{"https://query2.finance.yahoo.com/v1/test/getcrumb"}); @@ -38,14 +51,24 @@ namespace yfinance { parameters.Add({"crumb", m_crumb}); } m_session.SetParameters(parameters); + + // Merge headers if needed, but for now just append/set + // Note: SetHeader replaces all headers. We might want to preserve User-Agent. + cpr::Header session_header; + session_header.insert({"User-Agent", m_user_agent}); + + // Add cookie via headers (cpr::Session::SetCookies did not work) + if (!m_cookie.empty()) { + session_header.insert({"Cookie", m_cookie}); + } + + // Merge other headers if (!headers.empty()) { - // Merge headers if needed, but for now just append/set - // Note: SetHeader replaces all headers. We might want to preserve User-Agent. - headers.insert({"User-Agent", m_user_agent}); - m_session.SetHeader(headers); - } else { - m_session.SetHeader(cpr::Header{{"User-Agent", m_user_agent}}); + session_header.merge(headers); } + + m_session.SetHeader(session_header); + return m_session.Get(); } diff --git a/cpp/session_pool.cpp b/cpp/session_pool.cpp new file mode 100644 index 0000000..130e2a0 --- /dev/null +++ b/cpp/session_pool.cpp @@ -0,0 +1,43 @@ +#include "../hpp/session_pool.h" + +namespace yfinance { + SessionPool::SessionPool(size_t size) : sem(size) { + // Make sure size is bigger than 0 + assert(size > 0); + + // Generate the first session object who will be copied + auto first_session = std::make_unique(); + + // Copy the first session into multiple sessions + for (size_t i = 0; i < (size - 1); i++) { + pool.emplace(std::make_unique(*first_session)); + } + + pool.push(std::move(first_session)); + } + + // Fetches a free session object, waits if no one is free + std::unique_ptr SessionPool::acquire() { + // Make sure there is an empty session + sem.acquire(); + + // Lock the mutex for queue + std::unique_lock lock(m_mutex); + + auto session = std::move(pool.front()); + pool.pop(); + + return session; + } + + void SessionPool::release(std::unique_ptr&& session) { + { + // Lock the mutex for queue + std::unique_lock lock(m_mutex); + + pool.push(std::move(session)); + } + + sem.release(); + } +} diff --git a/cpp/symbols.cpp b/cpp/symbols.cpp new file mode 100644 index 0000000..513bea1 --- /dev/null +++ b/cpp/symbols.cpp @@ -0,0 +1,72 @@ +#include "../hpp/symbols.h" + +#include "../hpp/session.h" +#include "../hpp/utils.h" + +#include +#include + +using json = nlohmann::json; + +namespace yfinance { + // Uses 8 threads maximum by default + Symbols::Symbols(const std::vector& symbols, int max_threads) + : sem{max_threads}, m_symbols(symbols), session_pool(max_threads) + {} + + std::vector Symbols::get_summaries(const std::string& module) { + + // Create an array to store the futures + std::vector> futures; + futures.reserve(m_symbols.size()); + + // Launch async tasks for API calls + for (const auto& symbol : m_symbols) { + futures.push_back( + std::async(std::launch::async, [this, &symbol, &module]() -> json { + // Make sure to cap the threads used with a semaphore + // So we won't end up with 300 threads for 300 API calls + sem.acquire(); + + json data{}; + + // Acquire a free session from session pool + auto session = session_pool.acquire(); + + cpr::Response r = session->Get(cpr::Url{ + Utils::Statics::Summary::v10 + symbol}, + cpr::Parameters{{"modules", module}}); + + if ((r.status_code == 200) && (!r.text.empty())) { + nlohmann::json quoteSummary = nlohmann::json::parse(r.text); + data = quoteSummary["quoteSummary"] + ["result"][0][module]; + } + else { + std::string error_message = + "Request failed with status code: " + std::to_string(r.status_code); + throw std::runtime_error(error_message); + } + + // Release the session + session_pool.release(std::move(session)); + // Release the semaphore + sem.release(); + + return data; + }) + ); + } + + // Collect results in order + std::vector data; + data.reserve(futures.size()); + + // Join the results + for (auto& fut : futures) { + data.push_back(fut.get()); + } + + return data; + } +} \ No newline at end of file diff --git a/demo/multi_threading_demo/.gitignore b/demo/multi_threading_demo/.gitignore new file mode 100644 index 0000000..f49fd16 --- /dev/null +++ b/demo/multi_threading_demo/.gitignore @@ -0,0 +1 @@ +x64/ \ No newline at end of file diff --git a/demo/multi_threading_demo/README.md b/demo/multi_threading_demo/README.md new file mode 100644 index 0000000..e1a963d --- /dev/null +++ b/demo/multi_threading_demo/README.md @@ -0,0 +1,31 @@ +# Multi Threaded Symbol Loading Demo +This demo compares multi thread approach to single threaded approach calling 10 symbols 10 times +Multi threaded approach takes slightly more time pre-initializing (which is NOT included in the benchmark) however, on multiple API calls, it can save more than 5x the time compared to the single threaded approach + +Currently only get_summary function is implemented + +## Results +This is the result that I have got on my machine + +### Multi Threaded +======================================================================= +=========================== SHOW TIMEIT RESULTS ======================= +======================================================================= + + Iterations completed : 10 + Total milliseconds :1574 + Average milliseconds :157 + Maxima milliseconds :320 + Minima milliseconds :102 + + +### Single Threaded +======================================================================= +=========================== SHOW TIMEIT RESULTS ======================= +======================================================================= + + Iterations completed : 10 + Total milliseconds :8327 + Average milliseconds :832 + Maxima milliseconds :1686 + Minima milliseconds :496 \ No newline at end of file diff --git a/demo/multi_threading_demo/multi_threading.cpp b/demo/multi_threading_demo/multi_threading.cpp new file mode 100644 index 0000000..adfbf74 --- /dev/null +++ b/demo/multi_threading_demo/multi_threading.cpp @@ -0,0 +1,48 @@ +#include "../../hpp/symbols.h" + +#include "../../hpp/benchmark.h" +#include "../../hpp/base.h" + +#include + +#include +#include +#include + +void benchmarkTestMT(yfinance::Symbols* symbols) { + auto data = symbols->get_summaries("price"); + + for (auto& quoteSummary : data) { + std::cout << quoteSummary.dump() << std::endl; + } +} + +void benchmarkTestST(const std::vector& symbol_names) { + for (auto& symbol_name : symbol_names) { + yfinance::Symbol symbol(symbol_name); + auto quoteSummary = symbol.get_summary("price"); + + std::cout << quoteSummary.dump() << std::endl; + } +} + +int main() { + // Load 10 different symbols + std::vector symbol_names = { + "NVDA", "TSLA", "HOOD", "PLTR", "BTC-USD", + "EURUSD=X", "^SPX", "^DJI", "^TYX", "SPY" + }; + + // Initialize Symbols object pre-API calls + yfinance::Symbols symbols(symbol_names); + + // Benchmark both multi thread and single thread version + auto f_mt = std::bind(&benchmarkTestMT, &symbols); + auto f_st = std::bind(&benchmarkTestST, symbol_names); + + auto mt_result = Benchmarking::Timeit(10, f_mt); + auto st_result = Benchmarking::Timeit(10, f_st); + + std::cout << mt_result << std::endl; + std::cout << st_result << std::endl; +} \ No newline at end of file diff --git a/demo/tsla_options_3w.cpp b/demo/tsla_options_3w.cpp index cb1b47f..5eef41e 100644 --- a/demo/tsla_options_3w.cpp +++ b/demo/tsla_options_3w.cpp @@ -1,3 +1,6 @@ +// Windows.h define macros for min/max which collides with std::min/std::max +#define NOMINMAX + #include #include #include diff --git a/hpp/benchmark.h b/hpp/benchmark.h index 14a64e5..7c62ea6 100644 --- a/hpp/benchmark.h +++ b/hpp/benchmark.h @@ -2,6 +2,7 @@ #include "structures.h" #include "utils.h" +#include namespace Benchmarking { template diff --git a/hpp/session.h b/hpp/session.h index fecaedb..87e8d63 100644 --- a/hpp/session.h +++ b/hpp/session.h @@ -6,6 +6,9 @@ namespace yfinance { class Session { public: + Session(); + Session(const Session& session); + // Save a static session object for single thread use static Session& getInstance(); cpr::Response Get(cpr::Url url, cpr::Parameters parameters = {}, cpr::Header headers = {}); @@ -14,9 +17,9 @@ namespace yfinance { const std::string& getCrumb() const { return m_crumb; } private: - Session(); void refreshCredentials(); + std::string m_cookie; std::string m_crumb; cpr::Session m_session; std::string m_user_agent; diff --git a/hpp/session_pool.h b/hpp/session_pool.h new file mode 100644 index 0000000..56a8f18 --- /dev/null +++ b/hpp/session_pool.h @@ -0,0 +1,36 @@ +#pragma once + +#include "session.h" + +#include +#include +#include + + +#ifndef YF_MAX_THREADS +// Defines the maximum amount of threads and session objects to be used +// 32 by default +#define YF_MAX_THREADS 32 +#endif + +namespace yfinance { + /* + A session pool that uses the same cookie/crumb for each instance + in order save API calls + */ + class SessionPool { + public: + SessionPool(size_t size); + ~SessionPool() = default; + + std::unique_ptr acquire(); + void release(std::unique_ptr&& session); + + private: + std::queue> pool; + std::mutex m_mutex; + std::counting_semaphore sem; + }; +} + + diff --git a/hpp/symbols.h b/hpp/symbols.h new file mode 100644 index 0000000..e1dcfa6 --- /dev/null +++ b/hpp/symbols.h @@ -0,0 +1,26 @@ +#pragma once + +#include "session_pool.h" + +#include +#include +#include +#include + +#include + +namespace yfinance { + class Symbols { + public: + // max_threads cannot be bigger than YF_MAX_THREADS + Symbols(const std::vector& symbols, int max_threads = 8); + ~Symbols() = default; + + std::vector get_summaries(const std::string& module); + private: + std::vector m_symbols; + std::counting_semaphore sem; + SessionPool session_pool; + + }; +}