Added multi-threaded symbol loading, update bazel

.bazelrc

@@ -1 +1,5 @@
-build --cxxopt='-std=c++17'
+# Use this if MSVC
+# build --cxxopt=/std:c++20
+
+build --cxxopt=-std=c++20
+

.gitignore

@@ -1,2 +1,21 @@
+# Bazel
 bazel-*
 MODULE.bazel.lock
+
+# Build folders, libs, includes
+include/
+lib/
+x64/
+yfinance-cpp/
+
+# Visual Studio
+.vs/
+*.sln
+*.vcxproj
+*.vcxproj.user
+*.vcxproj.filters
+
+# Other extensions
+*.dll
+*.pdb
+*.exp

BUILD

@@ -1,3 +1,5 @@
+load("@rules_cc//cc:defs.bzl", "cc_library", "cc_binary", "cc_test")
+
 cc_library(
     name = "yfinance",
     srcs = glob(["cpp/*.cpp"]),

MODULE.bazel

@@ -3,7 +3,8 @@ module(
     version = "0.0.1",
 )
 
-bazel_dep(name = "nlohmann_json", version = "3.11.3")
+bazel_dep(name = "rules_cc", version = "0.0.9")
+bazel_dep(name = "nlohmann_json", version = "3.12.0.bcr.1")
 bazel_dep(name = "cpr", version = "1.14.1")
 bazel_dep(name = "curl", version = "8.8.0")
 

cpp/session.cpp

@@ -15,10 +15,23 @@ namespace yfinance {
         refreshCredentials();
     }
 
+    // Create a copy constructor
+    Session::Session(const Session& session) {
+        m_user_agent = USER_AGENT;
+        m_crumb = session.m_crumb;
+        m_cookie = session.m_cookie;
+    }
+
     void Session::refreshCredentials() {
         // 1. Get cookie from fc.yahoo.com
         m_session.SetUrl(cpr::Url{"https://fc.yahoo.com"});
-        m_session.Get();
+        cpr::Response r1 = m_session.Get();
+
+        // Save the cookie to std::string
+        for (const auto& cookie : r1.cookies) {
+            if (!m_cookie.empty()) m_cookie += "; ";
+            m_cookie += cookie.GetName() + "=" + cookie.GetValue();
+        }
 
         // 2. Get crumb from getcrumb
         m_session.SetUrl(cpr::Url{"https://query2.finance.yahoo.com/v1/test/getcrumb"});
@@ -38,14 +51,24 @@ namespace yfinance {
             parameters.Add({"crumb", m_crumb});
         }
         m_session.SetParameters(parameters);
+
+        // Merge headers if needed, but for now just append/set
+        // Note: SetHeader replaces all headers. We might want to preserve User-Agent.
+        cpr::Header session_header;
+        session_header.insert({"User-Agent", m_user_agent});
+
+        // Add cookie via headers (cpr::Session::SetCookies did not work)
+        if (!m_cookie.empty()) {
+            session_header.insert({"Cookie", m_cookie});
+        }
+
+        // Merge other headers
         if (!headers.empty()) {
-            // Merge headers if needed, but for now just append/set
-            // Note: SetHeader replaces all headers. We might want to preserve User-Agent.
-            headers.insert({"User-Agent", m_user_agent});
-            m_session.SetHeader(headers);
-        } else {
-            m_session.SetHeader(cpr::Header{{"User-Agent", m_user_agent}});
+            session_header.merge(headers);
         }
+
+        m_session.SetHeader(session_header);
+
         return m_session.Get();
     }
 

cpp/session_pool.cpp

@@ -0,0 +1,43 @@
+#include "../hpp/session_pool.h"
+
+namespace yfinance {
+	SessionPool::SessionPool(size_t size) : sem(size) {
+		// Make sure size is bigger than 0
+		assert(size > 0);
+
+		// Generate the first session object who will be copied
+		auto first_session = std::make_unique<Session>();
+
+		// Copy the first session into multiple sessions
+		for (size_t i = 0; i < (size - 1); i++) {
+			pool.emplace(std::make_unique<Session>(*first_session));
+		}
+
+		pool.push(std::move(first_session));
+	}
+
+	// Fetches a free session object, waits if no one is free
+	std::unique_ptr<Session> SessionPool::acquire() {
+		// Make sure there is an empty session
+		sem.acquire();
+
+		// Lock the mutex for queue
+		std::unique_lock<std::mutex> lock(m_mutex);
+
+		auto session = std::move(pool.front());
+		pool.pop();
+
+		return session;
+	}
+
+	void SessionPool::release(std::unique_ptr<Session>&& session) {
+		{
+			// Lock the mutex for queue
+			std::unique_lock<std::mutex> lock(m_mutex);
+
+			pool.push(std::move(session));
+		}
+
+		sem.release();
+	}
+}

cpp/symbols.cpp

@@ -0,0 +1,72 @@
+#include "../hpp/symbols.h"
+
+#include "../hpp/session.h"
+#include "../hpp/utils.h"
+
+#include <cpr/cpr.h>
+#include <future>
+
+using json = nlohmann::json;
+
+namespace yfinance {
+	// Uses 8 threads maximum by default
+	Symbols::Symbols(const std::vector<std::string>& symbols, int max_threads) 
+		: sem{max_threads}, m_symbols(symbols), session_pool(max_threads)
+	{}
+
+	std::vector<nlohmann::json> Symbols::get_summaries(const std::string& module) {
+
+		// Create an array to store the futures
+		std::vector<std::future<json>> futures;
+		futures.reserve(m_symbols.size());
+
+		// Launch async tasks for API calls
+		for (const auto& symbol : m_symbols) {
+			futures.push_back(
+				std::async(std::launch::async, [this, &symbol, &module]() -> json {
+					// Make sure to cap the threads used with a semaphore
+					// So we won't end up with 300 threads for 300 API calls
+					sem.acquire();
+
+					json data{};
+
+					// Acquire a free session from session pool
+					auto session = session_pool.acquire();
+
+					cpr::Response r = session->Get(cpr::Url{
+							Utils::Statics::Summary::v10 + symbol},
+							cpr::Parameters{{"modules", module}});
+
+					if ((r.status_code == 200) && (!r.text.empty())) {
+						nlohmann::json quoteSummary = nlohmann::json::parse(r.text);
+						data = quoteSummary["quoteSummary"]
+							["result"][0][module];
+					}
+					else {
+						std::string error_message =
+							"Request failed with status code: " + std::to_string(r.status_code);
+						throw std::runtime_error(error_message);
+					}
+
+					// Release the session
+					session_pool.release(std::move(session));
+					// Release the semaphore
+					sem.release();
+
+					return data;
+				})
+			);
+		}
+
+		// Collect results in order
+		std::vector<json> data;
+		data.reserve(futures.size());
+
+		// Join the results
+		for (auto& fut : futures) {
+			data.push_back(fut.get());
+		}
+
+		return data;
+	}
+}

demo/multi_threading_demo/.gitignore

@@ -0,0 +1 @@
+x64/

demo/multi_threading_demo/README.md

@@ -0,0 +1,31 @@
+# Multi Threaded Symbol Loading Demo
+This demo compares multi thread approach to single threaded approach calling 10 symbols 10 times
+Multi threaded approach takes slightly more time pre-initializing (which is NOT included in the benchmark) however, on multiple API calls, it can save more than 5x the time compared to the single threaded approach
+
+Currently only get_summary function is implemented
+
+## Results
+This is the result that I have got on my machine
+
+### Multi Threaded
+=======================================================================
+=========================== SHOW TIMEIT RESULTS =======================
+=======================================================================
+
+        Iterations completed    : 10
+        Total milliseconds      :1574
+        Average milliseconds    :157
+        Maxima milliseconds     :320
+        Minima milliseconds     :102
+
+
+### Single Threaded
+=======================================================================
+=========================== SHOW TIMEIT RESULTS =======================
+=======================================================================
+
+        Iterations completed    : 10
+        Total milliseconds      :8327
+        Average milliseconds    :832
+        Maxima milliseconds     :1686
+        Minima milliseconds     :496

demo/multi_threading_demo/multi_threading.cpp

@@ -0,0 +1,48 @@
+#include "../../hpp/symbols.h"
+
+#include "../../hpp/benchmark.h"
+#include "../../hpp/base.h"
+
+#include <nlohmann/json.hpp>
+
+#include <iostream>
+#include <vector>
+#include <string>
+
+void benchmarkTestMT(yfinance::Symbols* symbols) {
+	auto data = symbols->get_summaries("price");
+
+	for (auto& quoteSummary : data) {
+		std::cout << quoteSummary.dump() << std::endl;
+	}
+}
+
+void benchmarkTestST(const std::vector<std::string>& symbol_names) {
+	for (auto& symbol_name : symbol_names) {
+		yfinance::Symbol symbol(symbol_name);
+		auto quoteSummary = symbol.get_summary("price");
+
+		std::cout << quoteSummary.dump() << std::endl;
+	}
+}
+
+int main() {
+	// Load 10 different symbols
+	std::vector<std::string> symbol_names = {
+		"NVDA", "TSLA", "HOOD", "PLTR", "BTC-USD",
+		"EURUSD=X", "^SPX", "^DJI", "^TYX", "SPY"
+	};
+
+	// Initialize Symbols object pre-API calls
+	yfinance::Symbols symbols(symbol_names);
+
+	// Benchmark both multi thread and single thread version
+	auto f_mt = std::bind(&benchmarkTestMT, &symbols);
+	auto f_st = std::bind(&benchmarkTestST, symbol_names);
+
+	auto mt_result = Benchmarking::Timeit(10, f_mt);
+	auto st_result = Benchmarking::Timeit(10, f_st);
+
+	std::cout << mt_result << std::endl;
+	std::cout << st_result << std::endl;
+}

demo/tsla_options_3w.cpp

@@ -1,3 +1,6 @@
+// Windows.h define macros for min/max which collides with std::min/std::max
+#define NOMINMAX
+
 #include <iostream>
 #include <ctime>
 #include <iomanip>

hpp/benchmark.h

@@ -2,6 +2,7 @@
 #include "structures.h"
 #include "utils.h"
 
+#include <utility>
 
 namespace Benchmarking {
     template<typename Function, typename ... Args>

hpp/session.h

@@ -6,6 +6,9 @@ namespace yfinance {
 
     class Session {
     public:
+        Session();
+        Session(const Session& session);
+        // Save a static session object for single thread use
         static Session& getInstance();
 
         cpr::Response Get(cpr::Url url, cpr::Parameters parameters = {}, cpr::Header headers = {});
@@ -14,9 +17,9 @@ namespace yfinance {
         const std::string& getCrumb() const { return m_crumb; }
 
     private:
-        Session();
         void refreshCredentials();
 
+        std::string m_cookie;
         std::string m_crumb;
         cpr::Session m_session;
         std::string m_user_agent;

hpp/session_pool.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include "session.h"
+
+#include <queue>
+#include <memory>
+#include <semaphore>
+
+
+#ifndef YF_MAX_THREADS
+// Defines the maximum amount of threads and session objects to be used
+// 32 by default
+#define YF_MAX_THREADS 32
+#endif
+
+namespace yfinance {
+	/*
+	A session pool that uses the same cookie/crumb for each instance
+	in order save API calls
+	*/ 
+	class SessionPool {
+	public:
+		SessionPool(size_t size);
+		~SessionPool() = default;
+
+		std::unique_ptr<Session> acquire();
+		void release(std::unique_ptr<Session>&& session);
+
+	private:
+		std::queue<std::unique_ptr<Session>> pool;
+		std::mutex m_mutex;
+		std::counting_semaphore<YF_MAX_THREADS> sem;
+	};
+}
+
+