From f9fa458db0485942b37f5fa30b8fab5507214013 Mon Sep 17 00:00:00 2001 From: MarketDataApp Date: Thu, 2 Jul 2026 13:30:28 -0300 Subject: [PATCH 1/3] Security hardening: Tier 1 fixes from SECURITY.md policy sweep - Percent-encode caller-supplied symbols in request paths (path traversal / query smuggling via untrusted input); valid symbols unaffected - Neutralize dot-segments in options.lookup() path while keeping valid slashes (dates) intact - Token obfuscation no longer reveals token length or any characters of short tokens - Bound API error messages extracted from response bodies (max 500 chars) - Malformed rate-limit headers no longer crash a successful request - Pin GitHub Actions to full commit SHAs; test workflow token read-only --- .github/workflows/publish.yml | 16 ++--- .github/workflows/test.yml | 11 +-- CHANGELOG.md | 11 +++ src/marketdata/client.py | 37 ++++++---- src/marketdata/resources/funds/candles.py | 4 +- src/marketdata/resources/options/chain.py | 3 +- .../resources/options/expirations.py | 3 +- src/marketdata/resources/options/lookup.py | 4 +- src/marketdata/resources/options/quotes.py | 4 +- src/marketdata/resources/options/strikes.py | 3 +- src/marketdata/resources/stocks/candles.py | 3 +- src/marketdata/resources/stocks/earnings.py | 3 +- src/marketdata/resources/stocks/news.py | 4 +- src/marketdata/retry.py | 4 +- src/marketdata/utils.py | 28 +++++++- src/tests/test_api_status.py | 9 +-- src/tests/test_client.py | 63 +++++++++++++++-- src/tests/test_options_chain.py | 4 +- src/tests/test_options_expirations.py | 35 +++------- src/tests/test_path_injection.py | 68 +++++++++++++++++++ src/tests/test_retry.py | 12 ++-- src/tests/test_utils.py | 32 ++++++++- 22 files changed, 272 insertions(+), 89 deletions(-) create mode 100644 src/tests/test_path_injection.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 33e7180..57ca2e6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,15 +16,15 @@ jobs: id-token: write steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Install uv - uses: astral-sh/setup-uv@v8.2.0 + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: version: "latest" - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.12" @@ -32,7 +32,7 @@ jobs: run: uv build - name: Publish to TestPyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 with: repository-url: https://test.pypi.org/legacy/ @@ -48,15 +48,15 @@ jobs: if: github.event_name == 'release' steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Install uv - uses: astral-sh/setup-uv@v8.2.0 + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: version: "latest" - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.12" @@ -64,4 +64,4 @@ jobs: run: uv build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 20c9eed..e7ed584 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,9 @@ on: pull_request: branches: [ main, master, develop ] +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest @@ -14,15 +17,15 @@ jobs: python-version: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Install uv - uses: astral-sh/setup-uv@v8.2.0 + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: version: "latest" - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ matrix.python-version }} @@ -45,7 +48,7 @@ jobs: ls -lh coverage.xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v7 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 55adc4b..70ba6ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Security + +- Caller-supplied symbols are now percent-encoded in request paths, preventing path traversal and query smuggling via untrusted input; valid symbols are unaffected +- `options.lookup()` neutralizes dot-segments in the lookup string so it cannot traverse to a different endpoint; valid lookup strings (including dates with slashes) are unaffected +- Token obfuscation in logs no longer reveals the token length, and never reveals any characters of short tokens +- API error messages extracted from response bodies are now bounded, so a malformed or hostile response cannot balloon exception messages and logs +- Malformed rate-limit headers no longer crash a successful request with a raw `KeyError`/`ValueError`; the SDK logs a warning and keeps the previous limits +- GitHub Actions are pinned to full commit SHAs in CI and publish workflows; the test workflow token is now read-only + ## [1.3.0] - 2026-06-10 ### Fixed diff --git a/src/marketdata/client.py b/src/marketdata/client.py index 4e7334e..e90efca 100644 --- a/src/marketdata/client.py +++ b/src/marketdata/client.py @@ -19,7 +19,7 @@ from marketdata.resources.stocks import StocksResource from marketdata.settings import settings from marketdata.types import UserRateLimits -from marketdata.utils import format_duration_log, obfuscate_token +from marketdata.utils import format_duration_log, obfuscate_token, resume_long_text class MarketDataClient: @@ -100,11 +100,14 @@ def _validate_response_status_code( raise_for_status: bool, ) -> None: def _get_response_errmsg(response: Response): + # Bound the error message so a malformed or hostile response body + # cannot balloon exception messages and log output. try: data = response.json() - return data["errmsg"] - except: - return response.text + errmsg = data["errmsg"] + except Exception: + errmsg = response.text + return resume_long_text(str(errmsg), max_length=500) def _validate_status(response: Response): try: @@ -170,14 +173,22 @@ def _setup_rate_limits(self): response_log_level=DEBUG, ) - def _extract_rate_limits(self, response: Response) -> UserRateLimits: + def _extract_rate_limits(self, response: Response) -> UserRateLimits | None: self.logger.debug(f"Extracting response rate limits from response headers") - return UserRateLimits( - requests_limit=int(response.headers["x-api-ratelimit-limit"]), - requests_remaining=int(response.headers["x-api-ratelimit-remaining"]), - requests_reset=int(response.headers["x-api-ratelimit-reset"]), - requests_consumed=int(response.headers["x-api-ratelimit-consumed"]), - ) + try: + return UserRateLimits( + requests_limit=int(response.headers["x-api-ratelimit-limit"]), + requests_remaining=int(response.headers["x-api-ratelimit-remaining"]), + requests_reset=int(response.headers["x-api-ratelimit-reset"]), + requests_consumed=int(response.headers["x-api-ratelimit-consumed"]), + ) + except (KeyError, ValueError) as e: + # Malformed response (e.g. missing or non-numeric rate-limit + # headers) must not crash the request that already succeeded. + self.logger.warning( + f"Could not extract rate limits from response headers: {e!r}" + ) + return None def _pre_request_logs(self, method: str, url: str, **kwargs): self.logger.debug(f"Making request to URL: {self.base_url}/{url}") @@ -221,6 +232,8 @@ def _make_request( ) if populate_rate_limits: - self.rate_limits = self._extract_rate_limits(response) + rate_limits = self._extract_rate_limits(response) + if rate_limits is not None: + self.rate_limits = rate_limits return response diff --git a/src/marketdata/resources/funds/candles.py b/src/marketdata/resources/funds/candles.py index 85b3f3b..9d21e23 100644 --- a/src/marketdata/resources/funds/candles.py +++ b/src/marketdata/resources/funds/candles.py @@ -9,7 +9,7 @@ from marketdata.params import universal_params from marketdata.resources.base import BaseResource from marketdata.sdk_error import MarketDataClientErrorResult, handle_exceptions -from marketdata.utils import get_data_records +from marketdata.utils import encode_path_segment, get_data_records @handle_exceptions @@ -32,7 +32,7 @@ def candles( ) url = self._build_url( - path=f"funds/candles/{input_params.resolution}/{symbol}/", + path=f"funds/candles/{encode_path_segment(input_params.resolution)}/{encode_path_segment(symbol)}/", user_universal_params=user_universal_params, input_params=input_params, extra_params=kwargs, diff --git a/src/marketdata/resources/options/chain.py b/src/marketdata/resources/options/chain.py index 8530cac..4424eeb 100644 --- a/src/marketdata/resources/options/chain.py +++ b/src/marketdata/resources/options/chain.py @@ -12,6 +12,7 @@ from marketdata.params import universal_params from marketdata.resources.base import BaseResource from marketdata.sdk_error import MarketDataClientErrorResult, handle_exceptions +from marketdata.utils import encode_path_segment @handle_exceptions @@ -36,7 +37,7 @@ def chain( ) url = self._build_url( - path=f"options/chain/{symbol}/", + path=f"options/chain/{encode_path_segment(symbol)}/", user_universal_params=user_universal_params, input_params=input_params, extra_params=kwargs, diff --git a/src/marketdata/resources/options/expirations.py b/src/marketdata/resources/options/expirations.py index 362ba4f..3313066 100644 --- a/src/marketdata/resources/options/expirations.py +++ b/src/marketdata/resources/options/expirations.py @@ -12,6 +12,7 @@ from marketdata.params import universal_params from marketdata.resources.base import BaseResource from marketdata.sdk_error import MarketDataClientErrorResult, handle_exceptions +from marketdata.utils import encode_path_segment @handle_exceptions @@ -45,7 +46,7 @@ def expirations( user_universal_params.date_format = None url = self._build_url( - path=f"options/expirations/{symbol}/", + path=f"options/expirations/{encode_path_segment(symbol)}/", user_universal_params=user_universal_params, input_params=input_params, extra_params=kwargs, diff --git a/src/marketdata/resources/options/lookup.py b/src/marketdata/resources/options/lookup.py index 86abb16..44b5a26 100644 --- a/src/marketdata/resources/options/lookup.py +++ b/src/marketdata/resources/options/lookup.py @@ -1,5 +1,4 @@ from typing import Annotated, Any -from urllib.parse import quote from marketdata.api_error import api_error_handler from marketdata.docs import docs @@ -13,6 +12,7 @@ from marketdata.params import universal_params from marketdata.resources.base import BaseResource from marketdata.sdk_error import MarketDataClientErrorResult, handle_exceptions +from marketdata.utils import encode_path @handle_exceptions @@ -44,7 +44,7 @@ def lookup( # All params are already in the path _format_date = lambda date: date.strftime("%d-%M-%Y") excluded_params = OptionsLookupInput.model_fields.keys() - lookup_quote = quote(input_params.lookup) + lookup_quote = encode_path(input_params.lookup) url = self._build_url( path=f"options/lookup/{lookup_quote}/", diff --git a/src/marketdata/resources/options/quotes.py b/src/marketdata/resources/options/quotes.py index f12ceca..0774fb3 100644 --- a/src/marketdata/resources/options/quotes.py +++ b/src/marketdata/resources/options/quotes.py @@ -18,7 +18,7 @@ from marketdata.params import universal_params from marketdata.resources.base import BaseResource from marketdata.sdk_error import MarketDataClientErrorResult, handle_exceptions -from marketdata.utils import merge_csv_texts +from marketdata.utils import encode_path_segment, merge_csv_texts @handle_exceptions @@ -50,7 +50,7 @@ def quotes( def _get_response(symbol: str) -> Response: url = self._build_url( - path=f"options/quotes/{symbol}/", + path=f"options/quotes/{encode_path_segment(symbol)}/", user_universal_params=user_universal_params, input_params=input_params, extra_params=kwargs, diff --git a/src/marketdata/resources/options/strikes.py b/src/marketdata/resources/options/strikes.py index 67c5de1..4298147 100644 --- a/src/marketdata/resources/options/strikes.py +++ b/src/marketdata/resources/options/strikes.py @@ -12,6 +12,7 @@ from marketdata.params import universal_params from marketdata.resources.base import BaseResource from marketdata.sdk_error import MarketDataClientErrorResult, handle_exceptions +from marketdata.utils import encode_path_segment @handle_exceptions @@ -40,7 +41,7 @@ def strikes( ) url = self._build_url( - path=f"options/strikes/{symbol}/", + path=f"options/strikes/{encode_path_segment(symbol)}/", user_universal_params=user_universal_params, input_params=input_params, extra_params=kwargs, diff --git a/src/marketdata/resources/stocks/candles.py b/src/marketdata/resources/stocks/candles.py index 107e35b..8a80438 100644 --- a/src/marketdata/resources/stocks/candles.py +++ b/src/marketdata/resources/stocks/candles.py @@ -20,6 +20,7 @@ from marketdata.resources.base import BaseResource from marketdata.sdk_error import MarketDataClientErrorResult, handle_exceptions from marketdata.utils import ( + encode_path_segment, get_data_records, merge_csv_texts, split_dates_by_timeframe, @@ -68,7 +69,7 @@ def _get_response( input_params.to_date = to_date url = self._build_url( - path=f"stocks/candles/{input_params.resolution}/{symbol}/", + path=f"stocks/candles/{encode_path_segment(input_params.resolution)}/{encode_path_segment(symbol)}/", user_universal_params=user_universal_params, input_params=input_params, extra_params=kwargs, diff --git a/src/marketdata/resources/stocks/earnings.py b/src/marketdata/resources/stocks/earnings.py index 60b53f3..3686ff7 100644 --- a/src/marketdata/resources/stocks/earnings.py +++ b/src/marketdata/resources/stocks/earnings.py @@ -12,6 +12,7 @@ from marketdata.params import universal_params from marketdata.resources.base import BaseResource from marketdata.sdk_error import MarketDataClientErrorResult, handle_exceptions +from marketdata.utils import encode_path_segment @handle_exceptions @@ -40,7 +41,7 @@ def earnings( ) url = self._build_url( - path=f"stocks/earnings/{symbol}/", + path=f"stocks/earnings/{encode_path_segment(symbol)}/", user_universal_params=user_universal_params, input_params=input_params, extra_params=kwargs, diff --git a/src/marketdata/resources/stocks/news.py b/src/marketdata/resources/stocks/news.py index 0167f01..5475d29 100644 --- a/src/marketdata/resources/stocks/news.py +++ b/src/marketdata/resources/stocks/news.py @@ -9,7 +9,7 @@ from marketdata.params import universal_params from marketdata.resources.base import BaseResource from marketdata.sdk_error import MarketDataClientErrorResult, handle_exceptions -from marketdata.utils import get_data_records +from marketdata.utils import encode_path_segment, get_data_records @handle_exceptions @@ -38,7 +38,7 @@ def news( ) url = self._build_url( - path=f"stocks/news/{symbol}/", + path=f"stocks/news/{encode_path_segment(symbol)}/", user_universal_params=user_universal_params, input_params=input_params, extra_params=kwargs, diff --git a/src/marketdata/retry.py b/src/marketdata/retry.py index 316b991..33404f1 100644 --- a/src/marketdata/retry.py +++ b/src/marketdata/retry.py @@ -50,9 +50,7 @@ def _compute_wait(retry_state) -> float: if exc is not None: response = getattr(exc, "response", None) if response is not None: - retry_after = parse_retry_after( - response.headers.get("Retry-After") - ) + retry_after = parse_retry_after(response.headers.get("Retry-After")) if retry_after is not None: return retry_after return initial_delay * 2 ** (retry_state.attempt_number - 1) diff --git a/src/marketdata/utils.py b/src/marketdata/utils.py index a71eb90..89e9a14 100644 --- a/src/marketdata/utils.py +++ b/src/marketdata/utils.py @@ -3,6 +3,7 @@ from enum import Enum from io import StringIO from typing import Any +from urllib.parse import quote import pytz @@ -143,8 +144,31 @@ def format_duration_log(duration_ms: float) -> str: def obfuscate_token(token: str) -> str: + # Fixed-width mask so the log output never reveals the token length, + # and the last 4 chars are only shown when they are a small fraction of the token. if not isinstance(token, str): return str(token) - if len(token) <= 4: + if len(token) <= 8: return "****" - return "*" * (len(token) - 4) + token[-4:] + return "****" + token[-4:] + + +def encode_path_segment(value: Any) -> str: + # Percent-encode a single URL path segment so caller-supplied input + # (e.g. a symbol) cannot smuggle extra path segments ("AAPL/../../user"), + # query params or fragments into the request. Valid symbols + # (alphanumerics, ".", "-", "_") are unaffected. + return quote(str(value), safe="") + + +def encode_path(value: str) -> str: + # Percent-encode a multi-segment URL path (e.g. an options lookup string, + # where "/" is valid inside dates). Literal slashes are kept, but + # dot-segments ("." / "..") are neutralized so caller-supplied input + # cannot traverse to a different endpoint. + quoted = quote(str(value)) + segments = [ + s.replace(".", "%2E") if s and s.strip(".") == "" else s + for s in quoted.split("/") + ] + return "/".join(segments) diff --git a/src/tests/test_api_status.py b/src/tests/test_api_status.py index cfd1095..30013b4 100644 --- a/src/tests/test_api_status.py +++ b/src/tests/test_api_status.py @@ -35,10 +35,7 @@ def test_api_status_data(load_json, respx_mock, client): API_STATUS_DATA.refresh(client) for service in ALL_SERVICES: - assert ( - API_STATUS_DATA.get_api_status(client, service) - == APIStatusResult.ONLINE - ) + assert API_STATUS_DATA.get_api_status(client, service) == APIStatusResult.ONLINE def test_api_status_data_offline(load_json, respx_mock, client): @@ -52,8 +49,7 @@ def test_api_status_data_offline(load_json, respx_mock, client): API_STATUS_DATA.refresh(client) for service in ALL_SERVICES: assert ( - API_STATUS_DATA.get_api_status(client, service) - == APIStatusResult.OFFLINE + API_STATUS_DATA.get_api_status(client, service) == APIStatusResult.OFFLINE ) @@ -180,6 +176,7 @@ def fake_thread(*args, **kwargs): return m import marketdata.api_status as mod + original_thread = mod.threading.Thread mod.threading.Thread = fake_thread try: diff --git a/src/tests/test_client.py b/src/tests/test_client.py index 30785e7..d4f3b21 100644 --- a/src/tests/test_client.py +++ b/src/tests/test_client.py @@ -76,9 +76,7 @@ def test_client_make_request_retry(client, respx_mock, monkeypatch): prices_calls = [ c for c in respx_mock.calls if c.request.url.path == "/v1/stocks/prices/" ] - status_calls = [ - c for c in respx_mock.calls if c.request.url.path == "/status/" - ] + status_calls = [c for c in respx_mock.calls if c.request.url.path == "/status/"] assert len(prices_calls) == 4 assert len(status_calls) == 1 assert respx_mock.calls.call_count == 6 @@ -403,7 +401,9 @@ def test_client_max_retries_one(respx_mock, monkeypatch): result = c.stocks.prices(symbols="AAPL") assert isinstance(result, MarketDataClientErrorResult) - prices_calls = [c for c in respx_mock.calls if c.request.url.path == "/v1/stocks/prices/"] + prices_calls = [ + c for c in respx_mock.calls if c.request.url.path == "/v1/stocks/prices/" + ] assert len(prices_calls) == 2 @@ -455,3 +455,58 @@ def test_client_init_base_url_and_api_version_logged_at_debug(respx_mock): # They must still be available, just at DEBUG. assert any("Base URL" in m for m in debug_messages) assert any("API Version" in m for m in debug_messages) + + +def test_response_errmsg_is_bounded(client): + # A hostile/malformed response body must not balloon exception messages + request = Request("GET", "https://api.marketdata.app/v1/stocks/quotes/AAPL/") + response = Response(502, text="x" * 100_000, request=request) + + with pytest.raises(RequestError) as exc_info: + client._validate_response_status_code( + response, retry_status_codes=lambda x: x > 500, raise_for_status=True + ) + assert len(str(exc_info.value)) < 1_000 + + +def test_extract_rate_limits_missing_headers_returns_none(client, caplog): + request = Request("GET", "https://api.marketdata.app/v1/stocks/quotes/AAPL/") + response = Response(200, json={}, request=request) + + # Bypass the conftest instance-level patch to test the real method + result = MarketDataClient._extract_rate_limits(client, response) + assert result is None + + +def test_extract_rate_limits_garbage_headers_returns_none(client): + request = Request("GET", "https://api.marketdata.app/v1/stocks/quotes/AAPL/") + response = Response( + 200, + json={}, + headers={ + "x-api-ratelimit-limit": "not-a-number", + "x-api-ratelimit-remaining": "99", + "x-api-ratelimit-reset": "60", + "x-api-ratelimit-consumed": "1", + }, + request=request, + ) + + result = MarketDataClient._extract_rate_limits(client, response) + assert result is None + + +def test_make_request_keeps_rate_limits_on_malformed_response(client, respx_mock): + # A malformed response (no rate-limit headers) must not crash the request + # nor clobber previously known rate limits + respx_mock.get("https://api.marketdata.app/v1/markets/status/").respond( + json={}, status_code=200 + ) + previous = client.rate_limits + # Remove the conftest instance-level patch so the real extraction runs + if "_extract_rate_limits" in client.__dict__: + del client.__dict__["_extract_rate_limits"] + + response = client._make_request(method="GET", url="markets/status/") + assert response.status_code == 200 + assert client.rate_limits == previous diff --git a/src/tests/test_options_chain.py b/src/tests/test_options_chain.py index 8a680d5..2995933 100644 --- a/src/tests/test_options_chain.py +++ b/src/tests/test_options_chain.py @@ -328,9 +328,7 @@ def test_options_chain_strike_limit_is_int_on_wire(load_json, respx_mock, client json=mock_data, status_code=200 ) - client.options.chain( - "AAPL", strike_limit=10, output_format=OutputFormat.INTERNAL - ) + client.options.chain("AAPL", strike_limit=10, output_format=OutputFormat.INTERNAL) params = respx_mock.calls.last.request.url.params assert params.get("strikeLimit") == "10" diff --git a/src/tests/test_options_expirations.py b/src/tests/test_options_expirations.py index 83cfc50..82a5f83 100644 --- a/src/tests/test_options_expirations.py +++ b/src/tests/test_options_expirations.py @@ -18,11 +18,7 @@ def test_options_expirations_str(): - timestamp = int( - datetime.datetime( - 2025, 1, 1, 0, 0, 0, 0, ET - ).timestamp() - ) + timestamp = int(datetime.datetime(2025, 1, 1, 0, 0, 0, 0, ET).timestamp()) instance = OptionsExpirations( s="ok", @@ -34,11 +30,7 @@ def test_options_expirations_str(): def test_options_expirations_human_readable_str(): - timestamp = int( - datetime.datetime( - 2025, 1, 1, 0, 0, 0, 0, ET - ).timestamp() - ) + timestamp = int(datetime.datetime(2025, 1, 1, 0, 0, 0, 0, ET).timestamp()) instance = OptionsExpirationsHumanReadable( Expirations=[timestamp], Date=timestamp, @@ -64,9 +56,7 @@ def test_get_options_expirations_response_200_internal(load_json, respx_mock, cl 1764910800, tz=ET ) assert expirations.expirations[0].date() == datetime.date(2025, 12, 5) - assert expirations.updated == datetime.datetime.fromtimestamp( - 1764941963, tz=ET - ) + assert expirations.updated == datetime.datetime.fromtimestamp(1764941963, tz=ET) def test_get_options_expirations_response_200_json(load_json, respx_mock, client): @@ -97,9 +87,7 @@ def test_get_options_expirations_human_response_200(load_json, respx_mock, clien 1765515600, tz=ET ) assert expirations.Expirations[0].date() == datetime.date(2025, 12, 12) - assert expirations.Date == datetime.datetime.fromtimestamp( - 1765561297, tz=ET - ) + assert expirations.Date == datetime.datetime.fromtimestamp(1765561297, tz=ET) def test_get_options_expirations_response_200_dataframe_pandas( @@ -205,9 +193,7 @@ def test_options_expirations_optional_updated(): assert isinstance(str(instance), str) -def test_get_options_expirations_columns_filter_dataframe_pandas( - respx_mock, client -): +def test_get_options_expirations_columns_filter_dataframe_pandas(respx_mock, client): """Issue #23: requesting `columns=["expirations"]` makes the API return only that column. The result must NOT be an empty DataFrame with the data silently moved into the index. @@ -247,9 +233,7 @@ def test_get_options_expirations_columns_filter_dataframe_pandas( pd.testing.assert_frame_equal(df, expected_df) -def test_get_options_expirations_columns_filter_dataframe_polars( - respx_mock, client -): +def test_get_options_expirations_columns_filter_dataframe_polars(respx_mock, client): """Issue #23 (regression guard for polars): filtering by a single column must keep the data accessible as a column. """ @@ -278,8 +262,7 @@ def test_get_options_expirations_columns_filter_dataframe_polars( # A single "expirations" column holding the timestamps converted to # US/Eastern datetimes, with nothing dropped. expected_expirations = [ - datetime.datetime.fromtimestamp(ts, tz=ET) - for ts in expiration_timestamps + datetime.datetime.fromtimestamp(ts, tz=ET) for ts in expiration_timestamps ] assert df.columns == ["expirations"] assert df["expirations"].to_list() == expected_expirations @@ -294,9 +277,7 @@ def test_get_options_expirations_partial_response_internal(respx_mock, client): "s": "ok", "expirations": expiration_timestamps, } - respx_mock.get( - "https://api.marketdata.app/v1/options/expirations/AAPL/" - ).respond( + respx_mock.get("https://api.marketdata.app/v1/options/expirations/AAPL/").respond( json=partial_data, status_code=200, ) diff --git a/src/tests/test_path_injection.py b/src/tests/test_path_injection.py new file mode 100644 index 0000000..4e7d9f9 --- /dev/null +++ b/src/tests/test_path_injection.py @@ -0,0 +1,68 @@ +"""Caller-supplied input must not be able to smuggle path segments, query +params or fragments into outbound request URLs (see SECURITY.md).""" + +from marketdata.input_types.base import OutputFormat + + +def test_symbol_path_traversal_is_encoded(load_json, respx_mock, client): + mock_data = load_json("stocks_earnings_response_200") + route = respx_mock.get(url__regex=r".*earnings.*").respond( + json=mock_data, status_code=200 + ) + + client.stocks.earnings(symbol="AAPL/../../user", output_format=OutputFormat.JSON) + + raw_path = route.calls.last.request.url.raw_path.decode() + assert raw_path.startswith("/v1/stocks/earnings/AAPL%2F..%2F..%2Fuser/") + assert "/../" not in raw_path + + +def test_symbol_query_smuggling_is_encoded(load_json, respx_mock, client): + mock_data = load_json("stocks_earnings_response_200") + route = respx_mock.get(url__regex=r".*earnings.*").respond( + json=mock_data, status_code=200 + ) + + client.stocks.earnings(symbol="AAPL?injected=1", output_format=OutputFormat.JSON) + + raw_path = route.calls.last.request.url.raw_path.decode() + assert raw_path.startswith("/v1/stocks/earnings/AAPL%3Finjected%3D1/") + assert "injected=1" not in str(route.calls.last.request.url.params) + + +def test_valid_symbol_is_unchanged(load_json, respx_mock, client): + mock_data = load_json("stocks_earnings_response_200") + route = respx_mock.get( + "https://api.marketdata.app/v1/stocks/earnings/BRK.B/" + ).respond(json=mock_data, status_code=200) + + client.stocks.earnings(symbol="BRK.B", output_format=OutputFormat.JSON) + + assert route.called + + +def test_lookup_dot_segments_are_encoded(load_json, respx_mock, client): + mock_data = load_json("options_lookup_response_200") + route = respx_mock.get(url__regex=r".*lookup.*").respond( + json=mock_data, status_code=200 + ) + + client.options.lookup(lookup="AAPL/../../user", output_format=OutputFormat.JSON) + + raw_path = route.calls.last.request.url.raw_path.decode() + assert "/../" not in raw_path + assert "/%2E%2E/" in raw_path + + +def test_lookup_valid_string_keeps_slashes(load_json, respx_mock, client): + mock_data = load_json("options_lookup_response_200") + route = respx_mock.get(url__regex=r".*lookup.*").respond( + json=mock_data, status_code=200 + ) + + client.options.lookup( + lookup="AAPL 7/28/2023 200 Call", output_format=OutputFormat.JSON + ) + + raw_path = route.calls.last.request.url.raw_path.decode() + assert raw_path.startswith("/v1/options/lookup/AAPL%207/28/2023%20200%20Call/") diff --git a/src/tests/test_retry.py b/src/tests/test_retry.py index 46785de..00b301d 100644 --- a/src/tests/test_retry.py +++ b/src/tests/test_retry.py @@ -13,9 +13,7 @@ def _make_retry_state(attempt_number: int, exc: Exception | None): if exc is not None: outcome = MagicMock() outcome.exception.return_value = exc - return type( - "S", (), {"attempt_number": attempt_number, "outcome": outcome} - )() + return type("S", (), {"attempt_number": attempt_number, "outcome": outcome})() def _retry_after_response(value: str) -> Response: @@ -66,7 +64,9 @@ def test_parse_retry_after_seconds(value, expected): def test_parse_retry_after_http_date_future(): - future = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=60) + future = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta( + seconds=60 + ) header = future.strftime("%a, %d %b %Y %H:%M:%S GMT") result = parse_retry_after(header) assert result is not None @@ -78,7 +78,9 @@ def test_parse_retry_after_http_date_past(): def test_parse_retry_after_naive_asctime_treated_as_utc(): - future = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=120) + future = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta( + seconds=120 + ) header = future.strftime("%a %b %d %H:%M:%S %Y") result = parse_retry_after(header) assert result is not None diff --git a/src/tests/test_utils.py b/src/tests/test_utils.py index 253d638..534d84e 100644 --- a/src/tests/test_utils.py +++ b/src/tests/test_utils.py @@ -6,6 +6,8 @@ from marketdata.input_types.base import DateFormat, OutputFormat from marketdata.utils import ( check_is_date, + encode_path, + encode_path_segment, format_duration_log, format_timestamp, merge_csv_texts, @@ -165,9 +167,35 @@ def test_format_duration_hundred_s(): def test_obfuscate_token(): - assert obfuscate_token("1234567890ABCD") == "**********ABCD" + # Fixed-width mask: never reveals token length + assert obfuscate_token("1234567890ABCD") == "****ABCD" assert obfuscate_token("ABCD") == "****" assert obfuscate_token("ABC") == "****" assert obfuscate_token("") == "****" - assert obfuscate_token("12345") == "*2345" + # Short tokens never reveal any characters + assert obfuscate_token("12345") == "****" + assert obfuscate_token("12345678") == "****" assert obfuscate_token(None) == "None" + + +def test_encode_path_segment(): + # Valid symbols pass through unchanged + assert encode_path_segment("AAPL") == "AAPL" + assert encode_path_segment("BRK.B") == "BRK.B" + assert encode_path_segment("AAPL250117C00150000") == "AAPL250117C00150000" + assert encode_path_segment(5) == "5" + # Path traversal and query/fragment smuggling are neutralized + assert encode_path_segment("AAPL/../../user") == "AAPL%2F..%2F..%2Fuser" + assert encode_path_segment("AAPL?a=b") == "AAPL%3Fa%3Db" + assert encode_path_segment("AAPL#frag") == "AAPL%23frag" + + +def test_encode_path(): + # Valid lookup strings keep their slashes, spaces are percent-encoded + assert encode_path("AAPL 7/28/2023 200 Call") == "AAPL%207/28/2023%20200%20Call" + # Dot-segments cannot traverse to another endpoint + assert encode_path("AAPL/../../user") == "AAPL/%2E%2E/%2E%2E/user" + assert encode_path("..") == "%2E%2E" + assert encode_path(".") == "%2E" + # Query/fragment smuggling is neutralized + assert encode_path("AAPL?a=b#frag") == "AAPL%3Fa%3Db%23frag" From 4d165bfd24814e5813b50fb4da795f59cb49f5a7 Mon Sep 17 00:00:00 2001 From: MarketDataApp Date: Thu, 2 Jul 2026 13:34:36 -0300 Subject: [PATCH 2/3] Use readable tag pins for Actions instead of commit SHAs Keep pypa/gh-action-pypi-publish on a fixed release tag (v1.14.0) rather than the moving release/v1 branch ref. --- .github/workflows/publish.yml | 16 ++++++++-------- .github/workflows/test.yml | 8 ++++---- CHANGELOG.md | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 57ca2e6..39c5e88 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,15 +16,15 @@ jobs: id-token: write steps: - name: Checkout code - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@v6 - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@v8.2.0 with: version: "latest" - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + uses: actions/setup-python@v6 with: python-version: "3.12" @@ -32,7 +32,7 @@ jobs: run: uv build - name: Publish to TestPyPI - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 + uses: pypa/gh-action-pypi-publish@v1.14.0 with: repository-url: https://test.pypi.org/legacy/ @@ -48,15 +48,15 @@ jobs: if: github.event_name == 'release' steps: - name: Checkout code - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + uses: actions/checkout@v6 - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@v8.2.0 with: version: "latest" - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + uses: actions/setup-python@v6 with: python-version: "3.12" @@ -64,4 +64,4 @@ jobs: run: uv build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 + uses: pypa/gh-action-pypi-publish@v1.14.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e7ed584..a5ecbfd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,15 +17,15 @@ jobs: python-version: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: actions/checkout@v6 - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + uses: astral-sh/setup-uv@v8.2.0 with: version: "latest" - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} @@ -48,7 +48,7 @@ jobs: ls -lh coverage.xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7 + uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 70ba6ac..9d6ee33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Token obfuscation in logs no longer reveals the token length, and never reveals any characters of short tokens - API error messages extracted from response bodies are now bounded, so a malformed or hostile response cannot balloon exception messages and logs - Malformed rate-limit headers no longer crash a successful request with a raw `KeyError`/`ValueError`; the SDK logs a warning and keeps the previous limits -- GitHub Actions are pinned to full commit SHAs in CI and publish workflows; the test workflow token is now read-only +- The PyPI publish action is pinned to a fixed release tag instead of a moving branch ref; the test workflow token is now read-only ## [1.3.0] - 2026-06-10 From a96397f85a50170a23b1a1393b208ce38bd393c8 Mon Sep 17 00:00:00 2001 From: MarketDataApp Date: Thu, 2 Jul 2026 13:47:06 -0300 Subject: [PATCH 3/3] Document CSV byte-faithfulness as out of scope in security policy --- SECURITY.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/SECURITY.md b/SECURITY.md index 50cd6b4..c4f72b8 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -48,6 +48,11 @@ Out of scope: (`uv.lock`) are tracked by Dependabot (see `.github/dependabot.yml`); report them upstream. We will bump the affected dependency here once a fixed version exists. +- **CSV formula injection.** CSV output is intentionally byte-faithful to the + API response: the SDK does not escape or rewrite cell values (e.g. values + beginning with `=`, `+`, `-`, `@`). Guarding against formula execution when a + CSV is opened in a spreadsheet application is the consuming application's + responsibility. ## Security Fix Policy