diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..056e872 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +# Each line is a file pattern followed by one or more owners. +# These owners are requested for review whenever a matching path changes. +# See https://docs.github.com/articles/about-codeowners + +* @bbalouki diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..e2d37c4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,102 @@ +name: Bug report +description: Report a parsing, build, or runtime defect in ITCHCPP. +title: "[Bug]: " +labels: ["bug", "triage"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to report a bug. Please search + [existing issues](https://github.com/bbalouki/itchcpp/issues) first to + avoid duplicates. + + - type: textarea + id: description + attributes: + label: Description + description: A clear and concise description of the bug. + validations: + required: true + + - type: textarea + id: repro + attributes: + label: Steps to reproduce + description: > + Minimal, compilable code (or CLI invocation) that reproduces the + issue. Attach or link a sample ITCH file if the bug is data-dependent. + render: cpp + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected behavior + validations: + required: true + + - type: textarea + id: actual + attributes: + label: Actual behavior + description: Include the full error message, stack trace, or sanitizer output if applicable. + validations: + required: true + + - type: input + id: version + attributes: + label: ITCHCPP version + description: Contents of `VERSION.txt`, the pip package version, or the commit SHA. + placeholder: e.g. 1.6.2 + validations: + required: true + + - type: dropdown + id: interface + attributes: + label: Interface + options: + - C++ library + - Python bindings (itchcpp) + - itch-tool CLI + - Build system / CMake + validations: + required: true + + - type: dropdown + id: os + attributes: + label: Operating system + multiple: true + options: + - Linux + - Windows + - macOS + validations: + required: true + + - type: input + id: compiler + attributes: + label: Compiler and version + placeholder: e.g. Clang 18, GCC 13, MSVC 19.38 + validations: + required: true + + - type: input + id: cxx-standard + attributes: + label: C++ standard + placeholder: e.g. 20 or 23 + validations: + required: false + + - type: textarea + id: extra + attributes: + label: Additional context + description: CMake configure flags, vcpkg/Conan setup, or anything else relevant. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..4d2f607 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Question or discussion + url: https://github.com/bbalouki/itchcpp/discussions + about: Ask usage questions or discuss ideas before filing an issue. + - name: Security vulnerability + url: https://github.com/bbalouki/itchcpp/security/policy + about: Report security issues privately, do not open a public issue. See SECURITY.md. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..ff420e9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,66 @@ +name: Feature request +description: Suggest an enhancement for ITCHCPP. +title: "[Feature]: " +labels: ["enhancement", "triage"] +body: + - type: markdown + attributes: + value: | + Thanks for suggesting an improvement. Please search + [existing issues](https://github.com/bbalouki/itchcpp/issues) first to + avoid duplicates. + + - type: textarea + id: problem + attributes: + label: Problem + description: What limitation or gap are you running into? Is it related to a specific message type, transport, or interface (C++, Python, itch-tool)? + validations: + required: true + + - type: textarea + id: solution + attributes: + label: Proposed solution + description: What would you like to happen? Include a sketch of the API or CLI surface if relevant. + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + validations: + required: false + + - type: dropdown + id: area + attributes: + label: Area + multiple: true + options: + - Parser / message decoding + - Feed ingestion / transport (MoldUDP64, SoupBinTCP, pcap) + - Book engine + - Analytics + - Interoperability (CSV/Arrow, itch-tool) + - Python bindings + - Build / packaging (CMake, vcpkg, Conan) + - Documentation + validations: + required: false + + - type: checkboxes + id: breaking + attributes: + label: Compatibility + options: + - label: This would require a breaking change to the public API (see [CONTRIBUTING](../../CONTRIBUTING.md#versioning-and-compatibility-policy)). + required: false + + - type: textarea + id: extra + attributes: + label: Additional context + validations: + required: false diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..8a42649 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,24 @@ +## Summary + + + +## Related issues + + + +## Type of change + +- [ ] Bug fix +- [ ] New feature +- [ ] Breaking change (see [CONTRIBUTING](../CONTRIBUTING.md#versioning-and-compatibility-policy)) +- [ ] Documentation +- [ ] Build / CI / tooling + +## Checklist + +- [ ] Code follows the style in [CLAUDE](../CLAUDE.md) (`snake_case`, `PascalCase` types, `m_` members, trailing return types, etc.). +- [ ] `clang-format` and `clang-tidy` pass locally. +- [ ] Tests were added or updated under `tests/` and `ctest` passes. +- [ ] Public API changes are documented with Doxygen `///` comments. +- [ ] [CHANGELOG](../CHANGELOG.md) is updated under `[Unreleased]`. +- [ ] Breaking changes to headers, wire formats, `itch-tool`, or the Python API are called out explicitly above. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..eace80a --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "ci" + + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "build" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de88302..ab00824 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,10 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + # Pinned off windows-latest: it moved to the Windows Server 2025 / VS 2026 + # image in 2026-06 and the vcpkg benchmark port's configure step does not + # yet build cleanly under that toolchain. Revisit once vcpkg catches up. + os: [ubuntu-latest, windows-2022, macos-latest] build_type: [Release, Debug] cpp_standard: [20, 23] runs-on: ${{ matrix.os }} @@ -30,9 +33,14 @@ jobs: path: | ${{ env.VCPKG_ROOT }} ~/.cache/vcpkg - key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json') }} - restore-keys: | - ${{ runner.os }}-vcpkg- + # Keyed on matrix.os (not runner.os) so a runner-image change, which can + # silently swap the compiler/toolset, busts the cache instead of reusing + # binaries or a CMakeCache.txt built by a different toolchain/generator. + # No restore-keys: a partial-match fallback can restore a vcpkg clone + # from before vcpkg.json last changed, whose git history doesn't + # contain the currently pinned builtin-baseline commit, and + # lukka/run-vcpkg then fails outright instead of re-cloning. + key: ${{ matrix.os }}-vcpkg-${{ hashFiles('vcpkg.json') }} - name: Set up vcpkg uses: lukka/run-vcpkg@v11 @@ -41,9 +49,9 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.BUILD_DIR }} - key: ${{ runner.os }}-${{ matrix.build_type }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }} + key: ${{ matrix.os }}-${{ matrix.build_type }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }} restore-keys: | - ${{ runner.os }}-${{ matrix.build_type }}-cmake- + ${{ matrix.os }}-${{ matrix.build_type }}-cmake- - name: Set VCPKG Triplet shell: bash @@ -114,9 +122,11 @@ jobs: path: | ${{ env.VCPKG_ROOT }} ~/.cache/vcpkg + # No restore-keys: a partial-match fallback can restore a vcpkg clone + # from before vcpkg.json last changed, whose git history doesn't + # contain the currently pinned builtin-baseline commit, and + # lukka/run-vcpkg then fails outright instead of re-cloning. key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json') }} - restore-keys: | - ${{ runner.os }}-vcpkg- - name: Set up vcpkg uses: lukka/run-vcpkg@v11 @@ -152,9 +162,11 @@ jobs: path: | ${{ env.VCPKG_ROOT }} ~/.cache/vcpkg + # No restore-keys: a partial-match fallback can restore a vcpkg clone + # from before vcpkg.json last changed, whose git history doesn't + # contain the currently pinned builtin-baseline commit, and + # lukka/run-vcpkg then fails outright instead of re-cloning. key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json') }} - restore-keys: | - ${{ runner.os }}-vcpkg- - name: Set up vcpkg uses: lukka/run-vcpkg@v11 @@ -196,9 +208,11 @@ jobs: path: | ${{ env.VCPKG_ROOT }} ~/.cache/vcpkg + # No restore-keys: a partial-match fallback can restore a vcpkg clone + # from before vcpkg.json last changed, whose git history doesn't + # contain the currently pinned builtin-baseline commit, and + # lukka/run-vcpkg then fails outright instead of re-cloning. key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json') }} - restore-keys: | - ${{ runner.os }}-vcpkg- - name: Set up vcpkg uses: lukka/run-vcpkg@v11 @@ -284,9 +298,10 @@ jobs: path: | ${{ env.VCPKG_ROOT }} ~/.cache/vcpkg + # No restore-keys: see build-and-test's Cache vcpkg step for why a + # partial-match fallback is unsafe here (it could also cross-match the + # non-python vcpkg cache, an even less appropriate restore target). key: ${{ runner.os }}-vcpkg-python-${{ hashFiles('vcpkg.json') }} - restore-keys: | - ${{ runner.os }}-vcpkg- - name: Set up vcpkg uses: lukka/run-vcpkg@v11 @@ -329,9 +344,11 @@ jobs: path: | ${{ env.VCPKG_ROOT }} ~/.cache/vcpkg + # No restore-keys: a partial-match fallback can restore a vcpkg clone + # from before vcpkg.json last changed, whose git history doesn't + # contain the currently pinned builtin-baseline commit, and + # lukka/run-vcpkg then fails outright instead of re-cloning. key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json') }} - restore-keys: | - ${{ runner.os }}-vcpkg- - name: Set up vcpkg uses: lukka/run-vcpkg@v11 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7e01d13..670e3c1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,7 +13,9 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-14] + # Pinned off windows-latest: see build.yml for why (VS 2026 image, vcpkg + # benchmark port not yet compatible). + os: [ubuntu-latest, windows-2022, macos-14] steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9232b53 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,87 @@ +# Changelog + +All notable changes to this project are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [1.6.2] - 2026-07-11 + +### Fixed + +- Sign-conversion warning in the Arrow/Parquet `WriteTable` chunk size + calculation. + +## [1.6.1] - 2026-07-11 + +### Fixed + +- Arrow/Parquet static targets now link correctly when shared libraries are + not built (`ITCH_WITH_ARROW` with static vcpkg triplets). + +## [1.6.0] - 2026-07-11 + +### Added + +- **Feed ingestion / transport** (`itch/transport/`): `MoldUdp64Decoder` and + `SoupBinDecoder` for live and recovery framing, a `PcapReader` for + `.pcap`/`.pcapng` captures, and a `SequenceTracker` for per-session gap + detection, all implemented in-house with no libpcap dependency. +- **Full-market book engine** (`itch::book::BookManager`, `L3Book`): allocation- + light, multi-symbol L3 order book reconstruction with an object pool, + intrusive FIFO price levels, flat sorted ladders, O(1) order lookup, BBO + change callbacks, and L2/L3 depth snapshots. +- **Zero-copy overlay API** (`itch/overlay.hpp`): lazy typed views + (`MessageView`, `AddOrderView`, ...) that decode only the fields the caller + reads. +- **Analytics** (`itch::analytics`): `BarBuilder` (time/tick/volume clocks), + `Vwap`/`Twap`, spread/mid/depth/queue-imbalance and order-flow-imbalance + helpers, NOII decoding, and auction (opening/closing/halt/IPO) reconstruction. +- **Interoperability**: a dependency-free CSV sink, optional Arrow/Parquet + export (`ITCH_WITH_ARROW`), the `itch-tool` CLI (`stats`, `inspect`, + `filter`, `convert`), and native Python bindings (`itchcpp`, pybind11) + mirroring the pure-Python `itch`/`itchfeed` package layout and semantics. +- **Simulation & ecosystem**: a timestamp-paced `ReplayEngine`, a full ITCH + encoder/writer (`itch/encoder.hpp`) with a guaranteed + `parse(encode(msg)) == msg` round-trip, a `VenuePolicy` extension seam + (`itch::venue::Nasdaq50`) for future venues/versions, and Conan packaging + alongside the existing vcpkg port. +- Doxygen documentation site (`docs/Doxyfile`) published to GitHub Pages on + every push to `main`, and a local `docs` CMake target + (`-DITCH_BUILD_DOCUMENTATION=ON`). +- `CONTRIBUTING.md` documenting the build matrix, code style, and the + versioning/ABI compatibility policy. + +## [1.1.0] - 2026-01-17 + +### Added + +- `itch::LimitOrderBook`, a single-symbol order book built from the message + stream, with an example and dedicated tests. +- CMake `FetchContent` support as an alternative to vcpkg for consuming the + library. + +### Changed + +- Reorganized packaging metadata (moved vcpkg port config to `packaging/`) + and renamed CMake targets/exports for consistency. +- Adopted trailing return types across the codebase and tightened compiler + warnings (`-Wshadow`, sign-conversion, unreferenced parameters). + +## [1.0.0] - 2025-10-29 + +### Added + +- Initial release: an allocation-free, single-pass NASDAQ TotalView-ITCH 5.0 + parser with all message types deserialized into a type-safe `itch::Message` + `std::variant`, plus `std::vector`- and callback-based parsing entry points + with optional message-type filtering. + +[Unreleased]: https://github.com/bbalouki/itchcpp/compare/v1.6.2...HEAD +[1.6.2]: https://github.com/bbalouki/itchcpp/compare/v1.6.1...v1.6.2 +[1.6.1]: https://github.com/bbalouki/itchcpp/compare/v1.6.0...v1.6.1 +[1.6.0]: https://github.com/bbalouki/itchcpp/compare/v1.1.0...v1.6.0 +[1.1.0]: https://github.com/bbalouki/itchcpp/compare/v1.0.0...v1.1.0 +[1.0.0]: https://github.com/bbalouki/itchcpp/releases/tag/v1.0.0 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..244197a --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,82 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment include: + +- Demonstrating empathy and kindness toward other people. +- Being respectful of differing opinions, viewpoints, and experiences. +- Giving and gracefully accepting constructive feedback. +- Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience. +- Focusing on what is best not just for us as individuals, but for the overall + community. + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or advances + of any kind. +- Trolling, insulting or derogatory comments, and personal or political + attacks. +- Public or private harassment. +- Publishing others' private information, such as a physical or email address, + without their explicit permission. +- Other conduct which could reasonably be considered inappropriate in a + professional setting. + +## Enforcement Responsibilities + +Project maintainers are responsible for clarifying and enforcing our standards +of acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +## Scope + +This Code of Conduct applies within all community spaces (issues, pull +requests, discussions) and also applies when an individual is officially +representing the project in public spaces. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the maintainers via a private +[security advisory](https://github.com/bbalouki/itchcpp/security/advisories/new) +or by opening a confidential contact through the repository. All complaints +will be reviewed and investigated promptly and fairly. + +All maintainers are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Maintainers will follow these Community Impact Guidelines in determining the +consequences for any action they deem in violation of this Code of Conduct: + +1. **Correction** - A private, written warning, providing clarity around the + nature of the violation. +2. **Warning** - A warning with consequences for continued behavior, including + temporary avoidance of interaction with the people involved. +3. **Temporary Ban** - A temporary ban from any sort of interaction or public + communication with the community. +4. **Permanent Ban** - A permanent ban from any sort of public interaction + within the community. + +## Attribution + +This Code of Conduct is adapted from the +[Contributor Covenant](https://www.contributor-covenant.org), version 2.1, +available at +https://www.contributor-covenant.org/version/2/1/code_of_conduct.html. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..be9349e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,53 @@ +# Security Policy + +## Supported Versions + +ITCHCPP follows [Semantic Versioning](https://semver.org). Security fixes are +made against the latest `1.x` minor release. Older minor versions are not +patched separately. + +| Version | Supported | +| ------- | ------------------ | +| 1.6.x | :white_check_mark: | +| < 1.6 | :x: | + +## Reporting a Vulnerability + +ITCHCPP decodes untrusted, attacker-influenceable binary input (ITCH frames, +MoldUDP64/SoupBinTCP transport framing, and `.pcap`/`.pcapng` captures), so +parser and transport bugs (buffer overflows, out-of-bounds reads, integer +overflow, panics/aborts on malformed input) are treated as security issues, +not just correctness bugs. + +**Please do not open a public GitHub issue for security vulnerabilities.** + +Instead, report privately using +[GitHub Security Advisories](https://github.com/bbalouki/itchcpp/security/advisories/new) +for this repository. Include: + +- A description of the vulnerability and its impact. +- Steps to reproduce, ideally a minimal input file or byte sequence (attach + under the advisory, not in a public issue or PR). +- The affected version (`VERSION.txt` / package version) or commit SHA. +- Whether the issue was found via fuzzing (e.g. `fuzz/parser_fuzzer`, + `fuzz/moldudp64_fuzzer`) or manual review. + +You should expect an initial response within **5 business days**. We will work +with you to confirm the issue, assess severity, and coordinate a fix and +disclosure timeline before any public disclosure. + +## Scope + +In scope: + +- The C++ parser, transport decoders, book engine, and encoder under + `include/itch/` and `src/`. +- The Python bindings (`itchcpp` package). +- The `itch-tool` CLI. + +Out of scope: + +- Vulnerabilities in third-party dependencies (report upstream; note them here + so we can track exposure). +- Denial of service via arbitrarily large, well-formed input (the parser is + designed for high-throughput streaming, not resource capping). diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 2e168e5..50aa60b 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -2,6 +2,11 @@ cmake_minimum_required(VERSION 3.25) # The Python bindings are built only when ITCH_BUILD_PYTHON is ON. pybind11 is # resolved through vcpkg or a system install. +# Force pybind11 onto the modern FindPython path; its default legacy +# FindPythonLibsNew.cmake wrapper calls the CMake-deprecated FindPythonInterp +# module, which recent CMake releases (4.x) no longer support, breaking +# configure in isolated build environments such as vcpkg's CI. +set(PYBIND11_FINDPYTHON ON) find_package(pybind11 CONFIG REQUIRED) # The compiled extension is the private `_itchcpp` module imported by the pure diff --git a/vcpkg.json b/vcpkg.json index f268b88..af28dbe 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -22,5 +22,5 @@ ] } }, - "builtin-baseline": "5bf0c55239da398b8c6f450818c9e28d36bf9966" + "builtin-baseline": "43643e1f5cf73db40d0d4bd610183348eb09b24e" }