diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5eb67c9a..322f7e345 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,21 @@ on: pull_request: branches: [main] +permissions: + contents: read + +# Cancel superseded PR runs only. Main/post-merge uses a unique run_id group so +# in-progress and queued proof runs are never cancelled. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_COMPILERCHECK: content + CCACHE_COMPRESS: "1" + CCACHE_MAXSIZE: 500M + jobs: build: strategy: @@ -18,60 +33,44 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - - name: Install Eigen3 (Linux) + - name: Install dependencies (Linux) if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install -y libeigen3-dev + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libeigen3-dev ccache - - name: Install Eigen3 (macOS) + - name: Install dependencies (macOS) if: runner.os == 'macOS' run: | - brew install eigen - echo "CMAKE_PREFIX_PATH_EXTRA=$(brew --prefix eigen)" >> "$GITHUB_ENV" - - - name: Configure - run: cmake -B build -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - - - name: Build - run: cmake --build build -j 4 - - - name: Test - run: ctest --test-dir build --output-on-failure - - - name: C ABI runtime source check - run: python3 scripts/check_c_abi_runtime.py - - - name: Broker-state hash coverage check - run: python3 scripts/check_broker_state_hash_coverage.py - - - name: Pending-order POD mirror up-to-date check - run: python3 scripts/gen_pending_order_mirror.py --check - - - name: Install PineForge - run: cmake --install build --prefix "$GITHUB_WORKSPACE/pf_install" - - - name: find_package smoke build - shell: bash - env: - WORKSPACE: ${{ github.workspace }} - BUILD_TYPE: ${{ matrix.build_type }} - run: | - set -euo pipefail - EXTRA="${CMAKE_PREFIX_PATH_EXTRA:-}" - PREFIX="${WORKSPACE}/pf_install" - # Pass prefix list to CMake with ';' — ':' is for env vars on Unix only. - CMAKE_PFX="${PREFIX}" - if [[ -n "$EXTRA" ]]; then - CMAKE_PFX="${PREFIX};${EXTRA}" - fi - cmake -S cmake/smoke_consumer -B smoke_build \ - -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ - -DCMAKE_PREFIX_PATH="${CMAKE_PFX}" - cmake --build smoke_build -j 4 - expected="$(tr -d '[:space:]' < VERSION)" - ./smoke_build/smoke_version | grep -qxF "${expected}" + export HOMEBREW_NO_AUTO_UPDATE=1 + brew install eigen ccache + echo "CMAKE_PREFIX_PATH=$(brew --prefix eigen)${CMAKE_PREFIX_PATH:+:$CMAKE_PREFIX_PATH}" >> "$GITHUB_ENV" + + - name: Restore ccache + uses: actions/cache@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}- + + - name: Verify (${{ matrix.build_type }}) + run: python3 scripts/ci_verify.py ${{ matrix.build_type == 'Release' && 'release' || 'debug' }} --build-dir build --jobs 4 --ccache + + - name: Stage and summarize diagnostics + if: always() + run: python3 scripts/collect_ci_diagnostics.py --build-dir build --profile ${{ matrix.build_type == 'Release' && 'release' || 'debug' }} + + - name: Retain CI diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: ci-diagnostics-${{ matrix.os }}-${{ matrix.build_type }} + path: ci-diagnostics/ + if-no-files-found: warn # Dedicated AddressSanitizer + UndefinedBehaviorSanitizer lane. Without this, # the determinism / NaN-propagation / handle-reuse / C-ABI-lifecycle probes @@ -81,25 +80,41 @@ jobs: # PINEFORGE_ENABLE_SANITIZERS). sanitizers: runs-on: ubuntu-latest + env: + ASAN_OPTIONS: detect_leaks=1:halt_on_error=1:abort_on_error=1 + UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 steps: - uses: actions/checkout@v4 + with: + fetch-depth: 1 - - name: Install Eigen3 + - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y libeigen3-dev - - - name: Configure (ASan + UBSan) - run: cmake -B build-asan -DCMAKE_BUILD_TYPE=Debug -DPINEFORGE_ENABLE_SANITIZERS=ON - - - name: Build - run: cmake --build build-asan -j 4 - - - name: Test under sanitizers - env: - ASAN_OPTIONS: detect_leaks=1:halt_on_error=1:abort_on_error=1 - UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 - run: ctest --test-dir build-asan --output-on-failure + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libeigen3-dev ccache + + - name: Restore ccache + uses: actions/cache@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ runner.arch }}-sanitizers-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ runner.arch }}-sanitizers- + + - name: Verify (sanitizers) + run: python3 scripts/ci_verify.py sanitizers --build-dir build-asan --jobs 4 --ccache + + - name: Stage and summarize diagnostics + if: always() + run: python3 scripts/collect_ci_diagnostics.py --build-dir build-asan --profile sanitizers + + - name: Retain CI diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: ci-diagnostics-sanitizers + path: ci-diagnostics/ + if-no-files-found: warn # The branch ruleset requires a status check named `build`, but the matrix # job above reports as `build (ubuntu-latest, Release)` and friends -- a bare diff --git a/.github/workflows/native-live.yml b/.github/workflows/native-live.yml index 529daf4af..932ebf58d 100644 --- a/.github/workflows/native-live.yml +++ b/.github/workflows/native-live.yml @@ -10,6 +10,20 @@ on: permissions: contents: read +# Cancel superseded PR runs only. Main/post-merge and manual workflow_dispatch +# use a unique run_id group so proof runs are never cancelled. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_COMPILERCHECK: content + CCACHE_COMPRESS: "1" + CCACHE_MAXSIZE: 500M + CURL_VERSION: "8.14.1" + CURL_SHA256: "f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd" + jobs: native-live: runs-on: ubuntu-latest @@ -18,18 +32,58 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: persist-credentials: false + fetch-depth: 1 + - name: Install native build dependencies run: | sudo apt-get update - sudo apt-get install -y cmake ninja-build libeigen3-dev libsqlite3-dev libssl-dev zlib1g-dev + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cmake ninja-build libeigen3-dev libsqlite3-dev libssl-dev zlib1g-dev ccache + + - name: Restore ccache + uses: actions/cache@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ runner.arch }}-native-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ runner.arch }}-native- + + - name: Identify native dependency environment + id: native-deps + shell: bash + run: | + mkdir -p build-native-deps + { + printf '%s\n' "${ImageOS:-unknown}" "${ImageVersion:-unknown}" "${GITHUB_WORKSPACE}" + cc --version + cmake --version + dpkg-query -W libssl-dev zlib1g-dev + } > build-native-deps/environment.txt + echo "identity=$(sha256sum build-native-deps/environment.txt | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT" + + - name: Restore pinned WebSocket-enabled libcurl + id: curl-cache + uses: actions/cache@v4 + with: + path: build-native-deps/curl-install + key: curl-${{ env.CURL_VERSION }}-${{ env.CURL_SHA256 }}-${{ steps.native-deps.outputs.identity }}-${{ hashFiles('.github/workflows/native-live.yml') }}-${{ runner.os }}-${{ runner.arch }} + - name: Build pinned WebSocket-enabled libcurl + env: + CURL_CACHE_HIT: ${{ steps.curl-cache.outputs.cache-hit }} run: | + set -euo pipefail + cmake_file="build-native-deps/curl-install/lib/cmake/CURL/CURLConfig.cmake" + if [[ "${CURL_CACHE_HIT}" == "true" && -f "$cmake_file" ]]; then + echo "Using cached libcurl at ${cmake_file}" + exit 0 + fi mkdir -p build-native-deps cd build-native-deps - curl --fail --location --proto '=https' --tlsv1.2 https://curl.se/download/curl-8.14.1.tar.xz -o curl.tar.xz - echo 'f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd curl.tar.xz' | sha256sum -c - + curl --fail --location --proto '=https' --tlsv1.2 \ + "https://curl.se/download/curl-${CURL_VERSION}.tar.xz" -o curl.tar.xz + echo "${CURL_SHA256} curl.tar.xz" | sha256sum -c - tar -xf curl.tar.xz - cmake -S curl-8.14.1 -B curl-build -G Ninja \ + cmake -S "curl-${CURL_VERSION}" -B curl-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$PWD/curl-install" \ -DBUILD_CURL_EXE=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON \ -DBUILD_TESTING=OFF -DCURL_USE_OPENSSL=ON -DENABLE_WEBSOCKETS=ON \ @@ -37,21 +91,20 @@ jobs: -DCURL_USE_LIBSSH2=OFF -DCURL_BROTLI=OFF -DCURL_ZSTD=OFF cmake --build curl-build -j 4 cmake --install curl-build - - name: Build native runner and tests - run: | - cmake -S . -B build-live -G Ninja -DCMAKE_BUILD_TYPE=Release \ - -DPINEFORGE_BUILD_LIVE_RUNNER=ON -DPINEFORGE_BUILD_TUTORIAL=OFF \ - -DCURL_DIR="$PWD/build-native-deps/curl-install/lib/cmake/CURL" - cmake --build build-live -j 4 - - name: Test native engine, parser, journal and transports - run: | - ctest --test-dir build-live --output-on-failure -j 4 - # This required lane uses the pinned WS-enabled curl. A skip is not - # sufficient here even though ordinary system-curl builds may skip. - build-live/bin/test_native_live_websocket - python3 scripts/check_c_abi_runtime.py - python3 scripts/check_broker_state_hash_coverage.py - - name: Install native executable - run: | - cmake --install build-live --prefix "$PWD/build-live/install" - build-live/install/bin/pineforge-live --help + + - name: Verify native live + run: python3 scripts/ci_verify.py native --build-dir build-live --jobs 4 --generator Ninja --curl-dir "${{ github.workspace }}/build-native-deps/curl-install/lib/cmake/CURL" --ccache --require-websocket + + - name: Stage and summarize diagnostics + if: always() + env: + CURL_CACHE_HIT: ${{ steps.curl-cache.outputs.cache-hit }} + run: python3 scripts/collect_ci_diagnostics.py --build-dir build-live --profile native + + - name: Retain CI diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: ci-diagnostics-native-live + path: ci-diagnostics/ + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 31995814c..becb41290 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,7 @@ docs/production-readiness-findings.md # Cloudflare wrangler local cache (never commit) .wrangler/ + +# Reusable CI cache and retained diagnostics (never source bundles) +.ccache/ +ci-diagnostics/ diff --git a/AGENTS.md b/AGENTS.md index e8963b9a2..ed2a2137b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,13 +7,9 @@ Both C++ unit tests and full corpus verification must pass. ```bash -# 1. Build and run unit tests (ctest) -cmake -B build -S . \ - -DCMAKE_BUILD_TYPE=Release \ - -DPINEFORGE_BUILD_TESTS=ON \ - -DPINEFORGE_BUILD_CORPUS_STRATEGIES=ON -cmake --build build -j$(nproc 2>/dev/null || echo 4) -ctest --test-dir build --output-on-failure +# 1. Run the same complete verification profile as CI +python3 scripts/ci_verify.py release --build-dir build --jobs 4 +# Relevant additional profiles: debug, sanitizers, native (see docs/ci.md) # 2. Run full validation corpus sweep (against TV exported trades) ./scripts/run_corpus.sh diff --git a/README.md b/README.md index 03bdbf23e..dfaf8ab18 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # PineForge -**The open-source PineScript v6 backtest engine that reproduces TradingView trade-for-trade.** +**An open-source C++17 engine for backtesting and forward execution, with PineScript support through code generation.** [![CI](https://img.shields.io/github/actions/workflow/status/pineforge-4pass/pineforge-engine/ci.yml?branch=main&label=ci&logo=github)](https://github.com/pineforge-4pass/pineforge-engine/actions) [![Parity](https://img.shields.io/badge/TradingView%20parity-4%2C190%20%2F%204%2C190%20probes-brightgreen)](#validation-scoreboard) @@ -74,13 +74,13 @@ Prefer zero install? The hosted server at **[mcp.pineforge.dev/mcp](https://mcp. ```bash git clone https://github.com/pineforge-4pass/pineforge-engine.git && cd pineforge-engine -cmake -B build -DCMAKE_BUILD_TYPE=Release -cmake --build build -j -ctest --test-dir build --output-on-failure # all enabled unit, replay and ABI checks +python3 scripts/ci_verify.py release --build-dir build --jobs 4 bash tutorial/run.sh # MACD on BTC/USDT, end to end python3 tutorial/run_stream.py # OHLCV warm-up → realtime trades ``` +The [shared local/CI verifier](docs/ci.md) includes source guards, tests and installed-package smoke checks. Its ABI check links callers against both the current library and a separately built, pinned historical library. Preparation uses local Git history and the configured compiler; see the [ABI fixture guide](tests/fixtures/settlement_cpp_abi/README.md) for shallow clones and repeated checks. CTest itself stays offline. + Prerequisites: CMake ≥ 3.16, a C++17 compiler (GCC ≥ 9, Clang ≥ 10, Apple Clang ≥ 12), Eigen 3.3+ (fetched automatically if absent), Python 3 for the tests (`-DPINEFORGE_BUILD_TESTS=OFF` for a library-only build). `cmake --install build --prefix /usr/local` installs `lib/libpineforge.a`, `include/pineforge/`, and the `find_package(PineForge)` config. ### Embedded in your own harness diff --git a/cmake/PineForgeVersion.cmake b/cmake/PineForgeVersion.cmake index dcc7aae02..ae61b158a 100644 --- a/cmake/PineForgeVersion.cmake +++ b/cmake/PineForgeVersion.cmake @@ -1,21 +1,27 @@ # PineForgeVersion.cmake # -# Resolve PineForge version from two sources, in priority order: -# 1. `git describe --tags --match 'v*' --abbrev=7 --dirty` (when in a git -# checkout). Strips leading 'v', exposes commits-since-tag + sha + dirty -# as PINEFORGE_VERSION_FULL. -# 2. The `VERSION` file at the repo root (always present in tarballs and -# Docker build contexts). +# PINEFORGE_VERSION_SOURCE (CACHE STRING, AUTO|FILE): +# AUTO (default) — historical behavior exactly: prefer +# `git describe --tags --match 'v*' --abbrev=7 --dirty` for MMP/FULL +# when it matches MAJOR.MINOR.PATCH after stripping a leading 'v'; +# otherwise use the VERSION file. Git SHA/dirty are observed the same way. +# FILE — MMP and FULL always come from VERSION (stable release identity). +# Git SHA/dirty are still observed as separate fields; checkout depth or +# missing tags cannot change MMP/FULL. Invalid values fail configure. # # Sets in the calling scope: # PINEFORGE_VERSION_MAJOR integer # PINEFORGE_VERSION_MINOR integer # PINEFORGE_VERSION_PATCH integer # PINEFORGE_VERSION_MMP "MAJOR.MINOR.PATCH" (for project(VERSION ...)) -# PINEFORGE_VERSION_FULL "MAJOR.MINOR.PATCH[-N-gSHA[-dirty]]" +# PINEFORGE_VERSION_FULL AUTO: describe-or-VERSION; FILE: VERSION # PINEFORGE_VERSION_GIT_SHA short sha or "unknown" # PINEFORGE_VERSION_DIRTY ON/OFF +set(PINEFORGE_VERSION_SOURCE "AUTO" CACHE STRING + "Version identity source: AUTO (git describe preferred) or FILE (VERSION MMP/FULL)") +set_property(CACHE PINEFORGE_VERSION_SOURCE PROPERTY STRINGS AUTO FILE) + function(_pineforge_read_version_file _out_mmp) set(_vfile "${CMAKE_CURRENT_SOURCE_DIR}/VERSION") if(NOT EXISTS "${_vfile}") @@ -30,6 +36,12 @@ function(_pineforge_read_version_file _out_mmp) endfunction() function(pineforge_resolve_version) + if(NOT PINEFORGE_VERSION_SOURCE STREQUAL "AUTO" AND + NOT PINEFORGE_VERSION_SOURCE STREQUAL "FILE") + message(FATAL_ERROR + "PINEFORGE_VERSION_SOURCE must be AUTO or FILE (got '${PINEFORGE_VERSION_SOURCE}')") + endif() + _pineforge_read_version_file(_file_mmp) set(_git_full "") @@ -67,9 +79,18 @@ function(pineforge_resolve_version) endif() endif() - set(_mmp "${_file_mmp}") - if(_git_full MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)") - set(_mmp "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") + if(PINEFORGE_VERSION_SOURCE STREQUAL "FILE") + set(_mmp "${_file_mmp}") + set(_full "${_file_mmp}") + else() + set(_mmp "${_file_mmp}") + if(_git_full MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)") + set(_mmp "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") + endif() + if(NOT _git_full) + set(_git_full "${_mmp}") + endif() + set(_full "${_git_full}") endif() string(REPLACE "." ";" _parts "${_mmp}") @@ -77,15 +98,11 @@ function(pineforge_resolve_version) list(GET _parts 1 _min) list(GET _parts 2 _pat) - if(NOT _git_full) - set(_git_full "${_mmp}") - endif() - set(PINEFORGE_VERSION_MAJOR "${_maj}" PARENT_SCOPE) set(PINEFORGE_VERSION_MINOR "${_min}" PARENT_SCOPE) set(PINEFORGE_VERSION_PATCH "${_pat}" PARENT_SCOPE) set(PINEFORGE_VERSION_MMP "${_mmp}" PARENT_SCOPE) - set(PINEFORGE_VERSION_FULL "${_git_full}" PARENT_SCOPE) + set(PINEFORGE_VERSION_FULL "${_full}" PARENT_SCOPE) set(PINEFORGE_VERSION_GIT_SHA "${_git_sha}" PARENT_SCOPE) set(PINEFORGE_VERSION_DIRTY "${_dirty}" PARENT_SCOPE) endfunction() diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 000000000..c72357c32 --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,82 @@ +# Local verification and CI + +Use the same entrypoint as GitHub Actions before publishing a change: + +```sh +python3 scripts/ci_verify.py release --build-dir build --jobs 4 +``` + +The command runs source guards, configures and rebuilds every enabled target, +prepares the real historical ABI provider, runs CTest, installs the package, +and builds and runs the `find_package` consumer. The smoke test compares the +installed library's reported version with `VERSION`. After a successful build, +a failing test suite does not hide a separate install or package failure. + +## Profiles + +| Profile | Build | Additional coverage | +| --- | --- | --- | +| `release` | Release, tutorial enabled | Standard CI checks and installed package | +| `debug` | Debug, tutorial enabled | The same checks without Release optimization | +| `sanitizers` | Debug, ASan and UBSan | Instrumented library, tests and installed consumer; Linux CI also requires leak detection | +| `native` | Release, live runner enabled | Parser, journal, transport tests and installed runner help | + +By default each profile uses `build-ci-`. Keep separate build directories +for different profiles and toolchains. The verifier never deletes a build tree +or replaces an incompatible historical ABI receipt. Use a fresh directory when +its configuration no longer matches. + +```sh +python3 scripts/ci_verify.py debug --jobs 4 +python3 scripts/ci_verify.py sanitizers --jobs 4 +python3 scripts/ci_verify.py native --curl-dir /path/to/curl/lib/cmake/CURL \ + --require-websocket --jobs 4 +``` + +The sanitizer profile uses the Linux CI ASan/UBSan environment, including +LeakSanitizer, which needs a supporting runtime. The native CI lane builds +checksum-pinned curl with WebSocket support and passes `--require-websocket`. +That flag turns a transport skip into failure. A local native run using system +curl may report the existing unsupported-WebSocket skip; that is not the required +Linux transport proof. + +Pass `--ccache` when ccache is installed. It caches compiler work, not complete +build directories or verification receipts. Compiler, source, header and flag +checks remain in effect. The native curl cache is separately bound to its source +checksum, workflow configuration, compiler/package environment and install path. + +## Version identity and historical ABI checks + +Verification explicitly uses `PINEFORGE_VERSION_SOURCE=FILE`: package MMP/FULL +identity comes from `VERSION`, while Git revision/dirty observations remain +separate. Git tags and checkout depth cannot change the smoke-test expectation. +Ordinary CMake builds retain `AUTO`, the existing git-describe-first behavior. +No release tag or VERSION value is rewritten by verification. + +The verifier fetches the pinned historical ABI commits `e60e571` (R2) and +`0e18690` (selected settlement, before exact reversal) without tags only when +each object is missing. It builds both full historical static libraries with +tests disabled, or validates and reuses their matching prepared receipts under +`settlement-abi-base/` and `settlement-abi-prior/`. Compiler, configuration and +version-source mismatches refuse reuse without deleting the old evidence. +Each profile needs matching providers; a Mac Release archive cannot replace +a Linux sanitizer build. CTest itself performs no network fetch. +The [ABI guide](../tests/fixtures/settlement_cpp_abi/README.md) describes the +actual old/new library pairs and their immutable inputs. + +## Failure evidence + +Each build directory contains `ci-summary.json` and full command logs under +`ci-logs/`, plus CTest JUnit when the installed CTest supports it. A version +failure records both actual and expected values. GitHub Actions retains compact +diagnostics, CTest logs and ABI receipts and publishes the stage results in its +job summary. Compiler objects, binaries and dependency caches are not uploaded +as diagnostics. + +Superseded pull-request runs are canceled. Main/post-merge and manual proof runs +use distinct concurrency groups and remain uncanceled. The required `build` +check still depends on every standard matrix lane; sanitizer and native-live +checks remain separate requirements. + +These checks do not run the parity campaign. Fixed-population Cloud measurement, +the actual gate and post-merge evidence remain separate acceptance steps. diff --git a/docs/native-refactor-progress.md b/docs/native-refactor-progress.md index 20f6dae8a..7472d2937 100644 --- a/docs/native-refactor-progress.md +++ b/docs/native-refactor-progress.md @@ -1,6 +1,6 @@ # Standalone native engine refactor -Status at 2026-09-12: **3 of 9 roadmap phases complete (33%)**. This counts +Status at 2026-09-13: **4 of 9 roadmap phases complete (44%)**. This counts completed phases; it is not an estimate of elapsed work or remaining time. The acceptance boundary is a standalone C++ backtest and forward-execution @@ -14,10 +14,10 @@ improvement resumes after the final native audit. | --- | --- | --- | | R0 | Native settlement and CI convergence | Complete, [PR #241](https://github.com/pineforge-4pass/pineforge-engine/pull/241) | | R1 | Native market orders, host, drivers, calendar and forward runner | Complete, [PR #243](https://github.com/pineforge-4pass/pineforge-engine/pull/243) | -| R2 | Resting orders, partial execution, owner-bound children and group effects | R2a prerequisite complete ([PR #244](https://github.com/pineforge-4pass/pineforge-engine/pull/244)); resting lifecycle implemented, validation and acceptance pending | -| R3 | Accounting, admission, risk and observation ownership | Open | +| R2 | Resting orders, partial execution, owner-bound children and group effects | Complete, [PR #246](https://github.com/pineforge-4pass/pineforge-engine/pull/246) | +| R3 | Accounting, admission, risk and observation ownership | Accounting implementation in progress; admission/risk/observation remain open | | R3a | Reversal execution and lifecycle settlement | Complete, [PR #242](https://github.com/pineforge-4pass/pineforge-engine/pull/242) | -| R3b | Migrate remaining physical fill paths to shared settlement | Open | +| R3b | Migrate remaining physical fill paths to shared settlement | Implementation in progress | | R4 | Complete codegen/adapter policy ownership and native independence | Open | | R5 | Final requirement and compatibility audit | Open | | R6 | Resume the parity improvement campaign after the audit | Queued | @@ -55,10 +55,9 @@ These results apply to R2a. They do not establish acceptance of the later resting-order implementation, and R2a is not an additional completed roadmap phase. -## Current R2 implementation +## Accepted R2 resting-order state machine -This branch implements the pinned native contract. Validation and review -repairs are still in progress: +PR #246 completed the native resting-order contract: 1. Limit, stop, stop-limit and trailing requests retain trigger state and partial remainders. Matching distinguishes actual tick prints from @@ -84,10 +83,17 @@ partial exits, exact deferred-receipt arithmetic and same-cursor execution price The sanitizer ABI control also supplies the historical v12 destructor needed for its RTTI; this changes only the link-test stub, not runtime behavior. -Final review of the publication candidate, remaining R2 acceptance coverage, -a new fixed-population 4190 comparison, the actual gate and published-head CI -remain required. R2 is not complete. The earlier R2a and separate AUTO-isolation -compatibility results do not substitute for this implementation's proof. +The final candidate `c57e8687d91027df09474b414369162e47b94bbb` passed independent +review, 326 local CTests (one macOS libcurl WebSocket skip), all nine PR checks +and four post-merge workflows. It was squash merged as +`e60e57156a54bb1c74c0c0a4f5a338fd924c046d`, with identical tree +`e209c89035ec884774f8f3e68e1dc118f26aaf51`. + +The completed Cloud comparison measured 72/72 cases and 4190/4190 probes with +zero grade, raw-trade, substantive-verifier, metadata, configuration or coverage +differences. The actual gate returned **FAIL: `target.not-positive` only**. +That neutral result is retained under the authorized refactor exception; +the baseline stayed unchanged and promotion was skipped. R2 preserves all currently supported calendar, session, timezone, DST and timeframe behavior. Source labels remain inert in the native kernel; @@ -97,6 +103,24 @@ The native R2 boundary is pinned; complete Pine lowering remains mandatory R4 work. The parity-improvement campaign remains paused through the final native audit. +## Current R3 accounting and fill migration + +The next implementation brings the remaining partial, bound, percent, scratch, +reversal, RAW and open/add fill paths onto the shared settlement owner. A +transient set of opening identities selects physical exposure without putting +Pine `from_entry` strings into the kernel. A non-applying account projection +supplies post-close equity to source sizing before one resolved reversal. + +The native owner retains historical paid costs, one current execution ticket, +physical residual quantities and deterministic observations. Adapters retain +quantity provenance, source slot policy and the existing exit-ordering rules. +Existing native request/event layouts and C ABI 4 remain stable. This work +requires implementation, native and real-caller tests, independent review and +the full publication proof below before R3b can be marked complete. + +R3's remaining admission, quota/day clocks, source counters and physical versus +script-visible observation ownership stay open and must finish before R4. + ## Publication and acceptance Publish reviewed increments in an evolving draft PR so implementation and diff --git a/docs/native-settlement.md b/docs/native-settlement.md index b5248fe1b..4ea89c837 100644 --- a/docs/native-settlement.md +++ b/docs/native-settlement.md @@ -36,6 +36,73 @@ epsilon silently discards it. `Flatten` explicitly closes all lots and avoids using rounded aggregate equality to mean a whole-book close. Quantities whose changes cannot be represented are refused before settlement. +The source `execute_partial_exit_qty` adapter retains its existing `1e-10` +FIFO endpoint policy. After its existing whole-book Flatten check, it may +translate a quantity ending at an interior whole-lot prefix into one selected +Flatten. It walks physical lots in FIFO order, stops before the next lot once +the source endpoint is reached, and selects complete opening incarnations, +independently of the logical order label that supplied the close quantity. +A genuine partial lot, an unowned selected lot, or an opening with a fragment +outside the prefix keeps the original scalar Reduce path. Selected Flatten +closes the exact live quantities and all their remaining paid entry costs in +one execution; it does not reproduce historical residual-cost discards. +The native Reduce contract, entry-scoped adapters and transaction-flow requests +retain their existing behavior. This source translation requires separate +compatibility measurement; it does not establish that every observed extra +trade row has the same cause. + +## Reversal to an exact exposure + +`execution::reverse_to_v1::ReverseTo{signed_units}`, declared in +``, closes the entire opposite live book +and opens exactly `abs(signed_units)` units in the requested direction. Positive +targets open long; negative targets open short. It is a separate, call-local +resolved-execution value available to trusted C++ adapters through four protected, +nonvirtual methods: + +| Method | Result and context | +| --- | --- | +| `inspect_native_reversal_v1(reversal, fill)` | Non-mutating `SettlementInspection` of physical effects and the current ticket. | +| `project_native_reversal_v1(reversal, fill)` | Non-mutating `AccountEffectProjection`, including account effects and the next cycle without consuming it. | +| `settle_native_reversal_at_v1(reversal, fill, context)` | `Result` from settlement using the supplied `PhysicalExecutionContext` and empty lifecycle effects. | +| `settle_reversal_with_lifecycle_v1(reversal, fill, lifecycle)` | `Result` from settlement using the engine's current time, interval and preceding exit-path facts with supplied transient lifecycle effects. | + +For example, reversing a held long position of `1` unit to a short target of +`0.1` opens the exact binary64 quantity supplied as `-0.1`. The opening quantity +is never reconstructed by adding the old position to the target and subtracting +the closed position. Every existing lot closes in roster order using whole-lot +Flatten allocation, and one new lot opens at the same resolved `Fill::price`. +Close and open effects share one current ticket and the existing financial +commit owner. No selection or partial-close scope participates in this seam. + +A valid call requires a nonflat book opposite to a finite, nonzero target. +Zero or nonfinite targets return `InvalidQuantity`; a flat or same-side book +returns `InvalidCloseTarget`; malformed physical books return `InvalidBook`. +There is no valid no-effect reversal. Callers use `Flatten` to close without +opening. If finite closed and opening quantities have a nonfinite gross sum, +the call returns `UnrepresentableQuantity` before quoting fees. A tiny target +such as `0.1` against a held quantity of `1e16` remains valid even when their +floating-point sum absorbs the target: after all old lots close, the exact new +quantity is representable on its own. Existing accounting, lifecycle and +sequence preflight checks still apply before effects. + +Inspection, projection and settlement share allocation and fee calculations. +Successful `opened_units` and projected `signed_units_after` preserve the signed +target bits; the resulting book contains one lot of exactly its absolute size. +Projection includes the opening's paid cost once and peeks the fresh cycle; +settlement revalidates the live book and consumes that cycle when it opens. +Invalid projections contain no usable account quote. These values grant no +saved or replayable settlement authority. + +`Transact` continues to describe signed transaction flow, including its existing +opposite-book remainder arithmetic. Native queued requests and their action and +event algebra are unchanged; `ReverseTo` is not an `Action` or request variant. +The source F7 reversal adapter uses this seam after resolving its opening size. +Other flow callers, including F8, retain `Transact`. This opening-intent repair +does not establish that the seven additional trade rows or the EURUSD (ERA) +quantity/margin differences observed during refactor verification are repaired; +those remain separate compatibility acceptance work. + ## Costs and marked equity Every opening lot receives its paid entry commission before its entry @@ -70,13 +137,40 @@ PnL, minus remaining paid entry costs of every fee type. At an unchanged mark, a native execution decreases this value by its current charge, subject to ordinary floating-point rounding. A rebate increases it. +## Selected exposure and account projection + +`execution::SelectedOpeningSet` selects one or more opening identities in the +current position cycle. The separately named selected settlement methods accept +it by const reference; it is not another `CloseScope` alternative or a queued +request field. All fragments of each selected opening participate in roster FIFO +order. Empty sets, zero or duplicate identities, missing openings and stale cycles +are refused before effects. Source adapters construct distinct identities from +`from_entry` or other source predicates; the native owner does not interpret them. + +`project_native_settlement_v1`, `project_native_settlement_scoped_v1` and +`project_native_settlement_selected_v1` quote the resulting physical and account +facts without changing the engine. The projection includes realized balance, +remaining paid entry costs, marked equity, signed exposure and resulting cycle. +It includes unselected surviving lots and any prospective opening's paid cost. +A valid no-effect action quotes the unchanged book at the supplied mark; invalid +input supplies no usable quote. An unavailable fresh cycle throws without +consuming it. Same-side additions retain the existing cycle. + +Projection and settlement share allocation and fee logic. Realized balance adds +close-row PnL in commit order, and marked equity applies each resulting lot's mark +and paid cost in roster order. A projection is data, not commit authority: later +settlement revalidates against the current book. Source percent sizing can consume +a close-only projection's equity before resolving one reversal opening quantity. + ## Integration and limits -The kernel currently serves full market exits, the close-opposite-then-enter -reversal family, and selected frozen-transaction and same-side materialization -paths. Existing source scheduling and dust decisions remain at their call -sites. The reversal helper consumes one already-resolved `Fill` and one current -fee; it does not slip again. `compat::pine` suspension selection stays at the +The kernel serves native execution and the source adapters' full, partial, +bound and percent closes, scratch fills, reversals, RAW orders and opening/add +paths. Callers resolve source scheduling, quantity grids, price and slot policy; +the owner applies the physical effects and accounting once. A separately matched +scratch fill remains its own execution. Reversal closing and opening share one +already-resolved price and one current ticket. +`compat::pine` suspension selection stays at the replacement caller. `settle_resolved_execution` remains the original two-argument symbol and forwards empty effects to `settle_execution_with_lifecycle`, the protected seam that consumes transient @@ -94,9 +188,9 @@ reallocate `pending_orders_`. Native settlement does not recognize source cases, rewrite supplied window/barrier facts, or install a callback/plan. Migrated frozen transactions and the final short-seed crossing settle their close/open effects in one native call. -Other legacy close loops are not yet -grouped into one parent execution; their per-row current ticket behavior is -not a claim about the native contract. +Production fill paths no longer use the old per-row close loops as a separate +accounting owner. Historical private helpers remain for source compatibility and +tests; they are not an alternate production execution path. The shared close builder now consumes historical entry costs. This changes the former reconstruction that converted both commission legs at exit-time @@ -114,6 +208,7 @@ lifecycle exceptions abort the owning run; callers must discard that failed run rather than retry a partially committed execution in place. Strong rollback on allocation failure is not promised. -The work does not remove `ShortSeedCollisionRole` or complete migration to a -generic queued order machine. Existing executable state remains represented -in ABI projections and fingerprints. +The work retains `ShortSeedCollisionRole` while its source-policy consumers +remain. Admission, source day/quota counters, script-visible observations and +complete Pine lowering remain separate refactor work. Existing executable state +remains represented in ABI projections and fingerprints. diff --git a/include/pineforge/engine.hpp b/include/pineforge/engine.hpp index 3624e74dd..729f246cb 100644 --- a/include/pineforge/engine.hpp +++ b/include/pineforge/engine.hpp @@ -21,6 +21,9 @@ #include "quantity_intent.hpp" #include "execution.hpp" #include "execution_close_scope.hpp" +#include "execution_close_selection.hpp" +#include "execution_projection.hpp" +#include "execution_reverse_to.hpp" #include "market_admission.hpp" #include "reservation_expansion.hpp" #include "order_cancellation.hpp" @@ -1840,6 +1843,53 @@ class BacktestEngine { const execution::Action& action, const execution::Fill& fill, const execution::PhysicalExecutionContext& context, execution::CloseScope scope); + // --- Additive selected close (not a CloseScope overload) --- + execution::SettlementInspection inspect_native_settlement_selected( + const execution::Action& action, + const execution::Fill& fill, + const execution::SelectedOpeningSet& selection) const; + execution::Result settle_native_execution_selected_at( + const execution::Action& action, + const execution::Fill& fill, + const execution::PhysicalExecutionContext& context, + const execution::SelectedOpeningSet& selection); + // Legacy lifecycle copy of current_bar_/fold flags, then selected commit. + // Empty lifecycle is valid. Do not fold selection into LifecycleEffects. + execution::Result settle_execution_selected_with_lifecycle( + const execution::Action& action, + const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::SelectedOpeningSet& selection); + // --- Additive non-applying account/effect projection --- + execution::AccountEffectProjection project_native_settlement_v1( + const execution::Action& action, + const execution::Fill& fill) const; + execution::AccountEffectProjection project_native_settlement_scoped_v1( + const execution::Action& action, + const execution::Fill& fill, + execution::CloseScope scope) const; + execution::AccountEffectProjection project_native_settlement_selected_v1( + const execution::Action& action, + const execution::Fill& fill, + const execution::SelectedOpeningSet& selection) const; + // Opposite-book reversal to an exact signed exposure. Unlike Transact, + // the opening is not a remainder of transaction units minus held units. + // These synchronous extensions share the settlement owner and do not + // introduce a queued native request or saved commit authority. + execution::SettlementInspection inspect_native_reversal_v1( + const execution::ReverseTo& reversal, + const execution::Fill& fill) const; + execution::AccountEffectProjection project_native_reversal_v1( + const execution::ReverseTo& reversal, + const execution::Fill& fill) const; + execution::Result settle_native_reversal_at_v1( + const execution::ReverseTo& reversal, + const execution::Fill& fill, + const execution::PhysicalExecutionContext& context); + execution::Result settle_reversal_with_lifecycle_v1( + const execution::ReverseTo& reversal, + const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle); // Native account value: realized balance plus marked physical lots minus // their remaining paid entry costs, for every fee type. No Pine sizing or // end-of-range reporting convention participates in this value. @@ -2641,68 +2691,9 @@ class BacktestEngine { // tape discriminating round(close) from close has been replayed. A // future CASH mismatch on a sub-tick feed is traced here first. double calc_qty(double fill_price) const { - const double basis = round_to_mintick(fill_price); - switch (default_qty_type_) { - case QtyType::FIXED: - return apply_qty_step(default_qty_value_); - case QtyType::PERCENT_OF_EQUITY: { - // KI-56's clean-room v6 flat/holding pair proves that an - // omitted percent-of-equity add sizes from mark-to-market - // equity AFTER the paid percent commission on surviving lots. - // This is a ledger rule, independent of pct=100, margin, or - // how the already-open position was sized. The open lot is - // marked at the ROUNDED close (see above): a sub-tick print - // never reaches the broker ledger. - // round 8 family R: the broker sizes from the equity at - // TEN SIGNIFICANT DIGITS (tv_money_round above) — the lot - // count flips one lot early/late whenever the exact equity - // sits within half a money unit of a lot boundary. - double equity = percent_commission_live_equity( - round_to_mintick(current_bar_.close)); - // Round 10 family AE (NASDAQ:AAPL@15, fast-scalper; campaign - // note log-20260905t223336z-67f0f181): the rounding is NOT - // confined to sub-unit lots. On a cent-priced integer-share - // book the exact equity is a cent sum, but the engine's - // ledger carries float-accumulation noise (1094521.68 held - // as 1094521.6800000002) and the quotient by the close can - // sit exactly on an integer in decimal (1094521.68 / 238.77 - // = 4584): TradingView floors the ten-digit equity's raw - // double quotient (4583.999999999999 -> 4583 shares); the - // noisy numerator (4584.000000000001) or the 1e-6 nudge of - // apply_qty_step both hand it 4584, one share the account - // cannot pay at the 238.78 fill, and the reversal is declined - // where TradingView admits it. Nine lab tv capital sweeps on - // AAPL (famae-sz-*): 897890.32 / 213.58 -> 4203 (the raw - // floor; 4204 in exact decimal), 897890.3200004 -> 4203 (the - // 4e-7 is rounded away before the division), 897890.321 -> - // 4204; 887295.33 / 211.11 -> 4202, .3300004 -> 4202, - // .331 -> 4203; 1094521.68 / 238.77 -> 4583 (the probe's - // row), 1094521.681 -> 4584 (dropped: 4584 x 238.78 > C). - // Every lot-stepped instrument therefore sizes from - // tv_money_round(equity) with the raw lot floor; only the - // corpus' continuous qty_step 0 keeps the exact arithmetic. - if (tv_money_lot_sizing()) equity = tv_money_round(equity); - if (!std::isfinite(equity)) return 0.0; - double cash = reserve_percent_commission(equity * (default_qty_value_ / 100.0)) / active_account_currency_fx(); - // Reject (qty 0) on a non-finite / non-positive fill price — a - // degenerate $0/NaN print must NOT size as the raw % number. - if (!(std::isfinite(basis) && basis > 0)) return 0.0; - // Rule 1 floors the rounded-equity quantity onto the lot - // grid via tv_money_floor_lot, including its guarded recovery - // of a representable cent-lot point. No representation nudge: - // apply_qty_step's epsilon would still over-size the pinned - // everybar 1.085 placement by one lot. - if (tv_money_lot_sizing()) { - return tv_money_floor_lot(cash / (basis * syminfo_.pointvalue), - qty_step_); - } - return apply_qty_step(cash / (basis * syminfo_.pointvalue)); - } - case QtyType::CASH: - return (std::isfinite(basis) && basis > 0) - ? apply_qty_step((default_qty_value_ / active_account_currency_fx()) / (basis * syminfo_.pointvalue)) : 0.0; - } - return apply_qty_step(default_qty_value_); + const double equity = default_qty_type_ == QtyType::PERCENT_OF_EQUITY + ? percent_commission_live_equity(round_to_mintick(current_bar_.close)) : 0.0; + return calc_default_qty_from_equity(fill_price, equity); } // TradingView freezes DEFAULT (qty=na) market-order sizing at the SIGNAL @@ -3897,6 +3888,55 @@ class BacktestEngine { const execution::LifecycleEffects& lifecycle, const execution::PhysicalExecutionContext& context, execution::CloseScope scope); + execution::Result settle_with_context_selected( + const execution::Action& action, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::PhysicalExecutionContext& context, + const execution::SelectedOpeningSet& selection); + execution::AccountEffectProjection project_with_membership( + const execution::Action& action, + const execution::Fill& fill, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected) const; + execution::SettlementInspection inspect_with_membership( + const execution::Action& action, + const execution::Fill& fill, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected) const; + execution::Result settle_with_membership( + const execution::Action& action, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::PhysicalExecutionContext& context, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected); + struct NativeSettlementStage; + execution::Status validate_native_settlement_book(double& held) const; + execution::Status allocate_native_settlement_closes( + NativeSettlementStage& stage, + const execution::CloseScope& book_or_opening, + double& remaining) const; + void finish_native_settlement_stage( + NativeSettlementStage& stage, const execution::Fill& fill) const; + execution::SettlementInspection inspect_native_settlement_stage( + const NativeSettlementStage& stage, const execution::Fill& fill) const; + execution::AccountEffectProjection project_native_settlement_stage( + const NativeSettlementStage& stage, const execution::Fill& fill) const; + execution::Result commit_native_settlement_stage( + NativeSettlementStage& stage, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::PhysicalExecutionContext& context); + void stage_native_settlement( + NativeSettlementStage& stage, + const execution::Action& action, + const execution::Fill& fill, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected, + const execution::LifecycleEffects* lifecycle) const; + void stage_native_settlement( + NativeSettlementStage& stage, + const execution::ReverseTo& reversal, + const execution::Fill& fill, + const execution::LifecycleEffects* lifecycle) const; enum class PositionReductionCause { SCRIPT_ORDER, // strategy.close / close_all / market exit / reversal BRACKET_EXIT, // a strategy.exit bracket leg fill @@ -4339,6 +4379,10 @@ class BacktestEngine { double active_account_currency_fx() const; void settle_position_after_partial_exit( PositionReductionCause cause); + void restore_source_partial_exit_slots(int pre_count, PositionReductionCause cause); + execution::Result settle_source_opening( + PositionSide requested, double fill_price, double qty, + const std::string& id, const std::string& comment, uint64_t incarnation); void enter_market_from_flat(const std::string& id, bool is_long, double fill_price, double explicit_qty, int explicit_qty_type, @@ -4353,6 +4397,11 @@ class BacktestEngine { PositionSide created_position_side, bool is_priced_entry, uint64_t entry_incarnation); + void add_to_pyramid_market_with_qty_provenance( + const std::string& id, bool is_long, double fill_price, double explicit_qty, + int explicit_qty_type, PositionSide created_position_side, + bool is_priced_entry, bool explicit_qty_prequantized, + uint64_t entry_incarnation); // `fill_price` is already resolved. Source sizing, direction and dust // selection stay here; purge_pending_exits is translated into exact // pending removals for the settlement coordinator. False does not @@ -4378,6 +4427,15 @@ class BacktestEngine { double fill_price, double explicit_qty, int explicit_qty_type, uint64_t entry_incarnation); + void sequential_same_tick_reversal_fill_with_qty_provenance( + const std::string& id, bool is_long, double fill_price, double explicit_qty, + int explicit_qty_type, bool explicit_qty_prequantized, + uint64_t entry_incarnation); + double calc_default_qty_from_equity(double fill_price, double equity) const; + double calc_qty_for_type_from_equity( + double fill_price, double qty_value, int qty_type, double equity) const; + double source_reversal_qty(double fill_price, double explicit_qty, + int explicit_qty_type, bool prequantized) const; void bind_exit_activation(PendingOrder& order); void bind_retained_exit_activations(); void unbind_exit_activations(); diff --git a/include/pineforge/execution_close_scope.hpp b/include/pineforge/execution_close_scope.hpp index 65294d9f2..d46335244 100644 --- a/include/pineforge/execution_close_scope.hpp +++ b/include/pineforge/execution_close_scope.hpp @@ -18,5 +18,8 @@ struct OpeningExposure { using CloseScope = std::variant; +static_assert(std::variant_size_v == 2, + "CloseScope is Book|OpeningExposure; do not add alternatives"); + } // inline namespace close_scope_v1 } // namespace pineforge::execution diff --git a/include/pineforge/execution_close_selection.hpp b/include/pineforge/execution_close_selection.hpp new file mode 100644 index 000000000..1a6e9e894 --- /dev/null +++ b/include/pineforge/execution_close_selection.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace pineforge::execution { +inline namespace close_selection_v1 { + +// Transient membership for one inspect/project/settle call. +// Run-local opening-request provenances in the engine's current cycle. +// Not a CloseScope alternative, not a Request field, not hashed, not stored. +struct SelectedOpeningSet { + std::int64_t cycle = 0; + std::vector incarnations; +}; + +} // inline namespace close_selection_v1 +} // namespace pineforge::execution diff --git a/include/pineforge/execution_projection.hpp b/include/pineforge/execution_projection.hpp new file mode 100644 index 000000000..68abfa45d --- /dev/null +++ b/include/pineforge/execution_projection.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include "execution.hpp" +#include +#include + +namespace pineforge::execution { +inline namespace settlement_projection_v1 { + +// Non-applying physical+account quote of one Action+Fill against the live book. +// Destroyed when the caller is done sizing. Not commit authority. +struct AccountEffectProjection { + Status status = Status::NoEffect; + + // Same physical aggregates SettlementInspection reports for this Action. + double closed_units = 0.0; + double opened_units = 0.0; + double resulting_abs_units = 0.0; + std::size_t resulting_lot_count = 0; + double resulting_abs_notional = 0.0; + double current_ticket = 0.0; + bool would_open = false; + bool incoming_short = false; + + // Account facts SettlementInspection lacks. Units are account currency. + // Copy net_profit_sum_, add each projected close-row pnl in roster order, + // then add initial_capital_. Do not sum rows first or regroup. + double realized_balance = 0.0; + // All paid costs on the projected resulting open book, including a + // prospective opening lot's allocated current-ticket share. 0 when empty. + double remaining_entry_cost = 0.0; + // Start from realized_balance; add each resulting lot's marked pnl minus + // its paid remaining entry cost in roster order at fill.price. Do not + // subtract opening cost twice. + double marked_equity = 0.0; + // 0 if projected book empty. Else the live position_cycle_seq_ when + // survivors remain. If would_open from flat or after emptying the old + // side, the peeked next_position_cycle_seq_ (NOT consumed). + std::int64_t cycle_after = 0; + // Signed resulting units (short negative). 0 when empty. + double signed_units_after = 0.0; +}; + +} // inline namespace settlement_projection_v1 +} // namespace pineforge::execution diff --git a/include/pineforge/execution_reverse_to.hpp b/include/pineforge/execution_reverse_to.hpp new file mode 100644 index 000000000..7f9d0cc70 --- /dev/null +++ b/include/pineforge/execution_reverse_to.hpp @@ -0,0 +1,14 @@ +#pragma once + +namespace pineforge::execution { +inline namespace reverse_to_v1 { + +// One resolved execution closes the opposite live book and opens this exact +// signed exposure. This is not a signed transaction flow or a queued request. +// The value is local to inspection/projection/settlement and is never retained. +struct ReverseTo { + double signed_units = 0.0; +}; + +} // inline namespace reverse_to_v1 +} // namespace pineforge::execution diff --git a/scripts/check_settlement_cpp_abi.py b/scripts/check_settlement_cpp_abi.py new file mode 100644 index 000000000..4dddf4767 --- /dev/null +++ b/scripts/check_settlement_cpp_abi.py @@ -0,0 +1,571 @@ +#!/usr/bin/env python3 +"""Actual old/new settlement ABI pairings. Compile and link; NEVER run callers. + +Requires separately prepared real e60 R2 and 0e R3 archives. No Git/network/build fallback +is performed by this CTest-time checker. Existing native/script ABI guards stay +separate and mandatory, including their old epoch and sanitizer RTTI controls. +""" +from __future__ import annotations +import argparse +import hashlib +import json +from pathlib import Path +import re +import shlex +import shutil +import subprocess +import tempfile + +from check_aggregate_cpp_versions import clean, body +from check_native_cpp_abi import HOST_EVENTS_CALLER, assembly_layout_values +from prepare_settlement_cpp_abi_base import ( + BASE_COMMIT, BASE_TREE, COPY_CACHE, PROVIDERS, authenticate_headers, compiler_identity, + extract_tar, identity, read_cache, run, +) + +ROOT = Path(__file__).resolve().parents[1] +ENGINE = 'pineforge::engine_script_run_v13::BacktestEngine::' +OLD_METHODS = ('inspect_native_settlement', 'inspect_native_settlement_scoped', + 'settle_native_execution_at', 'settle_native_execution_scoped_at', + 'settle_resolved_execution','settle_execution_with_lifecycle','settle_with_context') +OLD_PRIVATE = ('add_to_pyramid_market', 'sequential_same_tick_reversal_fill') +NEW_METHODS = ('inspect_native_settlement_selected', 'settle_native_execution_selected_at', + 'settle_execution_selected_with_lifecycle', 'project_native_settlement_v1', + 'project_native_settlement_scoped_v1', 'project_native_settlement_selected_v1') +NEW_PRIVATE = ('add_to_pyramid_market_with_qty_provenance', + 'sequential_same_tick_reversal_fill_with_qty_provenance') +REVERSAL_METHODS = ('inspect_native_reversal_v1', 'project_native_reversal_v1', + 'settle_native_reversal_at_v1', 'settle_reversal_with_lifecycle_v1') +REVERSAL_DOMAIN = 'reverse_to_v1::ReverseTo' +PRESERVED_ARCHIVE_SHA = 'e13d3d19ad4613c28beddfabb119f1dddadb2c39e474edd6304a2f75c7321f60' +PRESERVED_HEADERS_SHA = '1001102a496ae927ae98e111dd7dc68ab6c23ecc41a9eba00995144d9a532109' +FROZEN_NATIVE_HEADERS = ('native_order.hpp', 'native_order_identity.hpp', 'native_host.hpp', + 'native_run_spec.hpp', 'market_driver.hpp', 'native_calendar.hpp', + 'execution_consumer.hpp') + +COMMON = '''#include +#include +#include +#include +using E = pineforge::engine_script_run_v13::BacktestEngine; +namespace ex = pineforge::execution; +using A = ex::Action; +using F = ex::Fill; +using C = ex::PhysicalExecutionContext; +using S = ex::CloseScope; +using I = ex::SettlementInspection; +using R = ex::Result; +using L = ex::LifecycleEffects; +static_assert(std::is_same_v); +static_assert(std::variant_size_v == 3); +static_assert(std::variant_size_v == 2); +static_assert(std::variant_size_v == 16); +static_assert(std::variant_size_v == 3); +''' +OLD_CALLER = COMMON + ''' +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +auto old_inspect = &E::inspect_native_settlement; +auto old_scoped_inspect = &E::inspect_native_settlement_scoped; +auto old_settle = &E::settle_native_execution_at; +auto old_scoped_settle = &E::settle_native_execution_scoped_at; +int main(int argc, char** argv) { + auto* e = reinterpret_cast(argv); + A a = ex::Flatten{}; F f{100,"","",1}; C c{}; S s = ex::Book{}; + auto i0 = (e->*old_inspect)(a,f); auto i1 = (e->*old_scoped_inspect)(a,f,s); + auto r0 = (e->*old_settle)(a,f,c); auto r1 = (e->*old_scoped_settle)(a,f,c,s); + auto p0=e->settle_resolved_execution(a,f); auto p1=e->settle_execution_with_lifecycle(a,f,L{}); + auto p2=e->settle_with_context(a,f,L{},c); + return int(i0.closed_units+i1.closed_units+r0.closed_units+r1.closed_units+p0.closed_units+p1.closed_units+p2.closed_units); +} +''' +NEW_CALLER = COMMON + ''' +using Set = ex::SelectedOpeningSet; +using P = ex::AccountEffectProjection; +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +int main(int argc, char** argv) { + auto* e = reinterpret_cast(argv); + A a = ex::Flatten{}; F f{100,"","",1}; C c{}; L life{}; Set set{1,{1}}; + auto i = e->inspect_native_settlement_selected(a,f,set); + auto r = e->settle_native_execution_selected_at(a,f,c,set); + auto l = e->settle_execution_selected_with_lifecycle(a,f,life,set); + auto p0 = e->project_native_settlement_v1(a,f); + auto p1 = e->project_native_settlement_scoped_v1(a,f,ex::Book{}); + auto p2 = e->project_native_settlement_selected_v1(a,f,set); + return int(i.closed_units+r.closed_units+l.closed_units+p0.realized_balance+p1.remaining_entry_cost+p2.marked_equity); +} +''' +REVERSAL_CALLER = COMMON + ''' +using RT = ex::reverse_to_v1::ReverseTo; +using P = ex::AccountEffectProjection; +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +int main(int argc, char** argv) { + auto* e = reinterpret_cast(argv); + RT target{0.1}; F f{100,"","",1}; C c{}; L life{}; + auto i = e->inspect_native_reversal_v1(target,f); + auto p = e->project_native_reversal_v1(target,f); + auto r = e->settle_native_reversal_at_v1(target,f,c); + auto l = e->settle_reversal_with_lifecycle_v1(target,f,life); + return int(i.closed_units+p.realized_balance+r.closed_units+l.closed_units); +} +''' +PRIVATE_OLD_CALLER = COMMON + ''' +using Add = void(E::*)(const std::string&,bool,double,double,int,pineforge::PositionSide,bool,uint64_t); +using Seq = void(E::*)(const std::string&,bool,double,double,int,uint64_t); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +auto old_add = &E::add_to_pyramid_market; +auto old_sequential = &E::sequential_same_tick_reversal_fill; +int main(int argc,char** argv) { + auto* e = reinterpret_cast(argv); + (e->*old_add)("old",true,100,1,-1,pineforge::PositionSide::LONG,false,1); + (e->*old_sequential)("old",true,100,1,-1,1); + return argc; +} +''' +PRIVATE_NEW_CALLER = COMMON + ''' +using Add = void(E::*)(const std::string&,bool,double,double,int,pineforge::PositionSide,bool,bool,uint64_t); +using Seq = void(E::*)(const std::string&,bool,double,double,int,bool,uint64_t); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +int main(int argc,char** argv) { + auto* e = reinterpret_cast(argv); + e->add_to_pyramid_market_with_qty_provenance("new",true,100,1,-1,pineforge::PositionSide::LONG,false,true,1); + e->sequential_same_tick_reversal_fill_with_qty_provenance("new",true,100,1,-1,true,1); + return argc; +} +''' + +FIELDS = { + 'Result': [('status','ex::Status'),('closed_units','double'),('opened_units','double'), + ('current_ticket','double'),('first_trade_index','std::size_t'), + ('closed_trade_count','std::size_t'),('opened_lot_incarnation','uint64_t')], + 'SettlementInspection': [('status','ex::Status'),('closed_units','double'),('opened_units','double'), + ('resulting_abs_units','double'),('resulting_lot_count','std::size_t'), + ('resulting_abs_notional','double'),('current_ticket','double'),('would_open','bool'),('incoming_short','bool')], +} + + +def normalized(text: str) -> str: + return re.sub(r'\s+', ' ', clean(text)).strip() + + +def storage_declarations(header: str) -> list[str]: + """Read top-level named data declarations; skip method bodies/declarations. + + Actual compiler offsets/types below supplement this source-order guard. + String contents are irrelevant to storage declarations and may contain braces. + """ + text = re.sub(r'"(?:\\.|[^"\\])*"', '""', clean(header)) + text = body(text, r'class\s+BacktestEngine\s*\{', 'BacktestEngine') + statements, start, depth = [], 0, 0 + for at, char in enumerate(text): + if char == '{': + depth += 1 + elif char == '}': + depth -= 1 + if depth == 0: + prefix = text[start:at + 1] + before = prefix.split('{', 1)[0] + # Function definition (including const/noexcept) is not storage. + if ')' in before and '=' not in before: + start = at + 1 + elif char == ';' and depth == 0: + segment = text[start:at] + start = at + 1 + segment = re.sub(r'\b(?:public|protected|private)\s*:', '', segment).strip() + prefix = segment.split('=', 1)[0].split('{', 1)[0].strip() + if '(' in prefix or re.match(r'^(?:using|typedef|friend|static_assert|class|struct|enum)\b', prefix): + continue + if re.search(r'\b[A-Za-z_]\w*\s*(?:\[[^]]*\])?\s*$', prefix): + statements.append(re.sub(r'\s+', ' ', prefix)) + if len(statements) < 100: + raise RuntimeError('engine storage inventory unexpectedly small; inspect source parser before accepting ABI') + return statements + + +def frozen_shape(old_include: Path, current_include: Path, *, selected=False) -> tuple[list[str], dict]: + old_exec = (old_include/'pineforge/execution.hpp').read_text() + cur_exec = (current_include/'pineforge/execution.hpp').read_text() + actions = [re.search(r'using\s+Action\s*=\s*[^;]+;', clean(text)) for text in (old_exec,cur_exec)] + if any(alias is None for alias in actions) or normalized(actions[0].group()) != normalized(actions[1].group()): + raise RuntimeError('Action alternative identities/order changed') + for name in ['Result','SettlementInspection','Status','Fill','PhysicalExecutionContext','LifecycleEffects']: + pattern = r'(?:struct|enum\s+class)\s+' + name + r'\s*\{' + if normalized(body(old_exec, pattern, name)) != normalized(body(cur_exec, pattern, name)): + raise RuntimeError('frozen execution aggregate/enum changed: ' + name) + for name in FROZEN_NATIVE_HEADERS: + if normalized((old_include/'pineforge'/name).read_text()) != normalized((current_include/'pineforge'/name).read_text()): + raise RuntimeError('R3 must preserve native header layout/contracts: ' + name) + if selected: + for name in ('execution_close_selection.hpp', 'execution_projection.hpp'): + if normalized((old_include/'pineforge'/name).read_text()) != normalized((current_include/'pineforge'/name).read_text()): + raise RuntimeError('selected/projection header layout/contracts changed: ' + name) + scopes = [(directory/'pineforge/execution_close_scope.hpp').read_text() + for directory in (old_include,current_include)] + for name in ['Book','OpeningExposure']: + pattern = r'struct\s+' + name + r'\s*\{' + if normalized(body(scopes[0],pattern,name)) != normalized(body(scopes[1],pattern,name)): + raise RuntimeError('frozen CloseScope alternative changed: ' + name) + aliases = [re.search(r'using\s+CloseScope\s*=\s*[^;]+;',clean(text)) for text in scopes] + if any(alias is None for alias in aliases) or normalized(aliases[0].group()) != normalized(aliases[1].group()): + raise RuntimeError('CloseScope alternative identities/order changed') + old_engine = (old_include/'pineforge/engine.hpp').read_text() + cur_engine = (current_include/'pineforge/engine.hpp').read_text() + old_storage, current_storage = storage_declarations(old_engine), storage_declarations(cur_engine) + if old_storage != current_storage: + raise RuntimeError('engine named data declarations/order changed') + virtuals = lambda text: re.findall(r'\bvirtual\b[^;{]*(?:;|\{)', clean(text)) + if [normalized(v) for v in virtuals(old_engine)] != [normalized(v) for v in virtuals(cur_engine)]: + raise RuntimeError('engine virtual method inventory changed') + members = [re.search(r'\b([A-Za-z_]\w*)\s*(?:\[[^]]*\])?\s*$', declaration).group(1) + for declaration in old_storage if not declaration.startswith('static ')] + return members, {'engineStorage': old_storage, 'virtuals': [normalized(v) for v in virtuals(old_engine)]} + + +def layout_source(members: list[str], *, selected=False) -> tuple[str, int]: + values = [] + assertions = [] + for name in ['Result','SettlementInspection']: + values += [f'sizeof(ex::{name})',f'alignof(ex::{name})'] + for field, expected_type in FIELDS[name]: + assertions.append(f'static_assert(std::is_same_v);') + values.append(f'offsetof(ex::{name},{field})') + for index, status in enumerate(['Applied','NoEffect','InvalidPrice','InvalidQuantity','InvalidBook', + 'UnrepresentableQuantity','InvalidAccounting','InvalidLifecycle','InvalidCloseTarget']): + assertions.append(f'static_assert(int(ex::Status::{status})=={index});') + values += ['sizeof(ex::Status)','sizeof(A)','alignof(A)','std::variant_size_v', + 'sizeof(S)','alignof(S)','std::variant_size_v', + 'sizeof(E)','alignof(E)','sizeof(pineforge::PendingOrder)', + 'sizeof(pineforge::NativeStrategyHost)','sizeof(pineforge::NativeMarketEvent)', + 'sizeof(pineforge::NativeStateView)','sizeof(pineforge::native_order::Request)', + 'sizeof(pineforge::native_order::WorkingRequestCore)', + 'sizeof(pineforge::native_order::CommandEvent)'] + if selected: + values += ['sizeof(ex::SelectedOpeningSet)', 'alignof(ex::SelectedOpeningSet)', + 'sizeof(ex::AccountEffectProjection)', 'alignof(ex::AccountEffectProjection)'] + for member in members: + values += [f'offsetof(E,{member})', f'sizeof(decltype(E::{member}))', f'alignof(decltype(E::{member}))'] + return COMMON + '\n'.join(assertions) + '\nextern "C" const unsigned long long abi_layout[] = {\n' + ',\n'.join(values) + '\n};\n', len(values) + + +def defined_symbols(library: Path) -> str: + raw = run(['nm','-g','-C',str(library)]).decode('utf-8','replace') + return '\n'.join(line for line in raw.splitlines() if re.search(r'\b[TWtw]\s+', line)) + + +def validate_rejection(diagnostic: str, missing, domain=None) -> list[str]: + symbols = re.findall(r'^\s*"(.+)", referenced from:', diagnostic, re.M) + symbols += re.findall(r"undefined reference to [`'](.+)'", diagnostic) + symbols += re.findall(r'undefined symbol:\s*(.+)', diagnostic) + if not symbols: + raise RuntimeError('link failure has no recognized undefined-symbol diagnostics') + for method in missing: + matching = [symbol for symbol in symbols if symbol.startswith(ENGINE+method+'(')] + if not matching: + raise RuntimeError('link failure omits expected undefined method: '+method) + if domain and ('selected' in method or 'reversal' in method) and any(domain not in symbol for symbol in matching): + raise RuntimeError('method has wrong/missing parameter namespace: '+method) + unrelated = [symbol for symbol in symbols if not any(symbol.startswith(ENGINE+method+'(') for method in missing)] + if unrelated: + raise RuntimeError('link failure includes unrelated undefined symbols: '+', '.join(unrelated)) + return symbols + + +def load_base(args, destination: Path, current_cache: dict) -> tuple[Path, Path, Path, dict]: + return load_provider(args, destination, current_cache, args.base_receipt, PROVIDERS['e60'], + expect_present=(*OLD_METHODS,*OLD_PRIVATE), + expect_absent=(*NEW_METHODS,*NEW_PRIVATE,*REVERSAL_METHODS)) + + +def load_prior(args, destination: Path, current_cache: dict) -> tuple[Path, Path, Path, dict]: + return load_provider(args, destination, current_cache, args.prior_receipt, PROVIDERS['0e'], + expect_present=(*OLD_METHODS,*OLD_PRIVATE,*NEW_METHODS,*NEW_PRIVATE), + expect_absent=REVERSAL_METHODS) + + +def load_provider(args, destination: Path, current_cache: dict, receipt_path: Path | None, + provider: dict, *, expect_present, expect_absent) -> tuple[Path, Path, Path, dict]: + is_base = provider['commit'] == BASE_COMMIT + label = 'R2' if is_base else '0e18690' + if receipt_path is None or not receipt_path.is_file(): + if not is_base: + raise RuntimeError('real 0e18690 archive receipt missing; run scripts/prepare_settlement_cpp_abi_base.py ' + '--source-repo . --current-build BUILD --output BUILD/settlement-abi-prior ' + f'--commit {provider["commit"]} --tree {provider["tree"]} ' + '--header-manifest tests/fixtures/settlement_cpp_abi/0e18690/manifest.json before CTest') + raise RuntimeError('real R2 archive receipt missing; run scripts/prepare_settlement_cpp_abi_base.py ' + '--source-repo . --current-build BUILD --output BUILD/settlement-abi-base before CTest') + receipt = json.loads(receipt_path.read_text()) + if receipt.get('commit') != provider['commit'] or receipt.get('tree') != provider['tree']: + raise RuntimeError('real base receipt does not pin ' + ('e60 R2' if is_base else label)) + base_root = receipt_path.resolve().parent + resolve_artifact = lambda name: Path(name) if Path(name).is_absolute() else base_root/name + library = resolve_artifact(receipt['archive']) + headers = resolve_artifact(receipt.get('headers',provider['headers_name'])) + if identity(library)['sha256'] != receipt['archiveSha256'] or identity(headers)['sha256'] != receipt['headersSha256']: + raise RuntimeError('real ' + label + ' archive/header bytes do not match receipt') + if not library.read_bytes().startswith(b'!\n'): + raise RuntimeError('real base is not a static archive') + if len(run(['ar','-t',str(library)]).splitlines()) < 20: + raise RuntimeError('real base is not a full product archive') + extract_tar(headers.read_bytes(), destination) + authenticate_headers(destination, provider['manifest'], commit=provider['commit'], tree=provider['tree']) + if receipt.get('schemaVersion') == 'pineforge-settlement-abi-base/v1': + old_compiler = receipt['compiler'] + current_compiler = compiler_identity(args.compiler) + if any(old_compiler[key] != current_compiler[key] for key in ('target','sha256','version')): + raise RuntimeError('base compiler implementation/version/target differs; prepare base on this runner') + actual_settings = {key: current_cache[key] for key in COPY_CACHE if key in current_cache} + if receipt['copiedCurrentCache'] != actual_settings: + raise RuntimeError('base preparation compiler/configuration differs from current build; prepare a matching base') + generated = resolve_artifact(receipt['generatedInclude']) + if identity(generated/'pineforge/version.h')['sha256'] != receipt['generatedHeaderSha256']: + raise RuntimeError('base generated version header changed') + else: + if not is_base: + raise RuntimeError('0e18690 provider requires portable v1 receipt; use matching preparation') + # Explicit reuse of root's preserved Mac Release artifact, never a stub. + if receipt['archiveSha256'] != PRESERVED_ARCHIVE_SHA or receipt['headersSha256'] != PRESERVED_HEADERS_SHA: + raise RuntimeError('unrecognized legacy base receipt; use portable preparation') + if args.extra_flag or current_cache.get('PINEFORGE_ENABLE_SANITIZERS') == 'ON' or current_cache.get('PINEFORGE_ENABLE_COVERAGE') == 'ON': + raise RuntimeError('preserved uninstrumented R2 archive is not this lane; prepare a matching real base') + if not args.base_generated_include: + raise RuntimeError('preserved base requires --base-generated-include from its original build') + generated = args.base_generated_include.resolve() + identity(generated/'pineforge/version.h') + old_cache=read_cache(generated.parent/'CMakeCache.txt') + old_settings={key:old_cache[key] for key in COPY_CACHE if key in old_cache} + current_settings={key:current_cache[key] for key in COPY_CACHE if key in current_cache} + if old_settings != current_settings: + raise RuntimeError('preserved R2 build configuration differs; use matching portable preparation') + if identity(generated.parent/'lib/libpineforge.a')['sha256'] != receipt['archiveSha256']: + raise RuntimeError('preserved base cache/header directory no longer belongs to its archive') + symbols = defined_symbols(library) + for method in expect_present: + if ENGINE + method + '(' not in symbols: + raise RuntimeError('real old archive omits original symbol: ' + method) + for method in expect_absent: + if ENGINE + method + '(' in symbols: + raise RuntimeError('supplied old archive already exports new method: ' + method) + return library, destination/'include', generated, receipt + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--compiler', required=True) + parser.add_argument('--library', type=Path, required=True) + parser.add_argument('--include', type=Path, required=True) + parser.add_argument('--generated-include', type=Path, required=True) + parser.add_argument('--base-receipt', type=Path, required=True) + parser.add_argument('--prior-receipt', type=Path, + help='required for full proof: prepared real 0e18690 provider receipt') + parser.add_argument('--base-generated-include', type=Path) + parser.add_argument('--extra-flag', action='append', default=[]) + parser.add_argument('--receipt', type=Path, required=True) + parser.add_argument('--base-only', action='store_true', help='preparation control only; does not claim new API proof') + parser.add_argument('--old-rejections-only', action='store_true', + help='new public header callers versus actual old provider only; no current archive acceptance') + parser.add_argument('--public-only', action='store_true', + help='complete public API matrix before new private helpers are integrated; partial R3 proof') + args = parser.parse_args() + library, include = args.library.resolve(), args.include.resolve() + source = include.parent + cache = read_cache(library.parent.parent/'CMakeCache.txt') + if compiler_identity(cache['CMAKE_CXX_COMPILER'])['sha256'] != compiler_identity(args.compiler)['sha256']: + raise RuntimeError('--compiler differs from current archive build compiler') + if cache.get('PINEFORGE_ENABLE_SANITIZERS') == 'ON' and '-fsanitize=address,undefined' not in args.extra_flag: + raise RuntimeError('sanitizer archive requires --extra-flag=-fsanitize=address,undefined') + source_files = [source/'CMakeLists.txt', *[p for root in ('src','include','cmake') for p in (source/root).rglob('*') if p.is_file()]] + stale = [str(path.relative_to(source)) for path in source_files if path.stat().st_mtime > library.stat().st_mtime] + if sum((args.base_only,args.old_rejections_only,args.public_only)) > 1: + raise RuntimeError('select only one partial proof mode') + full_matrix = not (args.base_only or args.old_rejections_only or args.public_only) + if stale and not args.old_rejections_only: + raise RuntimeError('current archive predates source; full rebuild required: '+', '.join(stale)) + initial_identity = identity(library) + flags = shlex.split(cache.get('CMAKE_CXX_FLAGS','')) + common = [args.compiler,'-std=c++17',*flags,'-O0','-UNDEBUG','-ffp-contract=off', + '-fno-access-control','-Wno-invalid-'+'offsetof',*args.extra_flag] + if cache.get('PINEFORGE_ENABLE_SANITIZERS') == 'ON': + common += ['-DEIGEN_MAX_ALIGN_BYTES=0'] + args.receipt.parent.mkdir(parents=True, exist_ok=True) + log_root = Path(tempfile.mkdtemp(prefix=args.receipt.stem+'.artifacts-',dir=args.receipt.parent)) + report = {'schemaVersion':'pineforge-settlement-abi/v1','status':'incomplete', + 'mode':'base-only' if args.base_only else 'old-rejections-only' if args.old_rejections_only else 'public-only' if args.public_only else 'full-new-old-matrix', + 'currentArchive':{'path':str(library),**initial_identity}, + 'compiler':compiler_identity(args.compiler),'callerFlags':common[1:], + 'privateAccessScaffolding':'-fno-access-control only for ABI TUs; product visibility unchanged', + 'artifactsDirectory':str(log_root),'newArchivePairingComplete':False, + 'sourceSha256':{str(path.relative_to(source)):identity(path)['sha256'] for path in source_files}, + 'compiles':[],'links':[],'executedBinaries':0,'networkFetches':0} + try: + with tempfile.TemporaryDirectory(prefix='pineforge-settlement-abi-') as temporary: + scratch = Path(temporary) + old_library,old_include,old_generated,old_receipt = load_base(args,scratch/'r2',cache) + report['base']={'receiptSha256':identity(args.base_receipt)['sha256'], + 'archiveSha256':identity(old_library)['sha256'], + 'commit':BASE_COMMIT,'tree':BASE_TREE} + members,shape = frozen_shape(old_include,include) + report['frozenShape']=shape + if full_matrix: + prior_library,prior_include,prior_generated,prior_receipt = load_prior(args,scratch/'prior',cache) + report['prior']={'receiptSha256':identity(args.prior_receipt)['sha256'], + 'archiveSha256':identity(prior_library)['sha256'], + 'commit':prior_receipt['commit'],'tree':prior_receipt['tree']} + prior_members,prior_shape = frozen_shape(prior_include,include,selected=True) + report['priorFrozenShape']=prior_shape + + def compile_tu(name,text,headers,generated): + path=log_root/(name+'.cpp');path.write_text(text) + obj=log_root/(name+'.o') + argv=[*common,'-I',str(headers),'-I',str(generated),'-c',str(path),'-o',str(obj)] + run(argv,timeout=120,log=log_root/(name+'.compile.log')) + report['compiles'].append({'name':name,'argv':argv,'sourceSha256':identity(path)['sha256'],'objectSha256':identity(obj)['sha256']}) + return obj + + def link(name,obj,runtime,missing=(),domain=None): + argv=[*common,str(obj),str(runtime),'-pthread','-o',str(log_root/name)] + result=subprocess.run(argv,capture_output=True,text=True,timeout=120) + diagnostic=result.stdout+result.stderr + (log_root/(name+'.link.log')).write_text(diagnostic) + if not missing: + if result.returncode: raise RuntimeError(name+' positive pair failed:\n'+diagnostic) + else: + if result.returncode == 0: raise RuntimeError(name+' unexpectedly linked') + validate_rejection(diagnostic,missing,domain) + report['links'].append({'name':name,'argv':argv,'exitCode':result.returncode, + 'outcome':'expected-rejection' if missing else 'linked','requiredMissing':list(missing), + 'selectionDomain':domain,'parameterDomain':domain,'executed':False}) + + def compare_layout(name,headers,generated,layout_members,*,selected=False): + layout_text,word_count=layout_source(layout_members,selected=selected) + layouts=[] + for label,layout_headers,layout_generated in [(name,headers,generated), + ('current' if name == 'old' else 'current-selected',include,args.generated_include)]: + compile_tu(label+'-layout',layout_text,layout_headers,layout_generated) + src=log_root/(label+'-layout.cpp');asm=log_root/(label+'-layout.s') + run([*common,'-I',str(layout_headers),'-I',str(layout_generated),'-S',str(src),'-o',str(asm)],timeout=120, + log=log_root/(label+'-layout.assembly.log')) + layouts.append(assembly_layout_values(asm.read_text(),word_count)) + if layouts[0]!=layouts[1]: + raise RuntimeError('actual compiler '+name+'/current layout/offset/type-size arrays differ') + return {'wordCount':word_count,'values':layouts[0],'members':layout_members} + + # Compile every actual caller before interpreting any link outcome. + old=compile_tu('old-book-singleton',OLD_CALLER,old_include,old_generated) + cur_old=compile_tu('current-old-member-types',OLD_CALLER,include,args.generated_include) + private_old=compile_tu('old-private-f8-f11',PRIVATE_OLD_CALLER,old_include,old_generated) + current_private_old=compile_tu('current-old-private-types',PRIVATE_OLD_CALLER,include,args.generated_include) + old_events=compile_tu('old-host-events-return',HOST_EVENTS_CALLER,old_include,old_generated) + current_events=compile_tu('current-host-events-return',HOST_EVENTS_CALLER,include,args.generated_include) + report['layout']=compare_layout('old',old_include,old_generated,members) + if full_matrix: + report['priorLayout']=compare_layout('prior',prior_include,prior_generated,prior_members,selected=True) + if not args.base_only: + current_header=clean((include/'pineforge/engine.hpp').read_text()) + for method in (*NEW_METHODS,*(REVERSAL_METHODS if full_matrix else ())): + match=re.search(r'[^;{}]*\b'+method+r'\s*\([^;{}]*;',current_header) + if not match or re.search(r'\bvirtual\b',match.group()): + raise RuntimeError('new method must have one nonvirtual declaration: '+method) + preceding=current_header[:match.start()] + access=re.findall(r'\b(public|protected|private)\s*:',preceding) + if not access or access[-1]!='protected': + raise RuntimeError('new method must stay protected: '+method) + new=compile_tu('new-six-methods',NEW_CALLER,include,args.generated_include) + if not args.old_rejections_only and not args.public_only: + private_new=compile_tu('new-private-provenance',PRIVATE_NEW_CALLER,include,args.generated_include) + wrong_project=scratch/'wrong-project';shutil.copytree(include,wrong_project) + header=wrong_project/'pineforge/engine.hpp';changed=header.read_text();wrong_source=NEW_CALLER + wrong_names=[] + for method in NEW_METHODS[3:]: + future=method[:-2]+'v2';wrong_names.append(future) + changed=changed.replace(method,future);wrong_source=wrong_source.replace(method,future) + header.write_text(changed) + projection_header=wrong_project/'pineforge/execution_projection.hpp' + projection_header.write_text(projection_header.read_text().replace('settlement_projection_v1','settlement_projection_v2')) + wrong_source=wrong_source.replace('settlement_projection_v1','settlement_projection_v2') + wrong_projection=compile_tu('synthetic-future-project-v2',wrong_source,wrong_project,args.generated_include) + wrong_selection=scratch/'wrong-selection';shutil.copytree(include,wrong_selection) + header=wrong_selection/'pineforge/execution_close_selection.hpp' + changed=header.read_text();require_name='close_selection_v1' + if require_name not in changed: raise RuntimeError('selection namespace missing from real header') + header.write_text(changed.replace(require_name,'close_selection_v2')) + wrong_set=compile_tu('synthetic-selection-v2',NEW_CALLER.replace(require_name,'close_selection_v2'),wrong_selection,args.generated_include) + if full_matrix: + prior_selected=compile_tu('prior-selected-project',NEW_CALLER,prior_include,prior_generated) + reversal=compile_tu('new-four-reversal-methods',REVERSAL_CALLER,include,args.generated_include) + wrong_reversal=scratch/'wrong-reversal';shutil.copytree(include,wrong_reversal) + header=wrong_reversal/'pineforge/engine.hpp';changed=header.read_text();wrong_source=REVERSAL_CALLER + wrong_reversal_names=[] + for method in REVERSAL_METHODS: + future=method[:-2]+'v2';wrong_reversal_names.append(future) + changed=changed.replace(method,future);wrong_source=wrong_source.replace(method,future) + header.write_text(changed) + wrong_reversal_methods=compile_tu('synthetic-reversal-method-v2',wrong_source,wrong_reversal,args.generated_include) + wrong_target=scratch/'wrong-target';shutil.copytree(include,wrong_target) + header=wrong_target/'pineforge/execution_reverse_to.hpp';changed=header.read_text() + if 'reverse_to_v1' not in changed: + raise RuntimeError('reversal namespace missing from real header') + header.write_text(changed.replace('reverse_to_v1','reverse_to_v2')) + wrong_reverse_to=compile_tu('synthetic-reverse-to-v2',REVERSAL_CALLER.replace('reverse_to_v1','reverse_to_v2'),wrong_target,args.generated_include) + link('old-api-old-real',old,old_library) + link('old-private-old-real',private_old,old_library) + link('old-events-old-real',old_events,old_library) + if not args.old_rejections_only: + link('old-api-new-real',old,library) + link('current-old-api-new-real',cur_old,library) + link('old-private-new-real',private_old,library) + link('current-old-private-new-real',current_private_old,library) + link('old-events-new-real',old_events,library) + link('current-events-new-real',current_events,library) + if not args.base_only: + link('new-api-old-real-rejected',new,old_library,NEW_METHODS,'close_selection_v1::SelectedOpeningSet') + if not args.old_rejections_only: + link('new-api-new-real',new,library) + if not args.public_only: + link('new-private-new-real',private_new,library) + link('new-private-old-real-rejected',private_new,old_library,NEW_PRIVATE) + link('synthetic-project-v2-rejected',wrong_projection,library,wrong_names) + link('synthetic-selection-v2-rejected',wrong_set,library, + (NEW_METHODS[0],NEW_METHODS[1],NEW_METHODS[2],NEW_METHODS[5]),'close_selection_v2::SelectedOpeningSet') + if full_matrix: + link('prior-selected-prior-real',prior_selected,prior_library) + link('prior-selected-new-real',prior_selected,library) + link('current-selected-prior-real',new,prior_library) + link('new-reversal-new-real',reversal,library) + link('new-reversal-prior-real-rejected',reversal,prior_library,REVERSAL_METHODS,REVERSAL_DOMAIN) + link('synthetic-reversal-method-v2-rejected',wrong_reversal_methods,library,wrong_reversal_names,REVERSAL_DOMAIN) + link('synthetic-reverse-to-v2-rejected',wrong_reverse_to,library,REVERSAL_METHODS,'reverse_to_v2::ReverseTo') + if identity(library)!=initial_identity: raise RuntimeError('current archive changed during ABI proof') + if any(identity(source/name)['sha256']!=sha for name,sha in report['sourceSha256'].items()): + raise RuntimeError('source changed during ABI proof') + report['status']='base-control-only' if args.base_only else 'old-rejections-only' if args.old_rejections_only else 'public-matrix-only' if args.public_only else 'passed' + report['newArchivePairingComplete']=report['status']=='passed' + except Exception as error: + report['status']='failed';report['error']=str(error) + raise + finally: + args.receipt.write_text(json.dumps(report,indent=2,sort_keys=True)+'\n') + print(f"settlement ABI: {report['status']}; {len(report['compiles'])} compiles, {len(report['links'])} links, 0 executed binaries") + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/ci_verify.py b/scripts/ci_verify.py new file mode 100644 index 000000000..7dbe0d417 --- /dev/null +++ b/scripts/ci_verify.py @@ -0,0 +1,728 @@ +#!/usr/bin/env python3 +"""Shared local/CI verification driver. Stdlib only. Not a command generator. + +Profiles: release, debug, sanitizers, native. Default build dir build-ci-PROFILE. +Source guards, explicit configure, full rebuild, pinned e60 and 0e ABI prepare/reuse, +CTest, install+find_package+VERSION smoke, native help / required WebSocket. +Fail fast on configure/build. After a successful build collect independent +CTest and package failures in the same run. Never deletes source, tests, or +build trees; a mismatched ABI base is a refusal to a fresh --build-dir. +""" +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import sys +import time +from typing import Callable + +from prepare_settlement_cpp_abi_base import ( + BASE_COMMIT, + BASE_TREE, + PROVIDERS, + read_cache, + reusable_prepared_base, +) + +ROOT = Path(__file__).resolve().parents[1] +PROFILES = ('release', 'debug', 'sanitizers', 'native') +DEFAULT_JOBS = 4 +JOBS_MIN, JOBS_MAX = 1, 64 +SCHEMA = 'pineforge-ci-verify/v1' +SANITIZER_FLAG = '-fsanitize=address,undefined' +SANITIZER_RUN_ENV = { + 'ASAN_OPTIONS': 'detect_leaks=1:halt_on_error=1:abort_on_error=1', + 'UBSAN_OPTIONS': 'print_stacktrace=1:halt_on_error=1', +} +SOURCE_GUARD_SCRIPTS = ( + ('source-guard-c-abi', ['scripts/check_c_abi_runtime.py']), + ('source-guard-broker-hash', ['scripts/check_broker_state_hash_coverage.py']), + ('source-guard-pending-mirror', ['scripts/gen_pending_order_mirror.py', '--check']), + ('source-guard-native-versions', ['scripts/check_native_cpp_versions.py']), + ('source-guard-aggregate-versions', ['scripts/check_aggregate_cpp_versions.py']), +) + + +class ConfigError(Exception): + """CLI / profile validation; exit status 2.""" + + +@dataclass(frozen=True) +class Profile: + name: str + build_type: str + sanitizers: bool + live_runner: bool + tutorial: bool + + +PROFILE = { + 'release': Profile('release', 'Release', False, False, True), + 'debug': Profile('debug', 'Debug', False, False, True), + 'sanitizers': Profile('sanitizers', 'Debug', True, False, True), + 'native': Profile('native', 'Release', False, True, False), +} + + +@dataclass +class Completed: + returncode: int + stdout: bytes = b'' + stderr: bytes = b'' + + +Runner = Callable[..., Completed] + + +@dataclass +class VerifyConfig: + profile: Profile + source: Path + build_dir: Path + jobs: int + generator: str | None + curl_dir: Path | None + ccache_path: str | None + require_websocket: bool + runner: Runner + stream_output: bool = True + + +class Parser(argparse.ArgumentParser): + def error(self, message: str) -> None: + raise ConfigError(message) + + +def cmake_on(value: str | None) -> bool: + return (value or '').strip().upper() in {'1', 'ON', 'TRUE', 'YES', 'Y'} + + +def expected_version(source: Path) -> str: + return ''.join((source / 'VERSION').read_text().split()) + + +def default_build_dir(source: Path, profile: str) -> Path: + return source / f'build-ci-{profile}' + + +def call_runner(runner: Runner, argv: list[str], *, extra_env: dict[str, str] | None = None, + timeout: int = 600, combine_stderr: bool = True, + stream_output: bool = False) -> Completed: + return runner(argv, extra_env=extra_env, timeout=timeout, + combine_stderr=combine_stderr, stream_output=stream_output) + + +def ctest_supports_junit(runner: Runner) -> bool: + try: + result = call_runner(runner, ['ctest', '--help'], timeout=30, stream_output=False) + except (OSError, subprocess.TimeoutExpired): + return False + return '--output-junit' in (result.stdout + result.stderr).decode('utf-8', 'replace') + + +def source_guard_commands(source: Path) -> list[tuple[str, list[str]]]: + python = sys.executable + return [(name, [python, str(source / rel[0]), *rel[1:]]) for name, rel in SOURCE_GUARD_SCRIPTS] + + +def cmake_cache_definitions(cfg: VerifyConfig) -> dict[str, str]: + profile = cfg.profile + values = { + 'CMAKE_BUILD_TYPE': profile.build_type, + 'CMAKE_EXPORT_COMPILE_COMMANDS': 'ON', + 'PINEFORGE_BUILD_TESTS': 'ON', + 'PINEFORGE_BUILD_TUTORIAL': 'ON' if profile.tutorial else 'OFF', + 'PINEFORGE_BUILD_LIVE_RUNNER': 'ON' if profile.live_runner else 'OFF', + 'PINEFORGE_ENABLE_SANITIZERS': 'ON' if profile.sanitizers else 'OFF', + 'PINEFORGE_BUILD_CORPUS_STRATEGIES': 'OFF', + 'PINEFORGE_BUILD_BENCH_STRATEGIES': 'OFF', + 'PINEFORGE_BUILD_SPEED_BENCH': 'OFF', + 'PINEFORGE_BUILD_EXAMPLES': 'OFF', + 'PINEFORGE_ENABLE_COVERAGE': 'OFF', + 'PINEFORGE_STRICT_WARNINGS': 'OFF', + 'PINEFORGE_VERSION_SOURCE': 'FILE', + } + if cfg.curl_dir is not None: + values['CURL_DIR'] = str(cfg.curl_dir) + if cfg.ccache_path: + values['CMAKE_C_COMPILER_LAUNCHER'] = cfg.ccache_path + values['CMAKE_CXX_COMPILER_LAUNCHER'] = cfg.ccache_path + return values + + +def cmake_configure_argv(cfg: VerifyConfig) -> list[str]: + argv = ['cmake', '-S', str(cfg.source), '-B', str(cfg.build_dir)] + if cfg.generator: + argv += ['-G', cfg.generator] + argv += [f'-D{key}={value}' for key, value in cmake_cache_definitions(cfg).items()] + return argv + + +def parse_args(argv: list[str] | None, *, source: Path = ROOT) -> argparse.Namespace: + parser = Parser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + 'profile', choices=PROFILES, + help='release/debug keep tutorial ON and native OFF; ' + 'sanitizers enable PUBLIC ASan/UBSan; native enables the live runner') + parser.add_argument('--build-dir', type=Path, default=None, + help='default: /build-ci-PROFILE') + parser.add_argument('--jobs', type=int, default=DEFAULT_JOBS) + parser.add_argument('--generator', default=None) + parser.add_argument('--curl-dir', type=Path, default=None, + help='CURL CMake package dir (native WS-enabled curl install)') + parser.add_argument('--ccache', action='store_true', + help='require installed ccache and bind CMAKE_*_COMPILER_LAUNCHER') + parser.add_argument('--require-websocket', action='store_true', + help='native only: execute test_native_live_websocket and refuse skip (77)') + args = parser.parse_args(argv) + if args.build_dir is None: + args.build_dir = default_build_dir(source, args.profile) + return args + + +def validate_config(args: argparse.Namespace, *, source: Path = ROOT, + which: Callable[[str], str | None] = shutil.which) -> VerifyConfig: + if not JOBS_MIN <= args.jobs <= JOBS_MAX: + raise ConfigError(f'--jobs must be {JOBS_MIN}..{JOBS_MAX}') + if args.require_websocket and args.profile != 'native': + raise ConfigError('--require-websocket is only valid with the native profile') + if args.curl_dir is not None and not args.curl_dir.is_dir(): + raise ConfigError(f'--curl-dir is not a directory: {args.curl_dir}') + ccache_path = None + if args.ccache: + found = which('ccache') + if not found: + raise ConfigError('optional --ccache requires an installed ccache tool') + ccache_path = str(Path(found).resolve()) + for tool in ('cmake', 'ctest', 'git'): + if not which(tool): + raise ConfigError(f'required tool not found: {tool}') + build_dir = args.build_dir.expanduser() + build_dir = build_dir.resolve() if build_dir.is_absolute() else (Path.cwd() / build_dir).resolve() + source = source.resolve() + if build_dir == source: + raise ConfigError('--build-dir cannot be the source root') + return VerifyConfig( + profile=PROFILE[args.profile], + source=source, + build_dir=build_dir, + jobs=args.jobs, + generator=args.generator, + curl_dir=args.curl_dir.resolve() if args.curl_dir is not None else None, + ccache_path=ccache_path, + require_websocket=bool(args.require_websocket), + runner=default_runner, + ) + + +def default_runner(argv: list[str], *, extra_env: dict[str, str] | None = None, + timeout: int = 600, combine_stderr: bool = True, + stream_output: bool = True) -> Completed: + env = os.environ.copy() + if extra_env: + env.update(extra_env) + stderr = subprocess.STDOUT if combine_stderr else subprocess.PIPE + try: + result = subprocess.run( + argv, stdout=subprocess.PIPE, stderr=stderr, env=env, timeout=timeout) + except subprocess.TimeoutExpired as error: + out = error.stdout or b'' + err = b'' if combine_stderr else (error.stderr or b'') + if stream_output and out: + sys.stdout.buffer.write(out) + sys.stdout.buffer.flush() + return Completed(124, out, err) + except OSError as error: + return Completed(127, b'', str(error).encode()) + if stream_output and result.stdout: + sys.stdout.buffer.write(result.stdout) + sys.stdout.buffer.flush() + return Completed(result.returncode, result.stdout or b'', + b'' if combine_stderr else (result.stderr or b'')) + + +def compile_command_text(entry: dict) -> str: + if 'command' in entry: + return entry['command'] + return ' '.join(entry.get('arguments') or []) + + +def sanitizer_flag_on_library(build_dir: Path) -> bool: + path = build_dir / 'compile_commands.json' + if not path.is_file(): + raise RuntimeError('compile_commands.json missing; sanitizer PUBLIC flag cannot be verified') + entries = json.loads(path.read_text()) + library = [] + for entry in entries: + file = (entry.get('file') or '').replace('\\', '/') + if not file.endswith('/src/matrix.cpp') and Path(file).name != 'matrix.cpp': + continue + if '/src/' in file or Path(file).parent.name == 'src': + library.append(compile_command_text(entry)) + if not library: + raise RuntimeError('compile_commands.json has no src/matrix.cpp entry') + return all(SANITIZER_FLAG in command for command in library) + + +def stale_archive_sources(source: Path, archive: Path) -> list[str]: + if not archive.is_file(): + raise RuntimeError(f'full build did not produce {archive}') + newest = archive.stat().st_mtime + stale = [] + for root in (source / 'CMakeLists.txt', source / 'src', source / 'include'): + paths = [root] if root.is_file() else [path for path in root.rglob('*') if path.is_file()] + for path in paths: + if path.stat().st_mtime > newest: + stale.append(str(path.relative_to(source))) + return stale + + +def smoke_prefix(cache: dict[str, str], install_prefix: Path) -> str: + parts = [str(install_prefix)] + extra = cache.get('CMAKE_PREFIX_PATH') + if extra: + for item in extra.split(';'): + if item and item not in parts: + parts.append(item) + eigen = cache.get('Eigen3_DIR') + if eigen and eigen not in parts: + parts.append(eigen) + return ';'.join(parts) + + +def pinned_object_present(source: Path, runner: Runner, commit: str = BASE_COMMIT) -> bool: + result = call_runner( + runner, ['git', '-C', str(source), 'cat-file', '-e', commit + '^{commit}'], + timeout=30, stream_output=False) + return result.returncode == 0 + + +def tool_version_commands(cfg: VerifyConfig) -> list[list[str]]: + commands = [ + ['cmake', '--version'], + ['ctest', '--version'], + ['git', '--version'], + [sys.executable, '--version'], + ] + compiler = shutil.which('c++') or shutil.which('clang++') or shutil.which('g++') + if compiler: + commands.append([compiler, '--version']) + if cfg.ccache_path: + commands.append([cfg.ccache_path, '--version']) + return commands + + +class Driver: + def __init__(self, cfg: VerifyConfig): + self.cfg = cfg + self.logs = cfg.build_dir / 'ci-logs' + self.install_prefix = cfg.build_dir / 'ci-install' + self.smoke_dir = cfg.build_dir / 'ci-smoke' + self.summary_path = cfg.build_dir / 'ci-summary.json' + self.failures: list[dict] = [] + self.stages: list[dict] = [] + self.expected_version = expected_version(cfg.source) + self.actual_version: str | None = None + self.abi_action = 'not-started' + self.abi_prior_action = 'not-started' + self.summary: dict = { + 'schemaVersion': SCHEMA, + 'status': 'incomplete', + 'profile': cfg.profile.name, + 'source': str(cfg.source), + 'buildDir': str(cfg.build_dir), + 'installPrefix': str(self.install_prefix), + 'smokeDir': str(self.smoke_dir), + 'jobs': cfg.jobs, + 'generator': cfg.generator, + 'ccache': cfg.ccache_path, + 'requireWebsocket': cfg.require_websocket, + 'curlDir': str(cfg.curl_dir) if cfg.curl_dir else None, + 'versionSource': 'FILE', + 'cmakeDefinitions': cmake_cache_definitions(cfg), + 'expectedVersion': self.expected_version, + 'actualVersion': None, + 'exitCode': None, + 'abi': {'action': self.abi_action}, + 'abiPrior': {'action': self.abi_prior_action}, + 'stages': self.stages, + 'failures': self.failures, + } + + def write_summary(self) -> None: + self.summary['abi'] = {'action': self.abi_action} + self.summary['abiPrior'] = {'action': self.abi_prior_action} + self.summary['actualVersion'] = self.actual_version + self.summary['stages'] = self.stages + self.summary['failures'] = self.failures + self.summary_path.parent.mkdir(parents=True, exist_ok=True) + self.summary_path.write_text(json.dumps(self.summary, indent=2, sort_keys=True) + '\n') + + def invoke(self, name: str, argv: list[str], *, extra_env: dict[str, str] | None = None, + timeout: int = 600, combine_stderr: bool = True) -> Completed: + if self.cfg.ccache_path: + extra_env = {**(extra_env or {}), 'CCACHE_COMPILERCHECK': 'content'} + log = self.logs / f'{name}.log' + log.parent.mkdir(parents=True, exist_ok=True) + started = time.time() + header = '+ ' + shlex.join(map(str, argv)) + '\n' + stage = { + 'name': name, 'argv': list(map(str, argv)), 'exitCode': None, + 'status': 'running', 'log': str(log.relative_to(self.cfg.build_dir)), + 'extraEnvKeys': sorted(extra_env) if extra_env else [], + } + self.stages.append(stage) + log.write_text(header) + self.write_summary() + if self.cfg.stream_output: + print(f'ci_verify: {name}: {shlex.join(map(str, argv))}', flush=True) + try: + result = call_runner( + self.cfg.runner, list(map(str, argv)), extra_env=extra_env, timeout=timeout, + combine_stderr=combine_stderr, stream_output=self.cfg.stream_output) + except Exception as error: + result = Completed(1, b'', str(error).encode()) + body = result.stdout + (b'' if not result.stderr else b'\n' + result.stderr) + log.write_bytes(header.encode() + body) + stage.update({ + 'exitCode': result.returncode, + 'status': 'passed' if result.returncode == 0 else 'failed', + 'durationSeconds': round(time.time() - started, 3), + }) + if result.returncode != 0: + self.failures.append({ + 'stage': name, + 'exitCode': result.returncode, + 'log': stage['log'], + }) + self.write_summary() + return result + + def fail_stage(self, name: str, message: str, *, argv: list[str] | None = None) -> None: + log = self.logs / f'{name}.log' + log.parent.mkdir(parents=True, exist_ok=True) + log.write_text(message + '\n') + self.stages.append({ + 'name': name, + 'argv': argv or [], + 'exitCode': 1, + 'status': 'failed', + 'log': str(log.relative_to(self.cfg.build_dir)), + 'error': message, + 'extraEnvKeys': [], + }) + self.failures.append({ + 'stage': name, 'exitCode': 1, 'error': message, + 'log': str(log.relative_to(self.cfg.build_dir)), + }) + self.write_summary() + + def pass_stage(self, name: str, message: str, *, argv: list[str] | None = None) -> None: + log = self.logs / f'{name}.log' + log.parent.mkdir(parents=True, exist_ok=True) + log.write_text(message + '\n') + self.stages.append({ + 'name': name, + 'argv': argv or [], + 'exitCode': 0, + 'status': 'passed', + 'log': str(log.relative_to(self.cfg.build_dir)), + 'extraEnvKeys': [], + }) + self.write_summary() + + def sanitizer_env(self) -> dict[str, str] | None: + return dict(SANITIZER_RUN_ENV) if self.cfg.profile.sanitizers else None + + def collect_tool_versions(self) -> bool: + chunks = [] + failed = False + for argv in tool_version_commands(self.cfg): + result = call_runner( + self.cfg.runner, argv, timeout=30, stream_output=False) + chunks.append('+ ' + shlex.join(argv)) + chunks.append((result.stdout + result.stderr).decode('utf-8', 'replace')) + if result.returncode != 0 and Path(argv[0]).name in {'cmake', 'ctest', 'git'}: + failed = True + self.logs.mkdir(parents=True, exist_ok=True) + (self.logs / 'tool-versions.log').write_text('\n'.join(chunks) + '\n') + self.stages.append({ + 'name': 'tool-versions', + 'argv': ['tool-versions'], + 'exitCode': 1 if failed else 0, + 'status': 'failed' if failed else 'passed', + 'log': 'ci-logs/tool-versions.log', + 'extraEnvKeys': [], + }) + if failed: + self.failures.append({'stage': 'tool-versions', 'exitCode': 1}) + self.write_summary() + return not failed + + def verify_configured_profile(self, cache: dict[str, str]) -> str | None: + profile = self.cfg.profile + if cache.get('CMAKE_BUILD_TYPE') != profile.build_type: + return (f'CMAKE_BUILD_TYPE expected {profile.build_type!r} ' + f'got {cache.get("CMAKE_BUILD_TYPE")!r}') + if cache.get('PINEFORGE_VERSION_SOURCE') != 'FILE': + return (f'PINEFORGE_VERSION_SOURCE expected FILE ' + f'got {cache.get("PINEFORGE_VERSION_SOURCE")!r}') + for key, wanted in ( + ('PINEFORGE_BUILD_TESTS', True), + ('PINEFORGE_BUILD_TUTORIAL', profile.tutorial), + ('PINEFORGE_BUILD_LIVE_RUNNER', profile.live_runner), + ('PINEFORGE_ENABLE_SANITIZERS', profile.sanitizers), + ): + if cmake_on(cache.get(key)) != wanted: + return f'{key} expected {"ON" if wanted else "OFF"} got {cache.get(key)!r}' + if self.cfg.ccache_path: + for launcher in ('CMAKE_C_COMPILER_LAUNCHER', 'CMAKE_CXX_COMPILER_LAUNCHER'): + if cache.get(launcher) != self.cfg.ccache_path: + return f'{launcher} expected {self.cfg.ccache_path!r} got {cache.get(launcher)!r}' + return None + + def ensure_abi_base(self) -> None: + self.abi_action = self.ensure_prepared_provider( + self.cfg.build_dir / 'settlement-abi-base', BASE_COMMIT, BASE_TREE, + extra_argv=[], stage='abi-base', fetch_stage='abi-fetch') + + def ensure_abi_prior(self) -> None: + provider = PROVIDERS['0e'] + manifest = self.cfg.source / provider['manifest'].relative_to(ROOT) + self.abi_prior_action = self.ensure_prepared_provider( + self.cfg.build_dir / provider['default_output'], provider['commit'], provider['tree'], + extra_argv=['--commit', provider['commit'], '--tree', provider['tree'], + '--header-manifest', str(manifest)], + stage='abi-prior', fetch_stage='abi-prior-fetch') + + def ensure_prepared_provider(self, output: Path, commit: str, tree: str, *, + extra_argv: list[str], stage: str, fetch_stage: str) -> str: + prepare = [ + sys.executable, str(self.cfg.source / 'scripts/prepare_settlement_cpp_abi_base.py'), + '--source-repo', str(self.cfg.source), + '--current-build', str(self.cfg.build_dir), + '--output', str(output), + '--jobs', str(self.cfg.jobs), + *extra_argv, + ] + if output.exists(): + try: + receipt = reusable_prepared_base(output, self.cfg.build_dir, commit=commit, tree=tree) + except Exception as error: + self.fail_stage(stage, str(error), argv=['reuse-matching-receipt', str(output)]) + return 'refused' + self.pass_stage( + stage, + json.dumps({'action': 'reused', 'receipt': str(output / 'receipt.json'), + 'archiveSha256': receipt.get('archiveSha256')}, indent=2, sort_keys=True), + argv=['reuse-matching-receipt', str(output / 'receipt.json')]) + return 'reused' + if not pinned_object_present(self.cfg.source, self.cfg.runner, commit): + fetch = ['git', '-C', str(self.cfg.source), 'fetch', '--no-tags', '--depth=1', + 'origin', commit] + fetched = self.invoke(fetch_stage, fetch, timeout=120) + if fetched.returncode != 0: + return 'failed' + prepared = self.invoke(stage, prepare, timeout=1800) + return 'prepared' if prepared.returncode == 0 else 'failed' + + def run_smoke(self, cache: dict[str, str]) -> None: + prefix = smoke_prefix(cache, self.install_prefix) + configure = [ + 'cmake', '-S', str(self.cfg.source / 'cmake/smoke_consumer'), + '-B', str(self.smoke_dir), + f'-DCMAKE_BUILD_TYPE={self.cfg.profile.build_type}', + f'-DCMAKE_PREFIX_PATH={prefix}', + ] + if cache.get('Eigen3_DIR'): + configure.append(f'-DEigen3_DIR={cache["Eigen3_DIR"]}') + if self.invoke('smoke-configure', configure, timeout=120).returncode != 0: + return + if self.invoke( + 'smoke-build', + ['cmake', '--build', str(self.smoke_dir), '--parallel', str(self.cfg.jobs)], + timeout=300).returncode != 0: + return + binary = self.smoke_dir / 'smoke_version' + ran = self.invoke( + 'smoke-version', [str(binary)], extra_env=self.sanitizer_env(), + timeout=60, combine_stderr=False) + actual = ran.stdout.decode('utf-8', 'replace').strip() + self.actual_version = actual + self.summary['actualVersion'] = actual + if ran.returncode != 0: + self.failures[-1]['error'] = ( + f'version smoke exited {ran.returncode}; expected {self.expected_version!r} ' + f'got {actual!r}') + self.write_summary() + return + if actual != self.expected_version: + self.fail_stage( + 'smoke-version-check', + f'version smoke expected {self.expected_version!r} got {actual!r}', + argv=[str(binary)]) + return + self.pass_stage( + 'smoke-version-check', + f'printed VERSION {actual} matches {self.cfg.source / "VERSION"}', + argv=[str(binary)]) + + def run(self) -> int: + self.logs.mkdir(parents=True, exist_ok=True) + self.write_summary() + if not self.collect_tool_versions(): + return self.finish('failed', 1) + guard_failed = False + for name, argv in source_guard_commands(self.cfg.source): + if self.invoke(name, argv, timeout=120).returncode != 0: + guard_failed = True + if guard_failed: + return self.finish('failed', 1) + if self.invoke('configure', cmake_configure_argv(self.cfg), timeout=180).returncode != 0: + return self.finish('failed', 1) + cache_path = self.cfg.build_dir / 'CMakeCache.txt' + if not cache_path.is_file(): + self.fail_stage('configure-cache', f'missing {cache_path}') + return self.finish('failed', 1) + cache = read_cache(cache_path) + profile_error = self.verify_configured_profile(cache) + if profile_error: + self.fail_stage('profile-options', profile_error) + return self.finish('failed', 1) + self.pass_stage('profile-options', json.dumps({ + 'tutorial': cmake_on(cache.get('PINEFORGE_BUILD_TUTORIAL')), + 'liveRunner': cmake_on(cache.get('PINEFORGE_BUILD_LIVE_RUNNER')), + 'sanitizers': cmake_on(cache.get('PINEFORGE_ENABLE_SANITIZERS')), + 'versionSource': cache.get('PINEFORGE_VERSION_SOURCE'), + 'buildType': cache.get('CMAKE_BUILD_TYPE'), + }, indent=2, sort_keys=True)) + if self.cfg.profile.sanitizers: + try: + if not sanitizer_flag_on_library(self.cfg.build_dir): + self.fail_stage( + 'sanitizer-public-flag', + f'src/matrix.cpp compile command missing PUBLIC {SANITIZER_FLAG}') + return self.finish('failed', 1) + except Exception as error: + self.fail_stage('sanitizer-public-flag', str(error)) + return self.finish('failed', 1) + self.pass_stage('sanitizer-public-flag', f'library compile uses {SANITIZER_FLAG}') + if self.invoke( + 'build', + ['cmake', '--build', str(self.cfg.build_dir), '--parallel', str(self.cfg.jobs)], + timeout=1800).returncode != 0: + return self.finish('failed', 1) + archive = self.cfg.build_dir / 'lib' / 'libpineforge.a' + try: + stale = stale_archive_sources(self.cfg.source, archive) + except Exception as error: + self.fail_stage('stale-binaries', str(error)) + return self.finish('failed', 1) + if stale: + self.fail_stage( + 'stale-binaries', + 'libpineforge.a predates source; full rebuild required: ' + ', '.join(stale[:40])) + return self.finish('failed', 1) + self.pass_stage('stale-binaries', f'{archive} is newer than src/, include/, CMakeLists.txt') + live = self.cfg.build_dir / 'bin' / 'pineforge-live' + if self.cfg.profile.live_runner: + if not live.is_file(): + self.fail_stage('native-binary', f'native profile missing {live}') + return self.finish('failed', 1) + self.pass_stage('native-binary', str(live)) + elif live.is_file(): + self.fail_stage( + 'native-binary', + 'non-native profile produced pineforge-live; live runner must stay OFF') + return self.finish('failed', 1) + else: + self.pass_stage('native-binary', 'live runner absent as required for this profile') + + self.ensure_abi_base() + self.ensure_abi_prior() + + ctest = ['ctest', '--test-dir', str(self.cfg.build_dir), + '--output-on-failure', '--parallel', str(self.cfg.jobs)] + if ctest_supports_junit(self.cfg.runner): + ctest += ['--output-junit', str(self.cfg.build_dir / 'ctest-junit.xml')] + self.invoke('ctest', ctest, extra_env=self.sanitizer_env(), timeout=1800) + + installed = self.invoke( + 'install', + ['cmake', '--install', str(self.cfg.build_dir), '--prefix', str(self.install_prefix)], + timeout=180) + if installed.returncode == 0: + self.run_smoke(cache) + if self.cfg.profile.live_runner: + help_bin = self.install_prefix / 'bin' / 'pineforge-live' + if not help_bin.is_file(): + self.fail_stage('native-help', f'installed native executable missing: {help_bin}') + else: + self.invoke('native-help', [str(help_bin), '--help'], timeout=30) + if self.cfg.require_websocket: + ws = self.cfg.build_dir / 'bin' / 'test_native_live_websocket' + if not ws.is_file(): + self.fail_stage( + 'require-websocket', + f'--require-websocket cannot accept skip: missing {ws}') + else: + ran = self.invoke( + 'require-websocket', [str(ws)], extra_env=self.sanitizer_env(), timeout=120) + if ran.returncode == 77: + self.fail_stage( + 'require-websocket-skip', + '--require-websocket cannot accept skip (exit 77)', + argv=[str(ws)]) + status = 'passed' if not self.failures else 'failed' + return self.finish(status, 0 if status == 'passed' else 1) + + def finish(self, status: str, code: int) -> int: + self.summary['status'] = status + self.summary['exitCode'] = code + self.write_summary() + return code + + +def build_config(argv: list[str] | None = None, *, source: Path = ROOT, + runner: Runner | None = None, stream_output: bool | None = None, + which: Callable[[str], str | None] = shutil.which) -> VerifyConfig: + args = parse_args(argv, source=source) + cfg = validate_config(args, source=source, which=which) + if runner is not None: + cfg.runner = runner + if stream_output is not None: + cfg.stream_output = stream_output + return cfg + + +def main(argv: list[str] | None = None, *, runner: Runner | None = None, + stream_output: bool | None = None, + which: Callable[[str], str | None] = shutil.which) -> int: + try: + cfg = build_config(argv, runner=runner, stream_output=stream_output, which=which) + except ConfigError as error: + print(f'ci_verify: {error}', file=sys.stderr) + return 2 + driver = Driver(cfg) + try: + return driver.run() + except ConfigError as error: + print(f'ci_verify: {error}', file=sys.stderr) + driver.fail_stage('config', str(error)) + return driver.finish('failed', 2) + except Exception as error: + driver.fail_stage('driver', str(error)) + return driver.finish('failed', 1) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/collect_ci_diagnostics.py b/scripts/collect_ci_diagnostics.py new file mode 100644 index 000000000..4cfe2f604 --- /dev/null +++ b/scripts/collect_ci_diagnostics.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Retain compact CI diagnostics without copying binaries or dependency caches.""" +from __future__ import annotations +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess + +ARTIFACTS = { + "ci-summary.json": "ci-summary.json", + "ci-logs": "ci-logs", + "ctest-junit.xml": "ctest-junit.xml", + "Testing/Temporary/LastTest.log": "LastTest.log", + "settlement-abi-base/receipt.json": "settlement-abi-base-receipt.json", + "settlement-abi-base/configure.log": "settlement-abi-base-configure.log", + "settlement-abi-base/build.log": "settlement-abi-base-build.log", + "settlement-abi-prior/receipt.json": "settlement-abi-prior-receipt.json", + "settlement-abi-prior/configure.log": "settlement-abi-prior-configure.log", + "settlement-abi-prior/build.log": "settlement-abi-prior-build.log", + "settlement-abi-receipt.json": "settlement-abi-receipt.json", +} +ENV_KEYS = ("GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_SHA", "GITHUB_RUN_ID", + "GITHUB_RUN_ATTEMPT", "GITHUB_JOB", "RUNNER_OS", "RUNNER_ARCH", "CURL_CACHE_HIT") + + +def copy_regular(source: Path, target: Path) -> None: + if source.is_symlink(): + return + if source.is_file(): + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + elif source.is_dir(): + for item in source.iterdir(): + copy_regular(item, target / item.name) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--build-dir", type=Path, required=True) + parser.add_argument("--profile", choices=("release", "debug", "sanitizers", "native"), required=True) + parser.add_argument("--output", type=Path, default=Path("ci-diagnostics")) + args = parser.parse_args() + build = args.build_dir.resolve() + output = args.output.resolve() + if output == build or build in output.parents: + parser.error("--output must be outside --build-dir to avoid copying diagnostics into themselves") + args.output.mkdir(parents=True, exist_ok=True) + info = {key: os.environ.get(key, "") for key in ENV_KEYS} + info.update(buildDir=str(args.build_dir), profile=args.profile) + (args.output / "job-info.json").write_text(json.dumps(info, indent=2) + "\n") + missing = [] + for source, target in ARTIFACTS.items(): + path = args.build_dir / source + if path.exists() and not path.is_symlink(): + copy_regular(path, args.output / target) + else: + missing.append(source) + for directory in args.build_dir.glob("settlement-abi-receipt.artifacts-*"): + if directory.is_symlink() or not directory.is_dir(): + continue + for item in directory.rglob("*"): + if item.suffix in (".log", ".json", ".cpp", ".hpp"): + copy_regular(item, args.output / directory.name / item.relative_to(directory)) + (args.output / "missing.json").write_text(json.dumps(missing, indent=2) + "\n") + if shutil.which("ccache"): + result = subprocess.run(["ccache", "--show-stats"], capture_output=True, text=True) + (args.output / "ccache.log").write_text(result.stdout + result.stderr) + summary = {} + try: + summary = json.loads((args.build_dir / "ci-summary.json").read_text()) + except (OSError, ValueError): + pass + print(json.dumps({"profile": args.profile, "status": summary.get("status", "unavailable"), + "failures": summary.get("failures", []), "diagnostics": str(args.output)}, indent=2)) + destination = os.environ.get("GITHUB_STEP_SUMMARY") + if destination: + status = summary.get("status", "unavailable") + stages = summary.get("stages", []) + lines = [f"### Verification: {args.profile}", "", f"Status: **{status}**", "", + "| Stage | Result |", "| --- | --- |"] + lines += [f"| {stage.get('name', '?')} | {stage.get('status', '?')} |" for stage in stages] + lines += ["", "Full logs and ABI/CTest receipts are attached as CI diagnostics.", ""] + with Path(destination).open("a") as output: + output.write("\n".join(lines)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prepare_settlement_cpp_abi_base.py b/scripts/prepare_settlement_cpp_abi_base.py new file mode 100644 index 000000000..cb97a9ad1 --- /dev/null +++ b/scripts/prepare_settlement_cpp_abi_base.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +"""Build an exact historical settlement ABI provider from local Git objects. + +Run explicitly before CTest. This script never fetches Git history or dependencies. +The exact Git object and Eigen must already be available to the current build. +""" +from __future__ import annotations +import argparse +import hashlib +import io +import json +import os +from pathlib import Path +import platform +import shlex +import shutil +import subprocess +import tarfile + +BASE_COMMIT = 'e60e57156a54bb1c74c0c0a4f5a338fd924c046d' +BASE_TREE = 'e209c89035ec884774f8f3e68e1dc118f26aaf51' +ROOT = Path(__file__).resolve().parents[1] +HEADER_MANIFEST = ROOT / 'tests/fixtures/settlement_cpp_abi/e60e571/manifest.json' +PRIOR_COMMIT = '0e18690db3fb6bb4705841c2a86936dfa206380a' +PRIOR_TREE = 'c29545d262bf6dcc22e46aa3382422f2e9c68378' +PRIOR_HEADER_MANIFEST = ROOT / 'tests/fixtures/settlement_cpp_abi/0e18690/manifest.json' +PROVIDERS = { + 'e60': {'commit': BASE_COMMIT, 'tree': BASE_TREE, 'manifest': HEADER_MANIFEST, + 'default_output': 'settlement-abi-base', 'headers_name': 'r2-headers.tar'}, + '0e': {'commit': PRIOR_COMMIT, 'tree': PRIOR_TREE, 'manifest': PRIOR_HEADER_MANIFEST, + 'default_output': 'settlement-abi-prior', 'headers_name': 'headers.tar'}, +} +COPY_CACHE = ( + 'CMAKE_BUILD_TYPE', 'CMAKE_C_COMPILER', 'CMAKE_CXX_COMPILER', + 'CMAKE_C_COMPILER_TARGET', 'CMAKE_CXX_COMPILER_TARGET', + 'CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN', 'CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN', + 'CMAKE_C_COMPILER_LAUNCHER', 'CMAKE_CXX_COMPILER_LAUNCHER', + 'CMAKE_C_FLAGS', 'CMAKE_CXX_FLAGS', + 'CMAKE_C_FLAGS_DEBUG', 'CMAKE_CXX_FLAGS_DEBUG', + 'CMAKE_C_FLAGS_RELEASE', 'CMAKE_CXX_FLAGS_RELEASE', + 'CMAKE_C_FLAGS_RELWITHDEBINFO', 'CMAKE_CXX_FLAGS_RELWITHDEBINFO', + 'CMAKE_C_FLAGS_MINSIZEREL', 'CMAKE_CXX_FLAGS_MINSIZEREL', + 'CMAKE_EXE_LINKER_FLAGS', 'CMAKE_STATIC_LINKER_FLAGS', + 'CMAKE_POSITION_INDEPENDENT_CODE', 'CMAKE_INTERPROCEDURAL_OPTIMIZATION', + 'CMAKE_OSX_ARCHITECTURES', 'CMAKE_OSX_SYSROOT', 'CMAKE_OSX_DEPLOYMENT_TARGET', + 'CMAKE_SYSROOT', 'CMAKE_TOOLCHAIN_FILE', 'CMAKE_PREFIX_PATH', 'Eigen3_DIR', + 'CMAKE_DISABLE_FIND_PACKAGE_Eigen3', 'FETCHCONTENT_SOURCE_DIR_EIGEN', + 'PINEFORGE_ENABLE_SANITIZERS', 'PINEFORGE_ENABLE_COVERAGE', + 'PINEFORGE_VERSION_SOURCE', +) + + +def identity(path: Path) -> dict: + raw = path.read_bytes() + return {'sha256': hashlib.sha256(raw).hexdigest(), 'bytes': len(raw)} + + +def read_cache(path: Path) -> dict[str, str]: + found = {} + for line in path.read_text().splitlines(): + if line and not line.startswith(('#', '//')) and ':' in line and '=' in line: + name_type, value = line.split('=', 1) + found[name_type.split(':', 1)[0]] = value + return found + + +def copied_cache(cache: dict[str, str]) -> dict[str, str]: + return {key: cache[key] for key in COPY_CACHE if key in cache} + + +def refuse_existing_base(output: Path, reason: str) -> None: + raise RuntimeError(f'{reason}; use a fresh --build-dir (refusing to delete {output})') + + +def receipt_matches_current(receipt: dict, current_cache: dict, compiler: dict, *, + commit: str = BASE_COMMIT, tree: str = BASE_TREE) -> None: + """Raise if a prepared base must not be reused with this current build.""" + if receipt.get('schemaVersion') != 'pineforge-settlement-abi-base/v1': + raise RuntimeError('existing ABI base receipt is not portable v1') + if receipt.get('commit') != commit or receipt.get('tree') != tree: + label = 'e60 R2' if commit == BASE_COMMIT else commit + raise RuntimeError('existing ABI base does not pin ' + label) + old = receipt.get('compiler') or {} + for key in ('target', 'sha256', 'version'): + if old.get(key) != compiler.get(key): + raise RuntimeError('existing ABI base compiler implementation/version/target differs') + if receipt.get('copiedCurrentCache') != copied_cache(current_cache): + raise RuntimeError( + 'existing ABI base compiler/configuration/version-source/launcher differs from this build') + + +def reusable_prepared_base(output: Path, current_build: Path, *, + commit: str = BASE_COMMIT, tree: str = BASE_TREE) -> dict: + """Return an existing receipt that is safe to reuse. Never deletes output.""" + receipt_path = output / 'receipt.json' + if not output.exists(): + raise FileNotFoundError(str(output)) + if not receipt_path.is_file(): + refuse_existing_base(output, f'ABI base directory exists without a receipt at {receipt_path}') + receipt = json.loads(receipt_path.read_text()) + cache = read_cache(current_build / 'CMakeCache.txt') + compiler = compiler_identity(cache['CMAKE_CXX_COMPILER']) + try: + receipt_matches_current(receipt, cache, compiler, commit=commit, tree=tree) + except RuntimeError as error: + refuse_existing_base(output, str(error)) + resolve = lambda name: Path(name) if Path(name).is_absolute() else output / name + library = resolve(receipt['archive']) + headers = resolve(receipt.get('headers', 'r2-headers.tar')) + if not library.is_file() or not headers.is_file(): + refuse_existing_base(output, 'existing ABI base is missing archive/header artifacts') + if identity(library)['sha256'] != receipt['archiveSha256'] or identity(headers)['sha256'] != receipt['headersSha256']: + refuse_existing_base(output, 'existing ABI base archive/header bytes do not match receipt') + generated_name = receipt.get('generatedInclude') + expected_generated = receipt.get('generatedHeaderSha256') + if generated_name and expected_generated: + generated = resolve(generated_name) / 'pineforge/version.h' + if not generated.is_file() or identity(generated)['sha256'] != expected_generated: + refuse_existing_base(output, 'existing ABI base generated version header does not match receipt') + return receipt + + +def current_eigen_source(cache: dict[str, str], current: Path) -> Path | None: + """Reuse the current build's populated dependency without a network fetch.""" + candidates = [cache.get(key) for key in + ('FETCHCONTENT_SOURCE_DIR_EIGEN', 'Eigen3_SOURCE_DIR', 'eigen_SOURCE_DIR')] + candidates.append(str(Path(cache.get('FETCHCONTENT_BASE_DIR') or current / '_deps') / 'eigen-src')) + for value in candidates: + if not value: + continue + path = Path(value).resolve() + if (path / 'CMakeLists.txt').is_file() and (path / 'Eigen/Core').is_file(): + return path + return None + + +def run(argv, *, cwd=None, timeout=60, log: Path | None = None) -> bytes: + result = subprocess.run(argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) + if log: + log.write_bytes(result.stdout + result.stderr) + if result.returncode: + raise RuntimeError(f'command failed ({result.returncode}): {shlex.join(map(str, argv))}\n' + + result.stderr.decode('utf-8', 'replace')[-6000:]) + return result.stdout + + +def extract_tar(raw: bytes, destination: Path) -> None: + destination.mkdir(parents=True, exist_ok=False) + with tarfile.open(fileobj=io.BytesIO(raw)) as archive: + members = archive.getmembers() + seen = set() + for member in members: + path = Path(member.name) + if path.is_absolute() or '..' in path.parts or member.name in seen: + raise RuntimeError('unsafe or duplicate archive member: ' + member.name) + if not member.isdir() and not member.isfile(): + raise RuntimeError('ABI preparation refuses archive links/special files: ' + member.name) + seen.add(member.name) + for member in members: + target = destination / member.name + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + else: + target.parent.mkdir(parents=True, exist_ok=True) + stream = archive.extractfile(member) + assert stream is not None + target.write_bytes(stream.read()) + target.chmod(member.mode & 0o777) + + +def authenticate_headers(directory: Path, manifest_path: Path = HEADER_MANIFEST, *, + commit: str = BASE_COMMIT, tree: str = BASE_TREE) -> dict: + manifest = json.loads(manifest_path.read_text()) + label = 'R2' if commit == BASE_COMMIT else commit[:7] + if manifest['commit'] != commit or manifest['tree'] != tree: + raise RuntimeError('tracked ' + label + ' header pin changed') + actual = {str(path.relative_to(directory)) for path in (directory / 'include').rglob('*') if path.is_file()} + if actual != set(manifest['files']): + raise RuntimeError(label + ' header closure differs from the tracked exact file inventory') + for name, expected in manifest['files'].items(): + raw = (directory / name).read_bytes() + blob = hashlib.sha1(b'blob ' + str(len(raw)).encode() + b'\0' + raw).hexdigest() + if identity(directory / name) != {'sha256': expected['sha256'], 'bytes': expected['bytes']} or blob != expected['git_blob']: + raise RuntimeError(label + ' header bytes differ: ' + name) + return manifest + + +def compiler_identity(command: str) -> dict: + resolved = shutil.which(command) if not Path(command).is_absolute() else command + if not resolved: + raise RuntimeError('compiler not found: ' + command) + implementation = Path(resolved).resolve() + if platform.system() == 'Darwin' and implementation.parent == Path('/usr/bin'): + implementation = Path(run(['xcrun', '--find', Path(resolved).name]).decode().strip()).resolve() + return {'command': str(resolved), 'implementation': str(implementation), + **identity(implementation), 'version': run([command, '--version']).decode().strip(), + 'target': run([command, '-dumpmachine']).decode().strip()} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source-repo', type=Path, required=True) + parser.add_argument('--current-build', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--jobs', type=int, default=4) + parser.add_argument('--commit', default=BASE_COMMIT) + parser.add_argument('--tree', default=BASE_TREE) + parser.add_argument('--header-manifest', type=Path, default=HEADER_MANIFEST) + args = parser.parse_args() + provider = next((pin for pin in PROVIDERS.values() + if (pin['commit'], pin['tree']) == (args.commit, args.tree)), None) + if provider is None: + raise RuntimeError('unsupported historical ABI commit/tree pin') + label = 'R2' if args.commit == BASE_COMMIT else args.commit[:7] + repo, current, output = args.source_repo.resolve(), args.current_build.resolve(), args.output.resolve() + if output.exists(): + raise RuntimeError('output must be new; reuse its existing receipt explicitly instead of rebuilding over it') + if not 1 <= args.jobs <= 64: + raise RuntimeError('--jobs must be 1..64') + cache = read_cache(current / 'CMakeCache.txt') + if Path(cache['CMAKE_HOME_DIRECTORY']).resolve() != repo: + raise RuntimeError('current CMake cache belongs to a different source repository') + try: + commit = run(['git', '-C', str(repo), 'rev-parse', args.commit + '^{commit}']).decode().strip() + tree = run(['git', '-C', str(repo), 'rev-parse', args.commit + '^{tree}']).decode().strip() + except RuntimeError as error: + raise RuntimeError(f'exact {label} source is unavailable; fetch the pinned object without tags ' + f'before preparation: git fetch --no-tags --depth=1 origin {args.commit}') from error + if commit != args.commit or tree != args.tree: + raise RuntimeError(label + ' commit/tree mismatch') + source_raw = run(['git', '-C', str(repo), 'archive', '--format=tar', args.commit, + 'CMakeLists.txt', 'VERSION', 'cmake', 'include', 'src']) + output.mkdir(parents=True) + (output / 'source.tar').write_bytes(source_raw) + source, build = output / 'source', output / 'build' + extract_tar(source_raw, source) + authenticate_headers(source, args.header_manifest, commit=args.commit, tree=args.tree) + settings = copied_cache(cache) + eigen_source = current_eigen_source(cache, current) + if eigen_source is not None: + settings['FETCHCONTENT_SOURCE_DIR_EIGEN'] = str(eigen_source) + settings.update({'PINEFORGE_BUILD_TESTS': 'OFF', 'PINEFORGE_BUILD_TUTORIAL': 'OFF', + 'PINEFORGE_BUILD_CORPUS_STRATEGIES': 'OFF', 'PINEFORGE_BUILD_LIVE_RUNNER': 'OFF', + 'PINEFORGE_BUILD_BENCH_STRATEGIES': 'OFF', 'PINEFORGE_BUILD_SPEED_BENCH': 'OFF', + 'CMAKE_EXPORT_COMPILE_COMMANDS': 'ON', 'FETCHCONTENT_FULLY_DISCONNECTED': 'ON'}) + configure = ['cmake', '-S', str(source), '-B', str(build), '-G', cache['CMAKE_GENERATOR'], + *[f'-D{key}={value}' for key, value in settings.items()]] + run(configure, timeout=120, log=output / 'configure.log') + build_argv = ['cmake', '--build', str(build), '--target', 'pineforge', '--parallel', str(args.jobs)] + run(build_argv, timeout=1800, log=output / 'build.log') + library = build / 'lib/libpineforge.a' + if not library.read_bytes().startswith(b'!\n'): + raise RuntimeError('base library is not a full static archive') + archive_members = run(['ar', '-t', str(library)]).decode().splitlines() + if len(archive_members) < 20: + raise RuntimeError('base archive is not the full ' + label + ' library') + headers = output / provider['headers_name'] + with tarfile.open(headers, 'w') as archive: + archive.add(source / 'include', arcname='include', recursive=True) + source_identity = {str(path.relative_to(source)): identity(path)['sha256'] + for path in sorted(source.rglob('*')) if path.is_file()} + receipt = {'schemaVersion': 'pineforge-settlement-abi-base/v1', 'commit': args.commit, 'tree': args.tree, + 'archive': str(library.relative_to(output)), 'archiveSha256': identity(library)['sha256'], + 'headers': headers.name, 'headersSha256': identity(headers)['sha256'], + 'generatedInclude': 'build/include', + 'generatedHeaderSha256': identity(build / 'include/pineforge/version.h')['sha256'], + 'sourceSha256': source_identity, 'sourceArchive': identity(output / 'source.tar'), + 'compiler': compiler_identity(cache['CMAKE_CXX_COMPILER']), + 'settings': settings, 'copiedCurrentCache': copied_cache(cache), + 'currentCacheSha256': identity(current / 'CMakeCache.txt')['sha256'], + 'baseCacheSha256': identity(build / 'CMakeCache.txt')['sha256'], + 'compileCommandsSha256': identity(build / 'compile_commands.json')['sha256'], + 'configureArgv': configure, 'buildArgv': build_argv, 'archiveMembers': archive_members, + 'executedBinaries': 0, 'networkFetches': 0, + 'sourceMode': 'immutable Git archive; generated version metadata uses VERSION fallback'} + (output / 'receipt.json').write_text(json.dumps(receipt, indent=2, sort_keys=True) + '\n') + print(str(output / 'receipt.json')) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/test_ci_verify.py b/scripts/test_ci_verify.py new file mode 100644 index 000000000..0046764a8 --- /dev/null +++ b/scripts/test_ci_verify.py @@ -0,0 +1,788 @@ +#!/usr/bin/env python3 +"""Driver control-flow, config, aggregation, and receipt-reuse tests for ci_verify. + +Source guards, tool help, git object presence, compiler identity, and VERSION +are real. Full engine configure/build/CTest is not substituted as a passing +root verification — those stages are scripted here so failure aggregation and +ordering can be asserted. Root must still run the actual profiles. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import shutil +import tempfile +import unittest + +import ci_verify +from ci_verify import ( + Completed, + ConfigError, + ROOT, + SANITIZER_FLAG, + SANITIZER_RUN_ENV, + cmake_cache_definitions, + cmake_configure_argv, + ctest_supports_junit, + default_build_dir, + default_runner, + expected_version, + main, + parse_args, + source_guard_commands, + validate_config, +) +from prepare_settlement_cpp_abi_base import ( + BASE_COMMIT, + BASE_TREE, + PRIOR_COMMIT, + PRIOR_TREE, + COPY_CACHE, + compiler_identity, + copied_cache, + identity, + receipt_matches_current, + reusable_prepared_base, +) + + +def write_cache(path: Path, values: dict[str, str]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + lines = [f'{key}:STRING={value}' for key, value in values.items()] + path.write_text('\n'.join(lines) + '\n') + + +def which_with_ccache(ccache: str | None): + def which(name: str) -> str | None: + if name == 'ccache': + return ccache + return shutil.which(name) + return which + + +class Scripted: + """Programmable command runner that still executes real source guards/tools.""" + + def __init__(self, build_dir: Path, source: Path, profile: str = 'release', **exits): + self.build_dir = Path(build_dir) + self.source = Path(source) + self.profile = profile + self.exits = exits + self.calls: list[list[str]] = [] + self.cxx = shutil.which('c++') or '/usr/bin/c++' + self.secret_seen_in_argv = False + + def __call__(self, argv, *, extra_env=None, timeout=600, combine_stderr=True, + stream_output=False) -> Completed: + argv = list(map(str, argv)) + self.calls.append(argv) + extra_env = extra_env or {} + joined = ' '.join(argv) + secret = os.environ.get('CI_VERIFY_TEST_SECRET') + if secret and secret in joined: + self.secret_seen_in_argv = True + if argv[0] == 'ctest' and '--help' in argv: + if self.exits.get('junit_help') == 'absent': + return Completed(0, b'CTest help without junit\n', b'') + return default_runner(argv, extra_env=None, timeout=timeout, + combine_stderr=True, stream_output=False) + if argv[0] in {'cmake', 'ctest', 'git'} and '--version' in argv: + return default_runner(argv, extra_env=None, timeout=timeout, + combine_stderr=True, stream_output=False) + if argv[0] == 'git' and 'cat-file' in argv: + key = 'prior-cat-file' if PRIOR_COMMIT + '^{commit}' in argv else 'cat-file' + if key in self.exits: + return Completed(int(self.exits[key]), b'', b'') + return default_runner(argv, extra_env=None, timeout=timeout, + combine_stderr=True, stream_output=False) + if argv[0] == 'git' and 'fetch' in argv: + key = 'prior-fetch' if PRIOR_COMMIT in argv else 'fetch' + return Completed(int(self.exits.get(key, 0)), b'fetched\n', b'') + if any(Path(part).name == 'prepare_settlement_cpp_abi_base.py' for part in argv): + key = 'prior-prepare' if PRIOR_COMMIT in argv else 'prepare' + return Completed(int(self.exits.get(key, 0)), b'prepared\n', b'') + if any(Path(part).suffix == '.py' for part in argv): + needles = { + 'source-guard-c-abi': 'check_c_abi_runtime.py', + 'source-guard-broker-hash': 'check_broker_state_hash_coverage.py', + 'source-guard-pending-mirror': 'gen_pending_order_mirror.py', + 'source-guard-native-versions': 'check_native_cpp_versions.py', + 'source-guard-aggregate-versions': 'check_aggregate_cpp_versions.py', + } + for key, needle in needles.items(): + if key in self.exits and any(needle in part for part in argv): + return Completed(int(self.exits[key]), b'', b'injected source-guard failure\n') + return default_runner(argv, extra_env=None, timeout=timeout, + combine_stderr=True, stream_output=False) + if argv[0] == 'cmake' and '-S' in argv and 'smoke_consumer' in joined: + return Completed(int(self.exits.get('smoke-configure', 0)), b'ok\n', b'') + if argv[0] == 'cmake' and '-S' in argv: + return self._configure() + if argv[0] == 'cmake' and '--build' in argv and str(self.build_dir / 'ci-smoke') in argv: + return Completed(int(self.exits.get('smoke-build', 0)), b'ok\n', b'') + if argv[0] == 'cmake' and '--build' in argv: + return self._build() + if argv[0] == 'cmake' and '--install' in argv: + return self._install() + if argv[0] == 'ctest': + env_ok = True + if self.profile == 'sanitizers': + env_ok = extra_env == SANITIZER_RUN_ENV + if not env_ok: + return Completed(1, b'', b'sanitizer env missing\n') + return Completed(int(self.exits.get('ctest', 0)), b'tests\n', b'') + if argv[0].endswith('smoke_version') or Path(argv[0]).name == 'smoke_version': + stdout = str(self.exits.get('smoke-stdout', expected_version(self.source))).encode() + b'\n' + stderr = str(self.exits.get('smoke-stderr', '')).encode() + return Completed(int(self.exits.get('smoke-version', 0)), stdout, stderr) + if Path(argv[0]).name == 'pineforge-live': + return Completed(int(self.exits.get('native-help', 0)), b'usage\n', b'') + if Path(argv[0]).name == 'test_native_live_websocket': + return Completed(int(self.exits.get('require-websocket', 0)), b'ws\n', b'') + if '--version' in argv: + return Completed(0, b'version\n', b'') + return Completed(1, b'', f'unhandled command: {argv}\n'.encode()) + + def _cache_values(self) -> dict[str, str]: + tutorial = 'OFF' if self.profile == 'native' else 'ON' + live = 'ON' if self.profile == 'native' else 'OFF' + sanitizers = 'ON' if self.profile == 'sanitizers' else 'OFF' + build_type = 'Debug' if self.profile in {'debug', 'sanitizers'} else 'Release' + values = { + 'CMAKE_HOME_DIRECTORY': str(self.source), + 'CMAKE_CXX_COMPILER': self.cxx, + 'CMAKE_C_COMPILER': self.cxx, + 'CMAKE_GENERATOR': 'Unix Makefiles', + 'CMAKE_BUILD_TYPE': build_type, + 'PINEFORGE_BUILD_TESTS': 'ON', + 'PINEFORGE_BUILD_TUTORIAL': tutorial, + 'PINEFORGE_BUILD_LIVE_RUNNER': live, + 'PINEFORGE_ENABLE_SANITIZERS': sanitizers, + 'PINEFORGE_VERSION_SOURCE': 'FILE', + } + if self.exits.get('cache_version_source'): + values['PINEFORGE_VERSION_SOURCE'] = self.exits['cache_version_source'] + if self.exits.get('launcher'): + values['CMAKE_C_COMPILER_LAUNCHER'] = self.exits['launcher'] + values['CMAKE_CXX_COMPILER_LAUNCHER'] = self.exits['launcher'] + return values + + def _configure(self) -> Completed: + code = int(self.exits.get('configure', 0)) + if code != 0: + return Completed(code, b'', b'configure failed\n') + write_cache(self.build_dir / 'CMakeCache.txt', self._cache_values()) + if self.profile == 'sanitizers': + commands = [{ + 'directory': str(self.build_dir), + 'file': str(self.source / 'src/matrix.cpp'), + 'command': f'{self.cxx} {SANITIZER_FLAG} -c src/matrix.cpp', + }] + if self.exits.get('sanitizer_flag') == 'absent': + commands[0]['command'] = f'{self.cxx} -c src/matrix.cpp' + (self.build_dir / 'compile_commands.json').write_text(json.dumps(commands)) + self._maybe_seed_abi_base() + self._maybe_seed_abi_base(prior=True) + return Completed(0, b'configured\n', b'') + + def _build(self) -> Completed: + code = int(self.exits.get('build', 0)) + if code != 0: + return Completed(code, b'', b'build failed\n') + archive = self.build_dir / 'lib' / 'libpineforge.a' + archive.parent.mkdir(parents=True, exist_ok=True) + archive.write_bytes(b'!\nci-verify-test\n') + if self.profile == 'native' or self.exits.get('create_native_binaries'): + binary = self.build_dir / 'bin' / 'pineforge-live' + binary.parent.mkdir(parents=True, exist_ok=True) + binary.write_bytes(b'live') + ws = self.build_dir / 'bin' / 'test_native_live_websocket' + ws.write_bytes(b'ws') + if self.exits.get('omit_ws_binary'): + ws = self.build_dir / 'bin' / 'test_native_live_websocket' + if ws.exists(): + ws.unlink() + return Completed(0, b'built\n', b'') + + def _install(self) -> Completed: + code = int(self.exits.get('install', 0)) + if code != 0: + return Completed(code, b'', b'install failed\n') + prefix = self.build_dir / 'ci-install' + (prefix / 'lib' / 'cmake' / 'PineForge').mkdir(parents=True, exist_ok=True) + if self.profile == 'native': + help_bin = prefix / 'bin' / 'pineforge-live' + help_bin.parent.mkdir(parents=True, exist_ok=True) + help_bin.write_bytes(b'live') + (self.build_dir / 'ci-smoke').mkdir(parents=True, exist_ok=True) + (self.build_dir / 'ci-smoke' / 'smoke_version').write_bytes(b'smoke') + return Completed(0, b'installed\n', b'') + + def _maybe_seed_abi_base(self, *, prior=False) -> None: + kind = self.exits.get('preexisting_prior' if prior else 'preexisting_base') + if not kind: + return + output = self.build_dir / ('settlement-abi-prior' if prior else 'settlement-abi-base') + output.mkdir(parents=True, exist_ok=True) + (output / 'sentinel').write_text('keep\n') + if kind == 'no-receipt': + return + archive = output / 'build' / 'lib' / 'libpineforge.a' + headers = output / ('headers.tar' if prior else 'r2-headers.tar') + generated = output / 'build' / 'include' / 'pineforge' / 'version.h' + archive.parent.mkdir(parents=True, exist_ok=True) + generated.parent.mkdir(parents=True, exist_ok=True) + archive.write_bytes(b'!\nbase\n') + headers.write_bytes(b'headers') + generated.write_text('#define PINEFORGE_VERSION_STRING "0.14.0"\n') + compiler = compiler_identity(self.cxx) + cache = copied_cache(self._cache_values()) + if kind == 'mismatch': + cache = dict(cache) + cache['PINEFORGE_VERSION_SOURCE'] = 'AUTO' + receipt = { + 'schemaVersion': 'pineforge-settlement-abi-base/v1', + 'commit': PRIOR_COMMIT if prior else BASE_COMMIT, + 'tree': PRIOR_TREE if prior else BASE_TREE, + 'archive': 'build/lib/libpineforge.a', + 'archiveSha256': identity(archive)['sha256'], + 'headers': headers.name, + 'headersSha256': identity(headers)['sha256'], + 'generatedInclude': 'build/include', + 'generatedHeaderSha256': identity(generated)['sha256'], + 'compiler': compiler, + 'copiedCurrentCache': cache, + } + (output / 'receipt.json').write_text(json.dumps(receipt, indent=2, sort_keys=True) + '\n') + + def names(self) -> list[str]: + names = [] + for argv in self.calls: + if argv[0] == 'cmake' and '--build' in argv and 'ci-smoke' not in ''.join(argv): + names.append('build') + elif argv[0] == 'cmake' and '--install' in argv: + names.append('install') + elif argv[0] == 'ctest' and '--test-dir' in argv: + names.append('ctest') + elif argv[0] == 'git' and 'fetch' in argv: + names.append('prior-fetch' if PRIOR_COMMIT in argv else 'fetch') + elif any(Path(part).name == 'prepare_settlement_cpp_abi_base.py' for part in argv): + names.append('prior-prepare' if PRIOR_COMMIT in argv else 'prepare') + elif argv[0] == 'cmake' and '-S' in argv and 'smoke_consumer' in ''.join(argv): + names.append('smoke-configure') + elif Path(argv[0]).name == 'smoke_version': + names.append('smoke-version') + elif Path(argv[0]).name == 'test_native_live_websocket': + names.append('require-websocket') + return names + + +def run_driver(profile: str, build_dir: Path, scripted: Scripted, extra: list[str] | None = None, + which=shutil.which) -> int: + argv = [profile, '--build-dir', str(build_dir), '--jobs', '2', *(extra or [])] + return main(argv, runner=scripted, stream_output=False, which=which) + + +def load_summary(build_dir: Path) -> dict: + return json.loads((build_dir / 'ci-summary.json').read_text()) + + +def stage_names(summary: dict) -> list[str]: + return [stage['name'] for stage in summary['stages']] + + +def failure_stages(summary: dict) -> list[str]: + return [item['stage'] for item in summary['failures']] + + +class ConfigValidation(unittest.TestCase): + def test_unknown_profile(self): + with self.assertRaises(ConfigError): + parse_args(['relaxed']) + self.assertEqual(main(['relaxed']), 2) + + def test_unknown_argument_is_rejected(self): + with self.assertRaises(ConfigError): + parse_args(['release', '--skip-tests']) + self.assertEqual(main(['release', '--skip-tests']), 2) + + def test_jobs_range(self): + with self.assertRaisesRegex(ConfigError, '--jobs'): + validate_config(parse_args(['release', '--jobs', '0'])) + with self.assertRaisesRegex(ConfigError, '--jobs'): + validate_config(parse_args(['release', '--jobs', '99'])) + self.assertEqual(main(['release', '--jobs', '0']), 2) + + def test_require_websocket_only_native(self): + with self.assertRaisesRegex(ConfigError, 'native'): + validate_config(parse_args(['release', '--require-websocket'])) + self.assertEqual(main(['debug', '--require-websocket']), 2) + + def test_ccache_requires_installed_tool(self): + args = parse_args(['release', '--ccache']) + with self.assertRaisesRegex(ConfigError, 'ccache'): + validate_config(args, which=which_with_ccache(None)) + self.assertEqual(main(['release', '--ccache'], which=which_with_ccache(None)), 2) + + def test_ccache_binds_compiler_launchers(self): + launcher = '/usr/bin/true' + cfg = validate_config(parse_args(['release', '--ccache', '--build-dir', 'build-ci-x']), + which=which_with_ccache(launcher)) + values = cmake_cache_definitions(cfg) + resolved = str(Path(launcher).resolve()) + self.assertEqual(values['CMAKE_C_COMPILER_LAUNCHER'], resolved) + self.assertEqual(values['CMAKE_CXX_COMPILER_LAUNCHER'], resolved) + self.assertEqual(values['PINEFORGE_VERSION_SOURCE'], 'FILE') + + def test_curl_dir_must_exist(self): + missing = ROOT / 'definitely-missing-curl-dir' + with self.assertRaisesRegex(ConfigError, 'curl-dir'): + validate_config(parse_args(['native', '--curl-dir', str(missing)])) + + def test_default_build_dirs_are_profile_isolated(self): + for name in ('release', 'debug', 'sanitizers', 'native'): + self.assertEqual(default_build_dir(ROOT, name), ROOT / f'build-ci-{name}') + args = parse_args([name], source=ROOT) + self.assertEqual(Path(args.build_dir), ROOT / f'build-ci-{name}') + + def test_build_dir_cannot_be_source_root(self): + with self.assertRaisesRegex(ConfigError, 'source root'): + validate_config(parse_args(['release', '--build-dir', str(ROOT)])) + + +class ProfileOptions(unittest.TestCase): + def definitions(self, profile: str, extra=None, which=shutil.which): + args = parse_args([profile, '--build-dir', 'tmp-build', *(extra or [])], source=ROOT) + cfg = validate_config(args, which=which) + return cmake_cache_definitions(cfg), cmake_configure_argv(cfg) + + def test_release_keeps_tutorial_on_native_off(self): + values, argv = self.definitions('release') + self.assertEqual(values['CMAKE_BUILD_TYPE'], 'Release') + self.assertEqual(values['PINEFORGE_BUILD_TUTORIAL'], 'ON') + self.assertEqual(values['PINEFORGE_BUILD_LIVE_RUNNER'], 'OFF') + self.assertEqual(values['PINEFORGE_ENABLE_SANITIZERS'], 'OFF') + self.assertEqual(values['PINEFORGE_BUILD_TESTS'], 'ON') + self.assertEqual(values['PINEFORGE_VERSION_SOURCE'], 'FILE') + self.assertIn('-DPINEFORGE_VERSION_SOURCE=FILE', argv) + self.assertNotIn('AUTO', ''.join(argv)) + + def test_debug_explicit(self): + values, _ = self.definitions('debug') + self.assertEqual(values['CMAKE_BUILD_TYPE'], 'Debug') + self.assertEqual(values['PINEFORGE_BUILD_TUTORIAL'], 'ON') + self.assertEqual(values['PINEFORGE_BUILD_LIVE_RUNNER'], 'OFF') + self.assertEqual(values['PINEFORGE_ENABLE_SANITIZERS'], 'OFF') + + def test_sanitizers_public_flag_expected_and_native_off(self): + values, argv = self.definitions('sanitizers') + self.assertEqual(values['CMAKE_BUILD_TYPE'], 'Debug') + self.assertEqual(values['PINEFORGE_ENABLE_SANITIZERS'], 'ON') + self.assertEqual(values['PINEFORGE_BUILD_LIVE_RUNNER'], 'OFF') + self.assertEqual(values['PINEFORGE_BUILD_TUTORIAL'], 'ON') + self.assertIn('-DPINEFORGE_ENABLE_SANITIZERS=ON', argv) + + def test_native_live_on_tutorial_off(self): + curl = tempfile.TemporaryDirectory() + self.addCleanup(curl.cleanup) + values, argv = self.definitions( + 'native', extra=['--generator', 'Ninja', '--curl-dir', curl.name, '--require-websocket']) + self.assertEqual(values['PINEFORGE_BUILD_LIVE_RUNNER'], 'ON') + self.assertEqual(values['PINEFORGE_BUILD_TUTORIAL'], 'OFF') + self.assertEqual(values['PINEFORGE_ENABLE_SANITIZERS'], 'OFF') + self.assertEqual(values['CURL_DIR'], str(Path(curl.name).resolve())) + self.assertIn('-G', argv) + self.assertIn('Ninja', argv) + + +class CopyCacheIdentity(unittest.TestCase): + def test_copy_cache_includes_version_source_and_launchers(self): + self.assertIn('PINEFORGE_VERSION_SOURCE', COPY_CACHE) + self.assertIn('CMAKE_C_COMPILER_LAUNCHER', COPY_CACHE) + self.assertIn('CMAKE_CXX_COMPILER_LAUNCHER', COPY_CACHE) + + def test_receipt_mismatch_on_version_source(self): + compiler = {'target': 't', 'sha256': 'abc', 'version': 'cc'} + receipt = { + 'schemaVersion': 'pineforge-settlement-abi-base/v1', + 'commit': BASE_COMMIT, + 'tree': BASE_TREE, + 'compiler': compiler, + 'copiedCurrentCache': {'PINEFORGE_VERSION_SOURCE': 'AUTO'}, + } + with self.assertRaisesRegex(RuntimeError, 'version-source/launcher'): + receipt_matches_current(receipt, {'PINEFORGE_VERSION_SOURCE': 'FILE'}, compiler) + + def test_receipt_match_on_identical_identity(self): + compiler = {'target': 't', 'sha256': 'abc', 'version': 'cc'} + cache = {'PINEFORGE_VERSION_SOURCE': 'FILE', 'CMAKE_CXX_COMPILER_LAUNCHER': '/ccache'} + receipt = { + 'schemaVersion': 'pineforge-settlement-abi-base/v1', + 'commit': BASE_COMMIT, + 'tree': BASE_TREE, + 'compiler': compiler, + 'copiedCurrentCache': copied_cache(cache), + } + receipt_matches_current(receipt, cache, compiler) + + +class ReceiptReuse(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + self.build = self.root / 'build' + self.output = self.build / 'settlement-abi-base' + self.cxx = shutil.which('c++') or '/usr/bin/c++' + self.output.mkdir(parents=True) + (self.output / 'sentinel').write_text('keep\n') + + def tearDown(self): + self.temporary.cleanup() + + def seed(self, *, version='FILE', corrupt=False, receipt=True, prior=False): + archive = self.output / 'build' / 'lib' / 'libpineforge.a' + headers = self.output / 'r2-headers.tar' + generated = self.output / 'build' / 'include' / 'pineforge' / 'version.h' + archive.parent.mkdir(parents=True, exist_ok=True) + generated.parent.mkdir(parents=True, exist_ok=True) + archive.write_bytes(b'!\nbase\n') + headers.write_bytes(b'headers') + generated.write_text('version\n') + write_cache(self.build / 'CMakeCache.txt', { + 'CMAKE_CXX_COMPILER': self.cxx, + 'PINEFORGE_VERSION_SOURCE': 'FILE', + 'CMAKE_BUILD_TYPE': 'Release', + }) + if not receipt: + return + current = copied_cache(read_cache_for_test(self.build / 'CMakeCache.txt')) + payload = { + 'schemaVersion': 'pineforge-settlement-abi-base/v1', + 'commit': PRIOR_COMMIT if prior else BASE_COMMIT, + 'tree': PRIOR_TREE if prior else BASE_TREE, + 'archive': 'build/lib/libpineforge.a', + 'archiveSha256': identity(archive)['sha256'], + 'headers': 'r2-headers.tar', + 'headersSha256': identity(headers)['sha256'], + 'generatedInclude': 'build/include', + 'generatedHeaderSha256': identity(generated)['sha256'], + 'compiler': compiler_identity(self.cxx), + 'copiedCurrentCache': dict(current), + } + if version != 'FILE': + payload['copiedCurrentCache']['PINEFORGE_VERSION_SOURCE'] = version + if corrupt: + payload['archiveSha256'] = '0' * 64 + (self.output / 'receipt.json').write_text(json.dumps(payload) + '\n') + + def test_matching_receipt_is_reused(self): + self.seed() + receipt = reusable_prepared_base(self.output, self.build) + self.assertEqual(receipt['commit'], BASE_COMMIT) + self.assertTrue((self.output / 'sentinel').is_file()) + + def test_mismatch_refuses_without_deletion(self): + self.seed(version='AUTO') + with self.assertRaisesRegex(RuntimeError, 'fresh --build-dir'): + reusable_prepared_base(self.output, self.build) + self.assertTrue((self.output / 'sentinel').is_file()) + self.assertTrue((self.output / 'receipt.json').is_file()) + + def test_missing_receipt_refuses_without_deletion(self): + self.seed(receipt=False) + with self.assertRaisesRegex(RuntimeError, 'fresh --build-dir'): + reusable_prepared_base(self.output, self.build) + self.assertTrue((self.output / 'sentinel').is_file()) + + def test_corrupt_archive_hash_refuses_without_deletion(self): + self.seed(corrupt=True) + with self.assertRaisesRegex(RuntimeError, 'fresh --build-dir'): + reusable_prepared_base(self.output, self.build) + self.assertTrue((self.output / 'sentinel').is_file()) + + def test_prior_reuse_pins_its_own_commit_and_tree(self): + self.seed(prior=True) + receipt = reusable_prepared_base(self.output, self.build, commit=PRIOR_COMMIT, tree=PRIOR_TREE) + self.assertEqual(receipt['commit'], PRIOR_COMMIT) + with self.assertRaisesRegex(RuntimeError, 'does not pin e60 R2'): + reusable_prepared_base(self.output, self.build) + with self.assertRaisesRegex(RuntimeError, 'does not pin 0e18690'): + reusable_prepared_base(self.output, self.build, commit=PRIOR_COMMIT, tree=BASE_TREE) + self.assertTrue((self.output / 'sentinel').is_file()) + + def test_prior_reuse_also_refuses_version_source_mismatch(self): + self.seed(prior=True, version='AUTO') + with self.assertRaisesRegex(RuntimeError, 'fresh --build-dir'): + reusable_prepared_base(self.output, self.build, commit=PRIOR_COMMIT, tree=PRIOR_TREE) + self.assertTrue((self.output / 'sentinel').is_file()) + + +def read_cache_for_test(path: Path) -> dict[str, str]: + from prepare_settlement_cpp_abi_base import read_cache + return read_cache(path) + + +class DriverOrderingAndAggregation(unittest.TestCase): + def test_local_ccache_uses_content_identity(self): + seen = [] + def runner(argv, **kwargs): + seen.append(kwargs["extra_env"]) + return Completed(0) + with tempfile.TemporaryDirectory() as temporary: + cfg = ci_verify.build_config( + ["release", "--build-dir", temporary, "--ccache"], runner=runner, + stream_output=False, which=lambda name: "/usr/bin/" + name) + driver = ci_verify.Driver(cfg) + driver.invoke("probe", ["compile"], extra_env={"PRESERVED_SETTING": "yes"}) + self.assertEqual(seen, [{"PRESERVED_SETTING": "yes", "CCACHE_COMPILERCHECK": "content"}]) + + def test_runner_type_error_is_not_retried(self): + calls = [] + def failing_runner(argv, **kwargs): + calls.append(argv) + raise TypeError("runner failure after dispatch") + with self.assertRaisesRegex(TypeError, "after dispatch"): + ci_verify.call_runner(failing_runner, ["build-once"]) + self.assertEqual(calls, [["build-once"]]) + + def run_profile(self, profile='release', extra=None, **exits): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + build_dir = Path(temporary.name) / 'build' + scripted = Scripted(build_dir, ROOT, profile, **exits) + code = run_driver(profile, build_dir, scripted, extra=extra) + summary = load_summary(build_dir) + return code, summary, scripted, build_dir + + def test_source_guards_are_real_and_pass_on_this_tree(self): + for name, argv in source_guard_commands(ROOT): + with self.subTest(name=name): + result = default_runner(argv, extra_env=None, timeout=120, + combine_stderr=True, stream_output=False) + self.assertEqual(result.returncode, 0, result.stdout[-2000:]) + + def test_build_failure_skips_ctest_and_install(self): + code, summary, scripted, _ = self.run_profile(build=1) + self.assertEqual(code, 1) + self.assertIn('build', failure_stages(summary)) + self.assertNotIn('ctest', scripted.names()) + self.assertNotIn('install', scripted.names()) + self.assertNotIn('smoke-version', scripted.names()) + + def test_configure_failure_skips_build(self): + code, summary, scripted, _ = self.run_profile(configure=1) + self.assertEqual(code, 1) + self.assertIn('configure', failure_stages(summary)) + self.assertNotIn('build', scripted.names()) + self.assertNotIn('ctest', scripted.names()) + + def test_source_guard_failure_skips_configure(self): + code, summary, scripted, _ = self.run_profile(**{'source-guard-c-abi': 1}) + self.assertEqual(code, 1) + self.assertIn('source-guard-c-abi', failure_stages(summary)) + self.assertFalse(any(argv[0] == 'cmake' and '-S' in argv for argv in scripted.calls)) + + def test_ctest_and_smoke_failures_are_both_collected(self): + code, summary, scripted, _ = self.run_profile(ctest=8, **{'smoke-stdout': '9.9.9'}) + self.assertEqual(code, 1) + self.assertIn('ctest', scripted.names()) + self.assertIn('install', scripted.names()) + self.assertIn('smoke-version', scripted.names()) + self.assertIn('ctest', failure_stages(summary)) + self.assertIn('smoke-version-check', failure_stages(summary)) + message = next(item['error'] for item in summary['failures'] if item['stage'] == 'smoke-version-check') + self.assertIn(repr(expected_version(ROOT)), message) + self.assertIn(repr('9.9.9'), message) + self.assertEqual(summary['expectedVersion'], expected_version(ROOT)) + self.assertEqual(summary['actualVersion'], '9.9.9') + + def test_ctest_failure_does_not_hide_install_failure(self): + code, summary, scripted, _ = self.run_profile(ctest=8, install=1) + self.assertEqual(code, 1) + self.assertIn('ctest', failure_stages(summary)) + self.assertIn('install', failure_stages(summary)) + self.assertNotIn('smoke-version', scripted.names()) + + def test_successful_scripted_release_exit_zero(self): + code, summary, scripted, build_dir = self.run_profile() + self.assertEqual(code, 0, summary['failures']) + self.assertEqual(summary['status'], 'passed') + self.assertEqual(summary['exitCode'], 0) + self.assertEqual(summary['versionSource'], 'FILE') + self.assertIn('abi-base', stage_names(summary)) + self.assertIn('abi-prior', stage_names(summary)) + names = stage_names(summary) + self.assertLess(names.index('build'), names.index('abi-base')) + self.assertLess(names.index('abi-base'), names.index('abi-prior')) + self.assertLess(names.index('abi-prior'), names.index('ctest')) + self.assertIn('ctest', scripted.names()) + self.assertIn('install', scripted.names()) + self.assertTrue((build_dir / 'ci-logs' / 'ctest.log').is_file()) + self.assertTrue((build_dir / 'ci-summary.json').is_file()) + ctest_argv = next(argv for argv in scripted.calls if argv[0] == 'ctest' and '--test-dir' in argv) + if ctest_supports_junit(scripted): + self.assertIn('--output-junit', ctest_argv) + self.assertEqual(Path(ctest_argv[ctest_argv.index('--output-junit') + 1]).resolve(), + (build_dir / 'ctest-junit.xml').resolve()) + + def test_junit_flag_omitted_when_unsupported(self): + code, summary, scripted, build_dir = self.run_profile(junit_help='absent') + self.assertEqual(code, 0, summary['failures']) + ctest_argv = next(argv for argv in scripted.calls if argv[0] == 'ctest' and '--test-dir' in argv) + self.assertNotIn('--output-junit', ctest_argv) + + def test_matching_base_is_reused_without_fetch_or_prepare(self): + code, summary, scripted, _ = self.run_profile(preexisting_base='match') + self.assertEqual(code, 0, summary['failures']) + self.assertEqual(summary['abi']['action'], 'reused') + self.assertNotIn('fetch', scripted.names()) + self.assertNotIn('prepare', scripted.names()) + + def test_mismatch_base_refuses_without_deletion_and_still_runs_ctest_install(self): + code, summary, scripted, build_dir = self.run_profile(preexisting_base='mismatch') + self.assertEqual(code, 1) + self.assertEqual(summary['abi']['action'], 'refused') + self.assertIn('fresh --build-dir', summary['failures'][0]['error']) + self.assertTrue((build_dir / 'settlement-abi-base' / 'sentinel').is_file()) + self.assertNotIn('prepare', scripted.names()) + self.assertIn('ctest', scripted.names()) + self.assertIn('install', scripted.names()) + + def test_fetch_no_tags_only_when_object_missing(self): + code, summary, scripted, _ = self.run_profile(**{'cat-file': 1}) + self.assertEqual(code, 0, summary['failures']) + fetch = next(argv for argv in scripted.calls if argv[0] == 'git' and 'fetch' in argv) + self.assertIn('--no-tags', fetch) + self.assertNotIn('--tags', fetch) + self.assertIn(BASE_COMMIT, fetch) + self.assertIn('prepare', scripted.names()) + + def test_present_object_does_not_fetch(self): + _, _, scripted, _ = self.run_profile() + self.assertNotIn('fetch', scripted.names()) + self.assertIn('prepare', scripted.names()) + + def test_matching_prior_is_reused_without_fetch_or_prepare(self): + code, summary, scripted, _ = self.run_profile(preexisting_prior='match') + self.assertEqual(code, 0, summary['failures']) + self.assertEqual(summary['abiPrior']['action'], 'reused') + self.assertNotIn('prior-fetch', scripted.names()) + self.assertNotIn('prior-prepare', scripted.names()) + + def test_mismatch_prior_refuses_without_deletion_and_keeps_other_checks(self): + code, summary, scripted, build_dir = self.run_profile(preexisting_prior='mismatch') + self.assertEqual(code, 1) + self.assertEqual(summary['abiPrior']['action'], 'refused') + self.assertIn('abi-prior', failure_stages(summary)) + self.assertIn('fresh --build-dir', summary['failures'][0]['error']) + self.assertTrue((build_dir / 'settlement-abi-prior' / 'sentinel').is_file()) + self.assertTrue((build_dir / 'settlement-abi-prior' / 'receipt.json').is_file()) + self.assertNotIn('prior-prepare', scripted.names()) + self.assertIn('ctest', scripted.names()) + self.assertIn('install', scripted.names()) + + def test_prior_fetch_is_exact_and_only_when_object_missing(self): + code, summary, scripted, build_dir = self.run_profile(**{'prior-cat-file': 1}) + self.assertEqual(code, 0, summary['failures']) + fetches = [argv for argv in scripted.calls if argv[0] == 'git' and 'fetch' in argv] + self.assertEqual(fetches, [['git', '-C', str(ROOT), 'fetch', '--no-tags', '--depth=1', + 'origin', PRIOR_COMMIT]]) + self.assertIn('abi-prior-fetch', stage_names(summary)) + prepare = next(argv for argv in scripted.calls + if any(Path(part).name == 'prepare_settlement_cpp_abi_base.py' for part in argv) + and PRIOR_COMMIT in argv) + self.assertEqual(prepare[prepare.index('--tree') + 1], PRIOR_TREE) + self.assertEqual(Path(prepare[prepare.index('--output') + 1]).resolve(), + (build_dir / 'settlement-abi-prior').resolve()) + self.assertEqual(prepare[prepare.index('--header-manifest') + 1], + str(ROOT / 'tests/fixtures/settlement_cpp_abi/0e18690/manifest.json')) + + def test_present_prior_object_does_not_fetch(self): + code, summary, scripted, _ = self.run_profile(**{'prior-cat-file': 0}) + self.assertEqual(code, 0, summary['failures']) + self.assertNotIn('prior-fetch', scripted.names()) + self.assertIn('prior-prepare', scripted.names()) + + def test_prior_fetch_failure_does_not_prepare_and_keeps_other_checks(self): + code, summary, scripted, _ = self.run_profile(**{'prior-cat-file': 1, 'prior-fetch': 1}) + self.assertEqual(code, 1) + self.assertEqual(summary['abiPrior']['action'], 'failed') + self.assertIn('abi-prior-fetch', failure_stages(summary)) + self.assertNotIn('prior-prepare', scripted.names()) + self.assertIn('ctest', scripted.names()) + self.assertIn('install', scripted.names()) + + def test_require_websocket_rejects_skip(self): + code, summary, scripted, _ = self.run_profile( + 'native', extra=['--require-websocket'], **{'require-websocket': 77}) + self.assertEqual(code, 1) + self.assertIn('require-websocket', scripted.names()) + self.assertIn('require-websocket-skip', failure_stages(summary)) + self.assertTrue(any('cannot accept skip' in (item.get('error') or '') + for item in summary['failures'])) + + def test_require_websocket_rejects_missing_binary(self): + code, summary, scripted, _ = self.run_profile( + 'native', extra=['--require-websocket'], omit_ws_binary=True) + self.assertEqual(code, 1) + self.assertIn('require-websocket', failure_stages(summary)) + self.assertNotIn('require-websocket', scripted.names()) + + def test_native_help_runs_after_install(self): + code, summary, scripted, _ = self.run_profile('native') + self.assertEqual(code, 0, summary['failures']) + self.assertIn('native-help', stage_names(summary)) + names = stage_names(summary) + self.assertLess(names.index('install'), names.index('native-help')) + + def test_sanitizer_missing_public_flag_fails_before_build(self): + code, summary, scripted, _ = self.run_profile('sanitizers', sanitizer_flag='absent') + self.assertEqual(code, 1) + self.assertIn('sanitizer-public-flag', failure_stages(summary)) + self.assertNotIn('build', scripted.names()) + + def test_sanitizers_profile_passes_public_flag_and_asan_env(self): + code, summary, scripted, _ = self.run_profile('sanitizers') + self.assertEqual(code, 0, summary['failures']) + self.assertIn('sanitizer-public-flag', stage_names(summary)) + ctest = next(stage for stage in summary['stages'] if stage['name'] == 'ctest') + self.assertEqual(ctest['extraEnvKeys'], ['ASAN_OPTIONS', 'UBSAN_OPTIONS']) + self.assertNotIn('ctest', failure_stages(summary)) + + def test_smoke_ignores_stderr_noise_and_checks_stdout(self): + code, summary, _, _ = self.run_profile(**{'smoke-stderr': 'AddressSanitizer noise'}) + self.assertEqual(code, 0, summary['failures']) + self.assertEqual(summary['actualVersion'], expected_version(ROOT)) + + def test_logs_and_summary_omit_environment_secrets(self): + secret = 's3cret-ci-verify-token-not-for-logs' + previous = os.environ.get('CI_VERIFY_TEST_SECRET') + os.environ['CI_VERIFY_TEST_SECRET'] = secret + os.environ['AWS_SECRET_ACCESS_KEY'] = secret + try: + code, summary, scripted, build_dir = self.run_profile() + self.assertEqual(code, 0, summary['failures']) + dumped = json.dumps(summary) + for path in (build_dir / 'ci-logs').glob('*'): + dumped += path.read_text(errors='replace') + self.assertNotIn(secret, dumped) + self.assertFalse(scripted.secret_seen_in_argv) + self.assertNotIn('AWS_SECRET_ACCESS_KEY', dumped) + finally: + os.environ.pop('AWS_SECRET_ACCESS_KEY', None) + if previous is None: + os.environ.pop('CI_VERIFY_TEST_SECRET', None) + else: + os.environ['CI_VERIFY_TEST_SECRET'] = previous + + def test_ctest_junit_detection_matches_real_ctest(self): + help_text = default_runner(['ctest', '--help'], extra_env=None, timeout=30, + combine_stderr=True, stream_output=False) + self.assertEqual(help_text.returncode, 0) + self.assertEqual(ctest_supports_junit(default_runner), + '--output-junit' in help_text.stdout.decode()) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_cmake_version_source.py b/scripts/test_cmake_version_source.py new file mode 100644 index 000000000..34f864664 --- /dev/null +++ b/scripts/test_cmake_version_source.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +"""Real CMake/Git fixtures for PINEFORGE_VERSION_SOURCE AUTO|FILE. + +Does not build the engine, bump VERSION, retag the runtime, or invoke git in +the source worktree. Each case is an isolated temporary tree. +""" +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CMAKE_DIR = ROOT / "cmake" +MODULE = CMAKE_DIR / "PineForgeVersion.cmake" +PROBE_NAME = "version_probe.txt" + +PROBE_CMAKELISTS = """cmake_minimum_required(VERSION 3.16) +list(APPEND CMAKE_MODULE_PATH "@CMAKE_DIR@") +include(PineForgeVersion) +pineforge_resolve_version() +file(WRITE "${CMAKE_BINARY_DIR}/version_probe.txt" + "PINEFORGE_VERSION_MAJOR=${PINEFORGE_VERSION_MAJOR}\\n" + "PINEFORGE_VERSION_MINOR=${PINEFORGE_VERSION_MINOR}\\n" + "PINEFORGE_VERSION_PATCH=${PINEFORGE_VERSION_PATCH}\\n" + "PINEFORGE_VERSION_MMP=${PINEFORGE_VERSION_MMP}\\n" + "PINEFORGE_VERSION_FULL=${PINEFORGE_VERSION_FULL}\\n" + "PINEFORGE_VERSION_GIT_SHA=${PINEFORGE_VERSION_GIT_SHA}\\n" + "PINEFORGE_VERSION_DIRTY=${PINEFORGE_VERSION_DIRTY}\\n" + "PINEFORGE_VERSION_SOURCE=${PINEFORGE_VERSION_SOURCE}\\n" +) +project(pineforge_version_probe VERSION ${PINEFORGE_VERSION_MMP} LANGUAGES NONE) +""" + + +def _which(name: str) -> str: + path = shutil.which(name) + if not path: + raise RuntimeError(name + " is required for version-source fixtures") + return path + + +def parse_probe(path: Path) -> dict[str, str]: + found: dict[str, str] = {} + for line in path.read_text().splitlines(): + if not line or "=" not in line: + raise AssertionError("unreadable probe line: " + line) + key, value = line.split("=", 1) + found[key] = value + return found + + +def cache_value(build: Path, name: str) -> str | None: + for line in (build / "CMakeCache.txt").read_text().splitlines(): + if line.startswith(name + ":") and "=" in line: + return line.split("=", 1)[1] + return None + + +class VersionSourceTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.cmake_bin = _which("cmake") + cls.git_bin = _which("git") + if not MODULE.is_file(): + raise RuntimeError("missing " + str(MODULE)) + + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory(prefix="pf-version-source-") + self.base = Path(self._tmp.name) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def git_env(self) -> dict[str, str]: + env = os.environ.copy() + for key in ( + "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_DIR_CEILING", + ): + env.pop(key, None) + env.update({ + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_TERMINAL_PROMPT": "0", + "GIT_AUTHOR_NAME": "version-source-test", + "GIT_AUTHOR_EMAIL": "version-source-test@pineforge.invalid", + "GIT_COMMITTER_NAME": "version-source-test", + "GIT_COMMITTER_EMAIL": "version-source-test@pineforge.invalid", + "GIT_AUTHOR_DATE": "2026-01-01T00:00:00Z", + "GIT_COMMITTER_DATE": "2026-01-01T00:00:00Z", + }) + return env + + def git(self, repo: Path, *args: str) -> str: + result = subprocess.run( + [self.git_bin, "-c", "init.defaultBranch=main", + "-c", "commit.gpgsign=false", "-c", "tag.gpgsign=false", *args], + cwd=repo, env=self.git_env(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False) + if result.returncode: + raise RuntimeError( + "git " + " ".join(args) + " failed (" + str(result.returncode) + "): " + + result.stderr.strip()) + return result.stdout.strip() + + def write_tree(self, source: Path, version: str) -> None: + source.mkdir(parents=True, exist_ok=True) + source.joinpath("VERSION").write_text(version + "\n", encoding="utf-8") + text = PROBE_CMAKELISTS.replace("@CMAKE_DIR@", CMAKE_DIR.as_posix()) + source.joinpath("CMakeLists.txt").write_text(text, encoding="utf-8") + + def init_repo(self, source: Path) -> None: + self.git(source, "init") + self.git(source, "add", "VERSION", "CMakeLists.txt") + self.git(source, "commit", "-m", "fixture") + + def expected_describe(self, repo: Path) -> str: + desc = self.git(repo, "describe", "--tags", "--match", "v*", "--abbrev=7", "--dirty") + if desc.startswith("v"): + desc = desc[1:] + return desc + + def expected_sha(self, repo: Path) -> str: + return self.git(repo, "rev-parse", "--short=7", "HEAD") + + def configure(self, source: Path, build: Path, *defs: str) -> subprocess.CompletedProcess[str]: + build.mkdir(parents=True, exist_ok=True) + return subprocess.run( + [self.cmake_bin, "-S", str(source), "-B", str(build), *defs], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=60) + + def probe(self, source: Path, label: str, *defs: str) -> dict[str, str]: + build = self.base / "build" / label + result = self.configure(source, build, *defs) + probe_path = build / PROBE_NAME + if result.returncode: + raise AssertionError( + "cmake configure failed (" + str(result.returncode) + "):\n" + result.stdout) + self.assertTrue(probe_path.is_file(), result.stdout) + values = parse_probe(probe_path) + values["_cache_source"] = cache_value(build, "PINEFORGE_VERSION_SOURCE") or "" + values["_log"] = result.stdout + values["_build"] = str(build) + return values + + def configure_fail(self, source: Path, label: str, *defs: str) -> str: + build = self.base / "build" / label + result = self.configure(source, build, *defs) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertFalse((build / PROBE_NAME).exists(), result.stdout) + return result.stdout + + def test_default_auto_matches_explicit_auto(self) -> None: + source = self.base / "auto-default" + self.write_tree(source, "0.14.0") + self.init_repo(source) + self.git(source, "tag", "v0.13.1") + omitted = self.probe(source, "omitted") + explicit = self.probe(source, "explicit", "-DPINEFORGE_VERSION_SOURCE=AUTO") + for key in ( + "PINEFORGE_VERSION_MAJOR", "PINEFORGE_VERSION_MINOR", "PINEFORGE_VERSION_PATCH", + "PINEFORGE_VERSION_MMP", "PINEFORGE_VERSION_FULL", "PINEFORGE_VERSION_GIT_SHA", + "PINEFORGE_VERSION_DIRTY", "PINEFORGE_VERSION_SOURCE", + ): + self.assertEqual(omitted[key], explicit[key], key) + full = self.expected_describe(source) + sha = self.expected_sha(source) + self.assertEqual(omitted["PINEFORGE_VERSION_SOURCE"], "AUTO") + self.assertEqual(omitted["_cache_source"], "AUTO") + self.assertEqual(explicit["_cache_source"], "AUTO") + self.assertEqual(omitted["PINEFORGE_VERSION_MMP"], "0.13.1") + self.assertEqual(omitted["PINEFORGE_VERSION_FULL"], full) + self.assertEqual(omitted["PINEFORGE_VERSION_GIT_SHA"], sha) + self.assertEqual(omitted["PINEFORGE_VERSION_DIRTY"], "OFF") + self.assertNotEqual(omitted["PINEFORGE_VERSION_MMP"], "0.14.0") + + def test_mismatched_tag_file_uses_version(self) -> None: + source = self.base / "mismatch" + self.write_tree(source, "0.14.0") + self.init_repo(source) + self.git(source, "tag", "v9.9.9") + source.joinpath("extra.txt").write_text("second commit\n", encoding="utf-8") + self.git(source, "add", "extra.txt") + self.git(source, "commit", "-m", "ahead of tag") + auto = self.probe(source, "mismatch-auto", "-DPINEFORGE_VERSION_SOURCE=AUTO") + file_mode = self.probe(source, "mismatch-file", "-DPINEFORGE_VERSION_SOURCE=FILE") + full = self.expected_describe(source) + sha = self.expected_sha(source) + self.assertEqual(auto["PINEFORGE_VERSION_MMP"], "9.9.9") + self.assertEqual(auto["PINEFORGE_VERSION_FULL"], full) + self.assertTrue(full.startswith("9.9.9-1-g"), full) + self.assertEqual(file_mode["PINEFORGE_VERSION_SOURCE"], "FILE") + self.assertEqual(file_mode["_cache_source"], "FILE") + self.assertEqual(file_mode["PINEFORGE_VERSION_MMP"], "0.14.0") + self.assertEqual(file_mode["PINEFORGE_VERSION_FULL"], "0.14.0") + self.assertEqual(file_mode["PINEFORGE_VERSION_MAJOR"], "0") + self.assertEqual(file_mode["PINEFORGE_VERSION_MINOR"], "14") + self.assertEqual(file_mode["PINEFORGE_VERSION_PATCH"], "0") + self.assertEqual(auto["PINEFORGE_VERSION_GIT_SHA"], sha) + self.assertEqual(file_mode["PINEFORGE_VERSION_GIT_SHA"], sha) + self.assertEqual(auto["PINEFORGE_VERSION_DIRTY"], "OFF") + self.assertEqual(file_mode["PINEFORGE_VERSION_DIRTY"], "OFF") + self.assertNotEqual(file_mode["PINEFORGE_VERSION_FULL"], auto["PINEFORGE_VERSION_FULL"]) + + def test_shallow_no_tag(self) -> None: + origin = self.base / "origin" + self.write_tree(origin, "2.3.4") + self.init_repo(origin) + self.git(origin, "tag", "v1.0.0") + origin.joinpath("later.txt").write_text("later\n", encoding="utf-8") + self.git(origin, "add", "later.txt") + self.git(origin, "commit", "-m", "untagged head") + clone = self.base / "shallow" + # Local clones otherwise copy all objects and are not shallow. + self.git(self.base, "clone", "--depth=1", "--no-tags", "--no-local", + str(origin), str(clone)) + tags = self.git(clone, "tag", "-l") + self.assertEqual(tags, "") + self.assertEqual(self.git(clone, "rev-parse", "--is-shallow-repository"), "true") + self.assertTrue((clone / ".git" / "shallow").is_file()) + describe = subprocess.run( + [self.git_bin, "describe", "--tags", "--match", "v*", "--abbrev=7", "--dirty"], + cwd=clone, env=self.git_env(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + self.assertNotEqual(describe.returncode, 0, describe.stdout + describe.stderr) + self.assertIn("No names found", describe.stderr) + sha = self.expected_sha(clone) + self.assertEqual(len(self.git(clone, "rev-list", "--all").splitlines()), 1) + auto = self.probe(clone, "shallow-auto", "-DPINEFORGE_VERSION_SOURCE=AUTO") + file_mode = self.probe(clone, "shallow-file", "-DPINEFORGE_VERSION_SOURCE=FILE") + for values, source in ((auto, "AUTO"), (file_mode, "FILE")): + self.assertEqual(values["PINEFORGE_VERSION_SOURCE"], source) + self.assertEqual(values["PINEFORGE_VERSION_MMP"], "2.3.4") + self.assertEqual(values["PINEFORGE_VERSION_FULL"], "2.3.4") + self.assertEqual(values["PINEFORGE_VERSION_GIT_SHA"], sha) + self.assertEqual(values["PINEFORGE_VERSION_DIRTY"], "OFF") + + def test_archive_no_git(self) -> None: + source = self.base / "archive" + self.write_tree(source, "0.14.0") + self.assertFalse((source / ".git").exists()) + auto = self.probe(source, "archive-auto") + file_mode = self.probe(source, "archive-file", "-DPINEFORGE_VERSION_SOURCE=FILE") + for values in (auto, file_mode): + self.assertEqual(values["PINEFORGE_VERSION_MMP"], "0.14.0") + self.assertEqual(values["PINEFORGE_VERSION_FULL"], "0.14.0") + self.assertEqual(values["PINEFORGE_VERSION_GIT_SHA"], "unknown") + self.assertEqual(values["PINEFORGE_VERSION_DIRTY"], "OFF") + self.assertEqual(auto["PINEFORGE_VERSION_SOURCE"], "AUTO") + self.assertEqual(file_mode["PINEFORGE_VERSION_SOURCE"], "FILE") + + def test_invalid_mode_fails_clearly(self) -> None: + source = self.base / "invalid" + self.write_tree(source, "0.14.0") + self.init_repo(source) + self.git(source, "tag", "v0.14.0") + for index, mode in enumerate(("BOGUS", "auto", "file", "GIT", "AUTO FILE", "")): + with self.subTest(mode=mode): + log = self.configure_fail( + source, "invalid-" + str(index), + "-DPINEFORGE_VERSION_SOURCE=" + mode) + self.assertIn("PINEFORGE_VERSION_SOURCE must be AUTO or FILE", log) + self.assertIn("got '" + mode + "'", log) + self.assertNotIn("Configuring done", log) + + def test_file_preserves_dirty_without_retagging_full(self) -> None: + source = self.base / "dirty" + self.write_tree(source, "3.2.1") + self.init_repo(source) + self.git(source, "tag", "v8.8.8") + source.joinpath("CMakeLists.txt").write_text( + source.joinpath("CMakeLists.txt").read_text(encoding="utf-8") + "\n# dirty\n", + encoding="utf-8") + full = self.expected_describe(source) + sha = self.expected_sha(source) + self.assertTrue(full.endswith("-dirty"), full) + auto = self.probe(source, "dirty-auto", "-DPINEFORGE_VERSION_SOURCE=AUTO") + file_mode = self.probe(source, "dirty-file", "-DPINEFORGE_VERSION_SOURCE=FILE") + self.assertEqual(auto["PINEFORGE_VERSION_MMP"], "8.8.8") + self.assertEqual(auto["PINEFORGE_VERSION_FULL"], full) + self.assertEqual(auto["PINEFORGE_VERSION_DIRTY"], "ON") + self.assertEqual(file_mode["PINEFORGE_VERSION_MMP"], "3.2.1") + self.assertEqual(file_mode["PINEFORGE_VERSION_FULL"], "3.2.1") + self.assertFalse(file_mode["PINEFORGE_VERSION_FULL"].endswith("-dirty")) + self.assertEqual(file_mode["PINEFORGE_VERSION_DIRTY"], "ON") + self.assertEqual(file_mode["PINEFORGE_VERSION_GIT_SHA"], sha) + self.assertEqual(auto["PINEFORGE_VERSION_GIT_SHA"], sha) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_settlement_cpp_abi.py b/scripts/test_settlement_cpp_abi.py new file mode 100644 index 000000000..d6376ac7d --- /dev/null +++ b/scripts/test_settlement_cpp_abi.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Narrow offline refusal tests for the settlement ABI tooling; no C++ execution.""" +import io +import json +from pathlib import Path +import shutil +import tempfile +import tarfile +from types import SimpleNamespace +import unittest + +from check_settlement_cpp_abi import ( + ENGINE, ROOT, REVERSAL_METHODS, REVERSAL_DOMAIN, frozen_shape, load_prior, + storage_declarations, validate_rejection, +) +from prepare_settlement_cpp_abi_base import BASE_COMMIT, BASE_TREE, extract_tar, read_cache + + +class AbiToolingTests(unittest.TestCase): + def test_mac_link_diagnostic_requires_only_named_method(self): + method='project_native_settlement_v1' + valid=f'Undefined symbols for architecture arm64:\n "{ENGINE}{method}(int) const", referenced from:\n _main\n' + self.assertEqual(len(validate_rejection(valid,[method])),1) + with self.assertRaisesRegex(RuntimeError,'unrelated'): + validate_rejection(valid+' "other_dependency()", referenced from:\n _main\n',[method]) + + def test_gnu_and_lld_diagnostics_keep_selection_domain(self): + method='inspect_native_settlement_selected' + symbol=ENGINE+method+'(pineforge::execution::close_selection_v1::SelectedOpeningSet const&) const' + for text in ["a.cpp: undefined reference to `"+symbol+"'",'ld.lld: error: undefined symbol: '+symbol]: + self.assertEqual(len(validate_rejection(text,[method],'close_selection_v1::SelectedOpeningSet')),1) + with self.assertRaisesRegex(RuntimeError,'namespace'): + validate_rejection(text,[method],'close_selection_v2::SelectedOpeningSet') + + def test_generic_failure_or_missing_one_method_cannot_pass(self): + with self.assertRaisesRegex(RuntimeError,'no recognized'): + validate_rejection('linker error: file not found',['project_native_settlement_v1']) + valid=f'undefined reference to `{ENGINE}project_native_settlement_v1(int)\'' + with self.assertRaisesRegex(RuntimeError,'omits expected'): + validate_rejection(valid,['project_native_settlement_v1','project_native_settlement_scoped_v1']) + + def reversal_diagnostic(self, style, methods=REVERSAL_METHODS, domain=REVERSAL_DOMAIN): + symbols=[ENGINE+method+'(pineforge::execution::'+domain+' const&, pineforge::execution::Fill const&)' + for method in methods] + formats={'mac': lambda symbol: ' "'+symbol+'", referenced from:\n _main', + 'gnu': lambda symbol: "caller.cpp: undefined reference to `"+symbol+"'", + 'lld': lambda symbol: 'ld.lld: error: undefined symbol: '+symbol} + return '\n'.join(formats[style](symbol) for symbol in symbols) + + def test_reversal_rejection_requires_all_four_names_and_parameter_domain(self): + for style in ('mac','gnu','lld'): + with self.subTest(style=style): + valid=self.reversal_diagnostic(style) + self.assertEqual(len(validate_rejection(valid,REVERSAL_METHODS,REVERSAL_DOMAIN)),4) + for omitted in REVERSAL_METHODS: + with self.assertRaisesRegex(RuntimeError,'omits expected'): + validate_rejection(self.reversal_diagnostic(style,[m for m in REVERSAL_METHODS if m!=omitted]), + REVERSAL_METHODS,REVERSAL_DOMAIN) + for bad_domain in ('reverse_to_v2::ReverseTo','ReverseTo'): + with self.assertRaisesRegex(RuntimeError,'namespace'): + validate_rejection(self.reversal_diagnostic(style,domain=bad_domain), + REVERSAL_METHODS,REVERSAL_DOMAIN) + with self.assertRaisesRegex(RuntimeError,'unrelated'): + validate_rejection(valid+"\nundefined reference to `other_dependency()'", + REVERSAL_METHODS,REVERSAL_DOMAIN) + + def test_missing_prior_receipt_fails_with_exact_prepare_remedy(self): + with tempfile.TemporaryDirectory() as temporary: + root=Path(temporary) + for receipt in (None,root/'missing.json'): + with self.assertRaisesRegex(RuntimeError,'settlement-abi-prior .*--commit 0e18690'): + load_prior(SimpleNamespace(prior_receipt=receipt),root/'headers',{}) + self.assertFalse((root/'headers').exists()) + + def test_e60_receipt_cannot_be_used_for_reversal_prior(self): + with tempfile.TemporaryDirectory() as temporary: + root=Path(temporary);receipt=root/'wrong-provider.json' + receipt.write_text(json.dumps({'commit':BASE_COMMIT,'tree':BASE_TREE})) + with self.assertRaisesRegex(RuntimeError,'does not pin 0e18690'): + load_prior(SimpleNamespace(prior_receipt=receipt),root/'headers',{}) + self.assertFalse((root/'headers').exists()) + + def test_action_alternative_changes_are_frozen(self): + with tempfile.TemporaryDirectory() as temporary: + copied=Path(temporary)/'include' + shutil.copytree(ROOT/'include',copied) + header=copied/'pineforge/execution.hpp' + source=header.read_text() + original='std::variant' + self.assertIn(original,source) + header.write_text(source.replace(original,original[:-1]+', int>')) + with self.assertRaisesRegex(RuntimeError,'Action alternative'): + frozen_shape(ROOT/'include',copied) + + def test_same_size_member_change_or_added_padding_member_is_visible(self): + source=(ROOT/'include/pineforge/engine.hpp').read_text() + original=storage_declarations(source) + changed=source.replace('double position_entry_price_', 'uint64_t position_entry_price_',1) + self.assertNotEqual(original,storage_declarations(changed)) + changed=source.replace('double position_entry_price_', 'int abi_padding_member;\n double position_entry_price_',1) + self.assertNotEqual(original,storage_declarations(changed)) + self.assertTrue(any('abi_padding_member' in item for item in storage_declarations(changed))) + + def test_tar_links_and_traversal_are_refused_before_extraction(self): + for name,kind in [('../escape',tarfile.REGTYPE),('link',tarfile.SYMTYPE)]: + data=io.BytesIO() + with tarfile.open(fileobj=data,mode='w') as archive: + info=tarfile.TarInfo(name);info.type=kind;info.linkname='outside' + archive.addfile(info) + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaisesRegex(RuntimeError,'archive'): + extract_tar(data.getvalue(),Path(temporary)/'source') + + def test_cache_preserves_spaces_and_semicolon_paths(self): + with tempfile.TemporaryDirectory() as temporary: + path=Path(temporary)/'CMakeCache.txt' + path.write_text('// comment\nCMAKE_CXX_FLAGS:STRING=-fsanitize=address,undefined -fno-omit-frame-pointer\nCMAKE_PREFIX_PATH:PATH=/one;/two\n') + found=read_cache(path) + self.assertEqual(found['CMAKE_CXX_FLAGS'],'-fsanitize=address,undefined -fno-omit-frame-pointer') + self.assertEqual(found['CMAKE_PREFIX_PATH'],'/one;/two') + + +if __name__=='__main__': + unittest.main() diff --git a/src/engine_execution.cpp b/src/engine_execution.cpp index 1a344d6c1..fd83b629c 100644 --- a/src/engine_execution.cpp +++ b/src/engine_execution.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace pineforge { @@ -48,6 +49,43 @@ CloseScopeInspection inspect_close_scope( return {execution::Status::Applied, held}; } +CloseScopeInspection inspect_selected_opening_set( + const execution::SelectedOpeningSet& selection, + const execution::Action& action, + int64_t current_cycle, const std::vector& lots, + std::unordered_set& ids_out) { + ids_out.clear(); + if (std::holds_alternative(action) + || selection.cycle <= 0 || selection.cycle != current_cycle + || selection.incarnations.empty()) + return {execution::Status::InvalidCloseTarget}; + std::unordered_set live; + live.reserve(lots.size()); + for (const auto& lot : lots) live.insert(lot.entry_incarnation); + ids_out.reserve(selection.incarnations.size()); + for (const auto id : selection.incarnations) { + if (id == 0 || !ids_out.insert(id).second + || live.find(id) == live.end()) { + ids_out.clear(); + return {execution::Status::InvalidCloseTarget}; + } + } + double held = 0.0; + for (const auto& lot : lots) { + if (ids_out.count(lot.entry_incarnation) == 0) continue; + held += lot.qty; + if (!std::isfinite(held)) { + ids_out.clear(); + return {execution::Status::InvalidBook}; + } + } + if (held == 0.0) { + ids_out.clear(); + return {execution::Status::InvalidCloseTarget}; + } + return {execution::Status::Applied, held}; +} + bool selected_for_close(const execution::CloseScope& scope, const PyramidEntry& lot) { if (std::holds_alternative(scope)) return true; @@ -84,8 +122,346 @@ CloseSplit next_close_split(const PyramidEntry& lot, bool closes, bool flatten, closed = next_closed; return {execution::Status::Applied, amount, kept}; } + +execution::AccountEffectProjection invalid_projection(execution::Status status) { + execution::AccountEffectProjection out; + out.status = status; + return out; +} } // namespace +struct BacktestEngine::NativeSettlementStage { + enum class Phase { Invalid, NoEffect, Ready }; + Phase phase = Phase::Invalid; + execution::Status status = execution::Status::NoEffect; + bool flatten = false; + bool scoped = false; + bool opposite = false; + bool closes = false; + bool use_selected = false; + PositionSide incoming = PositionSide::LONG; + bool was_long = false; + double requested = 0.0; + double selected_held = 0.0; + double allocation_requested = 0.0; + double closed = 0.0; + double opening = 0.0; + double ticket = 0.0; + double after_qty = 0.0; + double after_price = 0.0; + std::vector closing_indices; + std::vector closing_quantities; + std::vector current_costs; + std::vector survivors; + std::unordered_set selected_ids; +}; + +execution::Status BacktestEngine::validate_native_settlement_book(double& held) const { + using execution::Status; + held = 0.0; + if (position_side_ != PositionSide::FLAT + && position_side_ != PositionSide::LONG + && position_side_ != PositionSide::SHORT) + return Status::InvalidBook; + if ((position_side_ == PositionSide::FLAT) != pyramid_entries_.empty()) + return Status::InvalidBook; + for (const auto& lot : pyramid_entries_) { + if (!std::isfinite(lot.qty) || lot.qty <= 0.0 || !std::isfinite(lot.price)) + return Status::InvalidBook; + held += lot.qty; + if (!std::isfinite(held)) return Status::InvalidBook; + } + return Status::Applied; +} + +execution::Status BacktestEngine::allocate_native_settlement_closes( + NativeSettlementStage& stage, + const execution::CloseScope& book_or_opening, + double& remaining) const { + using execution::Status; + if (!stage.flatten || stage.scoped) + stage.survivors.reserve(pyramid_entries_.size()); + stage.closing_indices.reserve(stage.closes ? pyramid_entries_.size() : 0); + stage.closing_quantities.reserve(stage.closes ? pyramid_entries_.size() : 0); + double closed = 0.0; + for (size_t index = 0; index < pyramid_entries_.size(); ++index) { + const auto& lot = pyramid_entries_[index]; + const bool member = stage.use_selected + ? stage.selected_ids.count(lot.entry_incarnation) != 0 + : selected_for_close(book_or_opening, lot); + const auto split = next_close_split( + lot, stage.closes && member, stage.flatten, + stage.allocation_requested, closed, remaining); + if (split.status != Status::Applied) return split.status; + if (split.amount == 0.0) { + stage.survivors.push_back(lot); + continue; + } + stage.closing_indices.push_back(index); + stage.closing_quantities.push_back(split.amount); + if (split.kept > 0.0) { + auto survivor = lot; + const double scale = split.kept / lot.qty; + survivor.qty = split.kept; + survivor.max_runup *= scale; + survivor.max_drawdown *= scale; + survivor.entry_commission_account = open_entry_commission(lot) + - allocated_entry_commission(lot, split.amount); + stage.survivors.push_back(std::move(survivor)); + } + } + stage.closed = closed; + return Status::Applied; +} + +void BacktestEngine::stage_native_settlement( + NativeSettlementStage& stage, + const execution::Action& action, + const execution::Fill& fill, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected, + const execution::LifecycleEffects* lifecycle) const { + using execution::Status; + stage = NativeSettlementStage{}; + auto fail = [&](Status status) { + stage.phase = NativeSettlementStage::Phase::Invalid; + stage.status = status; + }; + auto no_effect = [&]() { + stage.phase = NativeSettlementStage::Phase::NoEffect; + stage.status = fill.commission_account && *fill.commission_account != 0.0 + ? Status::InvalidAccounting : Status::NoEffect; + }; + + if (!std::isfinite(fill.price)) { fail(Status::InvalidPrice); return; } + if (fill.commission_account && !std::isfinite(*fill.commission_account)) { + fail(Status::InvalidAccounting); + return; + } + + stage.flatten = std::holds_alternative(action); + const auto* reduce = std::get_if(&action); + const auto* transact = std::get_if(&action); + stage.requested = reduce ? reduce->units + : transact ? std::abs(transact->signed_units) : 0.0; + if (!std::isfinite(stage.requested) || (reduce && stage.requested < 0.0)) { + fail(Status::InvalidQuantity); + return; + } + double held = 0.0; + if (const auto status = validate_native_settlement_book(held); + status != Status::Applied) { + fail(status); + return; + } + if (lifecycle) { + if (auto invalid = validate_lifecycle_effects(*lifecycle)) { + fail(*invalid); + return; + } + } + + CloseScopeInspection selection; + if (selected) { + selection = inspect_selected_opening_set( + *selected, action, position_cycle_seq_, pyramid_entries_, + stage.selected_ids); + stage.use_selected = true; + stage.scoped = true; + } else { + selection = inspect_close_scope( + book_or_opening, action, position_cycle_seq_, pyramid_entries_, held); + stage.scoped = std::holds_alternative( + book_or_opening); + } + if (selection.status != Status::Applied) { + fail(selection.status); + return; + } + stage.selected_held = selection.held; + stage.allocation_requested = stage.scoped && reduce + ? std::min(stage.requested, selection.held) : stage.requested; + const double signed_held = position_side_ == PositionSide::SHORT + ? -selection.held : selection.held; + if (!stage.flatten && stage.requested == 0.0) { no_effect(); return; } + if ((stage.flatten || reduce) && pyramid_entries_.empty()) { + no_effect(); + return; + } + if (reduce && !order_action::plan(signed_held, + order_action::Reduce{stage.allocation_requested})) { + fail(Status::UnrepresentableQuantity); + return; + } + if (transact && !order_action::plan(signed_held, *transact)) { + fail(Status::UnrepresentableQuantity); + return; + } + + stage.incoming = transact && transact->signed_units < 0.0 + ? PositionSide::SHORT : PositionSide::LONG; + stage.opposite = transact && position_side_ != PositionSide::FLAT + && position_side_ != stage.incoming; + stage.closes = stage.flatten || reduce || stage.opposite; + stage.was_long = position_side_ == PositionSide::LONG; + + double remaining = stage.allocation_requested; + if (const auto status = allocate_native_settlement_closes( + stage, book_or_opening, remaining); status != Status::Applied) { + fail(status); + return; + } + stage.opening = !transact ? 0.0 : stage.opposite ? remaining : stage.requested; + if (stage.opening > 0.0 && stage.opposite && !stage.survivors.empty()) { + fail(Status::UnrepresentableQuantity); + return; + } + finish_native_settlement_stage(stage, fill); +} + +void BacktestEngine::stage_native_settlement( + NativeSettlementStage& stage, + const execution::ReverseTo& reversal, + const execution::Fill& fill, + const execution::LifecycleEffects* lifecycle) const { + using execution::Status; + stage = NativeSettlementStage{}; + auto fail = [&](Status status) { + stage.phase = NativeSettlementStage::Phase::Invalid; + stage.status = status; + }; + if (!std::isfinite(fill.price)) { fail(Status::InvalidPrice); return; } + if (fill.commission_account && !std::isfinite(*fill.commission_account)) { + fail(Status::InvalidAccounting); + return; + } + if (!std::isfinite(reversal.signed_units) || reversal.signed_units == 0.0) { + fail(Status::InvalidQuantity); + return; + } + double held = 0.0; + if (const auto status = validate_native_settlement_book(held); + status != Status::Applied) { + fail(status); + return; + } + if (lifecycle) { + if (auto invalid = validate_lifecycle_effects(*lifecycle)) { + fail(*invalid); + return; + } + } + stage.incoming = reversal.signed_units < 0.0 + ? PositionSide::SHORT : PositionSide::LONG; + if (position_side_ == PositionSide::FLAT || position_side_ == stage.incoming) { + fail(Status::InvalidCloseTarget); + return; + } + stage.flatten = true; + stage.opposite = true; + stage.closes = true; + stage.was_long = position_side_ == PositionSide::LONG; + stage.selected_held = held; + double remaining = 0.0; + if (const auto status = allocate_native_settlement_closes( + stage, execution::Book{}, remaining); status != Status::Applied) { + fail(status); + return; + } + // ReverseTo carries the exact new exposure independently of all closing + // lots. Neither transaction-flow subtraction nor absorption applies here. + stage.opening = std::abs(reversal.signed_units); + if (!std::isfinite(stage.closed + stage.opening)) { + fail(Status::UnrepresentableQuantity); + return; + } + finish_native_settlement_stage(stage, fill); +} + +void BacktestEngine::finish_native_settlement_stage( + NativeSettlementStage& stage, const execution::Fill& fill) const { + using execution::Status; + auto fail = [&](Status status) { + stage.phase = NativeSettlementStage::Phase::Invalid; + stage.status = status; + }; + stage.current_costs = quote_execution_commissions( + stage.closing_quantities, stage.opening, fill); + stage.ticket = 0.0; + for (double cost : stage.current_costs) { + if (!std::isfinite(cost)) { + fail(Status::InvalidAccounting); + return; + } + stage.ticket += cost; + if (!std::isfinite(stage.ticket)) { + fail(Status::InvalidAccounting); + return; + } + } + double after_qty = 0.0; + double weighted = 0.0; + for (const auto& lot : stage.survivors) { + after_qty += lot.qty; + weighted += lot.price * lot.qty; + } + if (stage.opening > 0.0) { + const double next = after_qty + stage.opening; + if (!std::isfinite(next) || (after_qty > 0.0 + && (next == after_qty || next == stage.opening))) { + fail(Status::UnrepresentableQuantity); + return; + } + after_qty = next; + weighted += fill.price * stage.opening; + } + const size_t after_lots = stage.survivors.size() + (stage.opening > 0.0 ? 1 : 0); + stage.after_qty = after_qty; + stage.after_price = after_lots == 1 + ? (stage.opening > 0.0 ? fill.price : stage.survivors.front().price) + : after_qty > 0.0 ? weighted / after_qty : 0.0; + stage.phase = NativeSettlementStage::Phase::Ready; + stage.status = Status::Applied; +} + +execution::SettlementInspection BacktestEngine::inspect_native_reversal_v1( + const execution::ReverseTo& reversal, const execution::Fill& fill) const { + NativeSettlementStage stage; + stage_native_settlement(stage, reversal, fill, nullptr); + return inspect_native_settlement_stage(stage, fill); +} + +execution::AccountEffectProjection BacktestEngine::project_native_reversal_v1( + const execution::ReverseTo& reversal, const execution::Fill& fill) const { + NativeSettlementStage stage; + stage_native_settlement(stage, reversal, fill, nullptr); + return project_native_settlement_stage(stage, fill); +} + +execution::Result BacktestEngine::settle_native_reversal_at_v1( + const execution::ReverseTo& reversal, const execution::Fill& fill, + const execution::PhysicalExecutionContext& context) { + const execution::LifecycleEffects lifecycle; + NativeSettlementStage stage; + stage_native_settlement(stage, reversal, fill, &lifecycle); + return commit_native_settlement_stage(stage, fill, lifecycle, context); +} + +execution::Result BacktestEngine::settle_reversal_with_lifecycle_v1( + const execution::ReverseTo& reversal, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle) { + execution::PhysicalExecutionContext context; + context.effective_time_ms = current_bar_.timestamp; + context.interval_index = bar_index_; + context.preceding_exit_path_prefix = fold_exit_path_extremes_; + if (!std::isnan(fold_exit_trail_peak_)) { + context.preceding_exit_trail_peak = fold_exit_trail_peak_; + } + NativeSettlementStage stage; + stage_native_settlement(stage, reversal, fill, &lifecycle); + return commit_native_settlement_stage(stage, fill, lifecycle, context); +} + execution::Result BacktestEngine::settle_resolved_execution( const execution::Action& action, const execution::Fill& fill) { return settle_execution_with_lifecycle(action, fill, {}); @@ -124,151 +500,89 @@ execution::Result BacktestEngine::settle_with_context( return settle_with_context_scoped(action, fill, lifecycle, context, execution::Book{}); } +execution::Result BacktestEngine::settle_native_execution_selected_at( + const execution::Action& action, const execution::Fill& fill, + const execution::PhysicalExecutionContext& context, + const execution::SelectedOpeningSet& selection) { + return settle_with_context_selected(action, fill, {}, context, selection); +} + +execution::Result BacktestEngine::settle_execution_selected_with_lifecycle( + const execution::Action& action, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::SelectedOpeningSet& selection) { + execution::PhysicalExecutionContext context; + context.effective_time_ms = current_bar_.timestamp; + context.interval_index = bar_index_; + context.preceding_exit_path_prefix = fold_exit_path_extremes_; + if (!std::isnan(fold_exit_trail_peak_)) { + context.preceding_exit_trail_peak = fold_exit_trail_peak_; + } + return settle_with_context_selected(action, fill, lifecycle, context, selection); +} + execution::Result BacktestEngine::settle_with_context_scoped( const execution::Action& action, const execution::Fill& fill, const execution::LifecycleEffects& lifecycle, const execution::PhysicalExecutionContext& context, execution::CloseScope scope) { + return settle_with_membership(action, fill, lifecycle, context, scope, nullptr); +} + +execution::Result BacktestEngine::settle_with_context_selected( + const execution::Action& action, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::PhysicalExecutionContext& context, + const execution::SelectedOpeningSet& selection) { + return settle_with_membership( + action, fill, lifecycle, context, execution::Book{}, &selection); +} + +execution::Result BacktestEngine::settle_with_membership( + const execution::Action& action, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::PhysicalExecutionContext& context, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected) { + NativeSettlementStage stage; + stage_native_settlement( + stage, action, fill, book_or_opening, selected, &lifecycle); + return commit_native_settlement_stage(stage, fill, lifecycle, context); +} + +execution::Result BacktestEngine::commit_native_settlement_stage( + NativeSettlementStage& stage, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::PhysicalExecutionContext& context) { using execution::Status; - if (!std::isfinite(fill.price)) return {Status::InvalidPrice}; - if (fill.commission_account && !std::isfinite(*fill.commission_account)) - return {Status::InvalidAccounting}; + if (stage.phase != NativeSettlementStage::Phase::Ready) + return {stage.status}; const auto no_effect = [&]() -> execution::Result { return {fill.commission_account && *fill.commission_account != 0.0 ? Status::InvalidAccounting : Status::NoEffect}; }; + if (stage.scoped && stage.closed == 0.0) return no_effect(); - const bool flatten = std::holds_alternative(action); - const auto* reduce = std::get_if(&action); - const auto* transact = std::get_if(&action); - const double requested = reduce ? reduce->units - : transact ? std::abs(transact->signed_units) : 0.0; - if (!std::isfinite(requested) || (reduce && requested < 0.0)) - return {Status::InvalidQuantity}; - - if (position_side_ != PositionSide::FLAT - && position_side_ != PositionSide::LONG - && position_side_ != PositionSide::SHORT) - return {Status::InvalidBook}; - if ((position_side_ == PositionSide::FLAT) != pyramid_entries_.empty()) - return {Status::InvalidBook}; - - double held = 0.0; - for (const auto& lot : pyramid_entries_) { - if (!std::isfinite(lot.qty) || lot.qty <= 0.0 || !std::isfinite(lot.price)) - return {Status::InvalidBook}; - held += lot.qty; - if (!std::isfinite(held)) return {Status::InvalidBook}; - } - if (auto invalid = validate_lifecycle_effects(lifecycle)) - return {*invalid}; - const auto selection = inspect_close_scope( - scope, action, position_cycle_seq_, pyramid_entries_, held); - if (selection.status != Status::Applied) return {selection.status}; - const bool scoped = std::holds_alternative(scope); - const double allocation_requested = scoped && reduce - ? std::min(requested, selection.held) : requested; - const double signed_held = position_side_ == PositionSide::SHORT - ? -selection.held : selection.held; - if (!flatten && requested == 0.0) return no_effect(); - if ((flatten || reduce) && pyramid_entries_.empty()) return no_effect(); - - // The scalar planner validates the request's representability. Physical - // FIFO allocation below is authoritative for the surviving position: it - // must not reject a valid decimal lot merely because a differently grouped - // scalar addition has a one-ulp difference. - if (reduce && !order_action::plan(signed_held, order_action::Reduce{allocation_requested})) - return {Status::UnrepresentableQuantity}; - if (transact && !order_action::plan(signed_held, *transact)) - return {Status::UnrepresentableQuantity}; - - const PositionSide incoming = transact && transact->signed_units < 0.0 - ? PositionSide::SHORT : PositionSide::LONG; - const bool opposite = transact && position_side_ != PositionSide::FLAT - && position_side_ != incoming; - const bool closes = flatten || reduce || opposite; - const bool was_long = position_side_ == PositionSide::LONG; - - // Stage all physical splits and financial rows before changing the book. - std::vector survivors; - std::vector closing_indices; - std::vector closing_quantities; - if (!flatten || scoped) survivors.reserve(pyramid_entries_.size()); - closing_indices.reserve(closes ? pyramid_entries_.size() : 0); - closing_quantities.reserve(closes ? pyramid_entries_.size() : 0); - double closed = 0.0; - double remaining = allocation_requested; - for (size_t index = 0; index < pyramid_entries_.size(); ++index) { - const auto& lot = pyramid_entries_[index]; - const auto split = next_close_split(lot, closes && selected_for_close(scope, lot), - flatten, allocation_requested, closed, remaining); - if (split.status != Status::Applied) return {split.status}; - const double amount = split.amount; - if (amount == 0.0) { - survivors.push_back(lot); - continue; - } - const double kept = split.kept; - closing_indices.push_back(index); - closing_quantities.push_back(amount); - if (kept > 0.0) { - auto survivor = lot; - const double scale = kept / lot.qty; - survivor.qty = kept; - survivor.max_runup *= scale; - survivor.max_drawdown *= scale; - survivor.entry_commission_account = open_entry_commission(lot) - - allocated_entry_commission(lot, amount); - survivors.push_back(std::move(survivor)); - } - } - - if (scoped && closed == 0.0) return no_effect(); - const double opening = !transact ? 0.0 : opposite ? remaining : requested; - if (opening > 0.0 && opposite && !survivors.empty()) - return {Status::UnrepresentableQuantity}; - const auto current_costs = quote_execution_commissions(closing_quantities, opening, fill); - for (double cost : current_costs) - if (!std::isfinite(cost)) return {Status::InvalidAccounting}; - const double opening_commission = current_costs.back(); std::vector closed_trades; - closed_trades.reserve(closing_indices.size()); - for (size_t i = 0; i < closing_indices.size(); ++i) { - const auto& lot = pyramid_entries_[closing_indices[i]]; - auto trade = build_close_trade_with_costs(lot, closing_quantities[i], - fill.price, was_long, allocated_entry_commission(lot, closing_quantities[i]), - current_costs[i], context); + closed_trades.reserve(stage.closing_indices.size()); + for (size_t i = 0; i < stage.closing_indices.size(); ++i) { + const auto& lot = pyramid_entries_[stage.closing_indices[i]]; + auto trade = build_close_trade_with_costs(lot, stage.closing_quantities[i], + fill.price, stage.was_long, + allocated_entry_commission(lot, stage.closing_quantities[i]), + stage.current_costs[i], context); trade.exit_id = fill.id; trade.exit_comment = fill.comment; closed_trades.push_back(std::move(trade)); } - if (opening > 0.0 && (position_side_ == PositionSide::FLAT || survivors.empty()) + if (stage.opening > 0.0 + && (position_side_ == PositionSide::FLAT || stage.survivors.empty()) && (next_position_cycle_seq_ <= 0 || next_position_cycle_seq_ == std::numeric_limits::max())) throw std::overflow_error("position cycle sequence exhausted"); - double after_qty = 0.0; - double weighted = 0.0; - for (const auto& lot : survivors) { - after_qty += lot.qty; - weighted += lot.price * lot.qty; - } - if (opening > 0.0) { - const double next = after_qty + opening; - if (!std::isfinite(next) || (after_qty > 0.0 - && (next == after_qty || next == opening))) - return {Status::UnrepresentableQuantity}; - after_qty = next; - weighted += fill.price * opening; - } - const size_t after_lots = survivors.size() + (opening > 0.0 ? 1 : 0); - const double after_price = after_lots == 1 - ? (opening > 0.0 ? fill.price : survivors.front().price) - : after_qty > 0.0 ? weighted / after_qty : 0.0; - if (!std::isfinite(after_qty) || !std::isfinite(after_price)) + if (!std::isfinite(stage.after_qty) || !std::isfinite(stage.after_price)) return {Status::InvalidAccounting}; - // A financial sum must remain representable; reporting-only percentages - // or excursion projections do not decide whether an action may settle. double next_profit = net_profit_sum_; double next_gross_profit = gross_profit_sum_; double next_gross_loss = gross_loss_sum_; @@ -285,27 +599,27 @@ execution::Result BacktestEngine::settle_with_context_scoped( || !std::isfinite(next_gross_loss) || !std::isfinite(next_intraday)) return {Status::InvalidAccounting}; validate_close_trade_counters(closed_trades.data(), closed_trades.size()); - if (opening > 0.0 && !survivors.empty() + if (stage.opening > 0.0 && !stage.survivors.empty() && position_entry_count_ == std::numeric_limits::max()) throw std::overflow_error("position entry counter exhausted"); - const bool will_reset = closed > 0.0 && survivors.empty(); - const bool will_open_quoted = opening > 0.0 - && (position_side_ == PositionSide::FLAT || survivors.empty()); + const bool will_reset = stage.closed > 0.0 && stage.survivors.empty(); + const bool will_open_quoted = stage.opening > 0.0 + && (position_side_ == PositionSide::FLAT || stage.survivors.empty()); if (auto invalid = preflight_settlement_lifecycle( lifecycle, will_reset, will_open_quoted)) return {*invalid}; const size_t first_trade = trades_.size(); const size_t first_action = stream_order_actions_.size(); - const size_t events = closed_trades.size() + (opening > 0.0 ? 1 : 0); + const size_t events = closed_trades.size() + (stage.opening > 0.0 ? 1 : 0); if (stream_observe_actions_) { if (events > std::numeric_limits::max() - stream_action_sequence_) throw std::overflow_error("stream action sequence overflow"); reserve_effects(stream_order_actions_, events); } reserve_effects(trades_, closed_trades.size()); - if (opening > 0.0) reserve_effects(pyramid_entries_, 1); + if (stage.opening > 0.0) reserve_effects(pyramid_entries_, 1); // Commit through the existing accounting/observation sinks. Allocation or // lifecycle exceptions still abort the owning engine run; this internal @@ -314,40 +628,40 @@ execution::Result BacktestEngine::settle_with_context_scoped( // unbind, authorized pending removals, then quoted opening bind. if (lifecycle.pre_close) apply_pre_close_lifecycle_batch(*lifecycle.pre_close); for (auto& trade : closed_trades) record_close_trade(std::move(trade)); - if (closed > 0.0) { - if (survivors.empty()) { + if (stage.closed > 0.0) { + if (stage.survivors.empty()) { reset_position_state_to_flat(); } else { - pyramid_entries_ = std::move(survivors); - position_qty_ = after_qty; - position_entry_price_ = after_price; + pyramid_entries_ = std::move(stage.survivors); + position_qty_ = stage.after_qty; + position_entry_price_ = stage.after_price; position_entry_count_ = static_cast(pyramid_entries_.size()); } } apply_authorized_pending_removals(lifecycle.removals); - if (opening > 0.0) { - PyramidEntry lot{fill.price, context.effective_time_ms, opening, fill.id, + if (stage.opening > 0.0) { + const double opening_commission = stage.current_costs.back(); + PyramidEntry lot{fill.price, context.effective_time_ms, stage.opening, fill.id, context.interval_index}; lot.entry_incarnation = fill.incarnation; lot.entry_comment = fill.comment; lot.entry_commission_account = opening_commission; if (position_side_ == PositionSide::FLAT) { - open_quoted_position(incoming, std::move(lot)); + open_quoted_position(stage.incoming, std::move(lot)); } else { - append_quoted_lot(std::move(lot), after_qty, after_price); + append_quoted_lot(std::move(lot), stage.after_qty, stage.after_price); } } if (stream_observe_actions_) stream_refresh_action_metadata(first_action, first_trade); - double ticket = 0.0; - for (double cost : current_costs) ticket += cost; execution::Result applied; applied.status = Status::Applied; - applied.closed_units = closed; - applied.opened_units = incoming == PositionSide::SHORT ? -opening : opening; - applied.current_ticket = ticket; + applied.closed_units = stage.closed; + applied.opened_units = stage.incoming == PositionSide::SHORT + ? -stage.opening : stage.opening; + applied.current_ticket = stage.ticket; applied.first_trade_index = first_trade; applied.closed_trade_count = closed_trades.size(); - applied.opened_lot_incarnation = opening > 0.0 ? fill.incarnation : 0; + applied.opened_lot_incarnation = stage.opening > 0.0 ? fill.incarnation : 0; return applied; } @@ -359,147 +673,212 @@ execution::SettlementInspection BacktestEngine::inspect_native_settlement( execution::SettlementInspection BacktestEngine::inspect_native_settlement_scoped( const execution::Action& action, const execution::Fill& fill, execution::CloseScope scope) const { + return inspect_with_membership(action, fill, scope, nullptr); +} + +execution::SettlementInspection BacktestEngine::inspect_native_settlement_selected( + const execution::Action& action, const execution::Fill& fill, + const execution::SelectedOpeningSet& selection) const { + return inspect_with_membership(action, fill, execution::Book{}, &selection); +} + +execution::SettlementInspection BacktestEngine::inspect_with_membership( + const execution::Action& action, const execution::Fill& fill, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected) const { + NativeSettlementStage stage; + stage_native_settlement( + stage, action, fill, book_or_opening, selected, nullptr); + return inspect_native_settlement_stage(stage, fill); +} + +execution::SettlementInspection BacktestEngine::inspect_native_settlement_stage( + const NativeSettlementStage& stage, const execution::Fill& fill) const { using execution::Status; execution::SettlementInspection out; - if (!std::isfinite(fill.price)) { - out.status = Status::InvalidPrice; - return out; - } - if (fill.commission_account && !std::isfinite(*fill.commission_account)) { - out.status = Status::InvalidAccounting; - return out; - } - const bool flatten = std::holds_alternative(action); - const auto* reduce = std::get_if(&action); - const auto* transact = std::get_if(&action); - const double requested = reduce ? reduce->units - : transact ? std::abs(transact->signed_units) : 0.0; - if (!std::isfinite(requested) || (reduce && requested < 0.0)) { - out.status = Status::InvalidQuantity; - return out; - } - if (position_side_ != PositionSide::FLAT - && position_side_ != PositionSide::LONG - && position_side_ != PositionSide::SHORT) { - out.status = Status::InvalidBook; - return out; - } - if ((position_side_ == PositionSide::FLAT) != pyramid_entries_.empty()) { - out.status = Status::InvalidBook; - return out; - } - double held = 0.0; - for (const auto& lot : pyramid_entries_) { - if (!std::isfinite(lot.qty) || lot.qty <= 0.0 || !std::isfinite(lot.price)) { - out.status = Status::InvalidBook; - return out; - } - held += lot.qty; - if (!std::isfinite(held)) { - out.status = Status::InvalidBook; - return out; - } - } - const auto selection = inspect_close_scope( - scope, action, position_cycle_seq_, pyramid_entries_, held); - if (selection.status != Status::Applied) { - out.status = selection.status; + if (stage.phase != NativeSettlementStage::Phase::Ready) { + out.status = stage.status; return out; } - const bool scoped = std::holds_alternative(scope); - const double allocation_requested = scoped && reduce - ? std::min(requested, selection.held) : requested; - const double signed_held = position_side_ == PositionSide::SHORT - ? -selection.held : selection.held; - if (!flatten && requested == 0.0) { - out.status = fill.commission_account && *fill.commission_account != 0.0 - ? Status::InvalidAccounting : Status::NoEffect; - return out; - } - if ((flatten || reduce) && pyramid_entries_.empty()) { + const double fx = active_account_currency_fx(); + out.status = Status::Applied; + out.closed_units = stage.closed; + out.opened_units = stage.incoming == PositionSide::SHORT + ? -stage.opening : stage.opening; + out.resulting_abs_units = stage.after_qty; + out.resulting_lot_count = stage.survivors.size() + (stage.opening > 0.0 ? 1 : 0); + out.resulting_abs_notional = stage.after_qty * std::abs(fill.price) + * syminfo_.pointvalue * fx; + out.current_ticket = stage.ticket; + out.would_open = stage.opening > 0.0; + out.incoming_short = stage.incoming == PositionSide::SHORT; + if (stage.closed == 0.0 && stage.opening == 0.0) { out.status = fill.commission_account && *fill.commission_account != 0.0 ? Status::InvalidAccounting : Status::NoEffect; - return out; - } - if (reduce && !order_action::plan(signed_held, order_action::Reduce{allocation_requested})) { - out.status = Status::UnrepresentableQuantity; - return out; + out.would_open = false; } - if (transact && !order_action::plan(signed_held, *transact)) { - out.status = Status::UnrepresentableQuantity; + return out; +} + +execution::AccountEffectProjection BacktestEngine::project_native_settlement_v1( + const execution::Action& action, const execution::Fill& fill) const { + return project_native_settlement_scoped_v1(action, fill, execution::Book{}); +} + +execution::AccountEffectProjection BacktestEngine::project_native_settlement_scoped_v1( + const execution::Action& action, const execution::Fill& fill, + execution::CloseScope scope) const { + return project_with_membership(action, fill, scope, nullptr); +} + +execution::AccountEffectProjection BacktestEngine::project_native_settlement_selected_v1( + const execution::Action& action, const execution::Fill& fill, + const execution::SelectedOpeningSet& selection) const { + return project_with_membership(action, fill, execution::Book{}, &selection); +} + +execution::AccountEffectProjection BacktestEngine::project_with_membership( + const execution::Action& action, const execution::Fill& fill, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected) const { + NativeSettlementStage stage; + stage_native_settlement( + stage, action, fill, book_or_opening, selected, nullptr); + return project_native_settlement_stage(stage, fill); +} + +execution::AccountEffectProjection BacktestEngine::project_native_settlement_stage( + const NativeSettlementStage& stage, const execution::Fill& fill) const { + using execution::Status; + if (stage.phase == NativeSettlementStage::Phase::Invalid) + return invalid_projection(stage.status); + + const auto unchanged_account = [&]() -> execution::AccountEffectProjection { + execution::AccountEffectProjection out; + out.status = Status::NoEffect; + const double fx = active_account_currency_fx(); + const double pv = syminfo_.pointvalue; + double abs_units = 0.0; + for (const auto& lot : pyramid_entries_) abs_units += lot.qty; + out.resulting_abs_units = abs_units; + out.resulting_lot_count = pyramid_entries_.size(); + out.resulting_abs_notional = + abs_units * std::abs(fill.price) * pv * fx; + double remaining = 0.0; + for (const auto& lot : pyramid_entries_) + remaining += open_entry_commission(lot); + out.realized_balance = initial_capital_ + net_profit_sum_; + out.remaining_entry_cost = remaining; + out.marked_equity = marked_equity(fill.price); + out.cycle_after = pyramid_entries_.empty() ? 0 : position_cycle_seq_; + out.signed_units_after = position_side_ == PositionSide::SHORT + ? -abs_units + : position_side_ == PositionSide::LONG ? abs_units : 0.0; + if (!std::isfinite(out.resulting_abs_units) + || !std::isfinite(out.resulting_abs_notional) + || !std::isfinite(out.realized_balance) + || !std::isfinite(out.remaining_entry_cost) + || !std::isfinite(out.marked_equity) + || !std::isfinite(out.signed_units_after)) + return invalid_projection(Status::InvalidAccounting); return out; + }; + + if (stage.phase == NativeSettlementStage::Phase::NoEffect) { + if (stage.status != Status::NoEffect) + return invalid_projection(stage.status); + return unchanged_account(); } - const PositionSide incoming = transact && transact->signed_units < 0.0 - ? PositionSide::SHORT : PositionSide::LONG; - const bool opposite = transact && position_side_ != PositionSide::FLAT - && position_side_ != incoming; - const bool closes = flatten || reduce || opposite; - std::vector survivors; - std::vector closing_quantities; - if (!flatten || scoped) survivors.reserve(pyramid_entries_.size()); - closing_quantities.reserve(closes ? pyramid_entries_.size() : 0); - double closed = 0.0; - double remaining = allocation_requested; - for (const auto& lot : pyramid_entries_) { - const auto split = next_close_split(lot, closes && selected_for_close(scope, lot), - flatten, allocation_requested, closed, remaining); - if (split.status != Status::Applied) { - out.status = split.status; - return out; - } - const double amount = split.amount; - if (amount == 0.0) { - survivors.push_back(lot); - continue; - } - const double kept = split.kept; - closing_quantities.push_back(amount); - if (kept > 0.0) { - auto survivor = lot; - survivor.qty = kept; - survivors.push_back(std::move(survivor)); - } - } - const double opening = !transact ? 0.0 : opposite ? remaining : requested; - if (opening > 0.0 && opposite && !survivors.empty()) { - out.status = Status::UnrepresentableQuantity; - return out; + if (stage.closed == 0.0 && stage.opening == 0.0) { + if (fill.commission_account && *fill.commission_account != 0.0) + return invalid_projection(Status::InvalidAccounting); + return unchanged_account(); } - const auto current_costs = quote_execution_commissions(closing_quantities, opening, fill); - double ticket = 0.0; - for (double cost : current_costs) { - if (!std::isfinite(cost)) { - out.status = Status::InvalidAccounting; - return out; - } - ticket += cost; + + if (stage.opening > 0.0 + && (position_side_ == PositionSide::FLAT || stage.survivors.empty()) + && (next_position_cycle_seq_ <= 0 + || next_position_cycle_seq_ == std::numeric_limits::max())) + throw std::overflow_error("position cycle sequence exhausted"); + + execution::PhysicalExecutionContext context; + std::vector closed_trades; + closed_trades.reserve(stage.closing_indices.size()); + for (size_t i = 0; i < stage.closing_indices.size(); ++i) { + const auto& lot = pyramid_entries_[stage.closing_indices[i]]; + auto trade = build_close_trade_with_costs(lot, stage.closing_quantities[i], + fill.price, stage.was_long, + allocated_entry_commission(lot, stage.closing_quantities[i]), + stage.current_costs[i], context); + closed_trades.push_back(std::move(trade)); } - double after_qty = 0.0; - for (const auto& lot : survivors) after_qty += lot.qty; - if (opening > 0.0) { - const double next = after_qty + opening; - if (!std::isfinite(next) || (after_qty > 0.0 - && (next == after_qty || next == opening))) { - out.status = Status::UnrepresentableQuantity; - return out; - } - after_qty = next; + + double realized = net_profit_sum_; + for (const auto& trade : closed_trades) { + if (!std::isfinite(trade.pnl) || !std::isfinite(trade.commission)) + return invalid_projection(Status::InvalidAccounting); + realized += trade.pnl; } + realized += initial_capital_; + if (!std::isfinite(realized)) + return invalid_projection(Status::InvalidAccounting); + const double fx = active_account_currency_fx(); + const double pv = syminfo_.pointvalue; + const PositionSide resulting_side = !stage.survivors.empty() + ? position_side_ + : stage.opening > 0.0 ? stage.incoming : PositionSide::FLAT; + const double direction = resulting_side == PositionSide::SHORT ? -1.0 : 1.0; + double remaining = 0.0; + double equity = realized; + for (const auto& lot : stage.survivors) { + if (!std::isfinite(lot.qty) || lot.qty <= 0.0 || !std::isfinite(lot.price)) + return invalid_projection(Status::InvalidAccounting); + const double paid = open_entry_commission(lot); + remaining += paid; + equity += direction * (fill.price - lot.price) * lot.qty * pv * fx - paid; + } + if (stage.opening > 0.0) { + const double paid = stage.current_costs.back(); + remaining += paid; + equity += direction * (fill.price - fill.price) * stage.opening * pv * fx + - paid; + } + if (!std::isfinite(remaining) || !std::isfinite(equity) + || !std::isfinite(stage.after_qty) + || !std::isfinite(stage.ticket)) + return invalid_projection(Status::InvalidAccounting); + + execution::AccountEffectProjection out; out.status = Status::Applied; - out.closed_units = closed; - out.opened_units = incoming == PositionSide::SHORT ? -opening : opening; - out.resulting_abs_units = after_qty; - out.resulting_lot_count = survivors.size() + (opening > 0.0 ? 1 : 0); - out.resulting_abs_notional = after_qty * std::abs(fill.price) * syminfo_.pointvalue * fx; - out.current_ticket = ticket; - out.would_open = opening > 0.0; - out.incoming_short = incoming == PositionSide::SHORT; - if (closed == 0.0 && opening == 0.0) { - out.status = fill.commission_account && *fill.commission_account != 0.0 - ? Status::InvalidAccounting : Status::NoEffect; - out.would_open = false; - } + out.closed_units = stage.closed; + out.opened_units = stage.incoming == PositionSide::SHORT + ? -stage.opening : stage.opening; + out.resulting_abs_units = stage.after_qty; + out.resulting_lot_count = stage.survivors.size() + (stage.opening > 0.0 ? 1 : 0); + out.resulting_abs_notional = + stage.after_qty * std::abs(fill.price) * pv * fx; + out.current_ticket = stage.ticket; + out.would_open = stage.opening > 0.0; + out.incoming_short = stage.incoming == PositionSide::SHORT; + out.realized_balance = realized; + out.remaining_entry_cost = remaining; + out.marked_equity = equity; + if (out.resulting_lot_count == 0) { + out.cycle_after = 0; + out.signed_units_after = 0.0; + } else if (!stage.survivors.empty()) { + out.cycle_after = position_cycle_seq_; + out.signed_units_after = position_side_ == PositionSide::SHORT + ? -stage.after_qty : stage.after_qty; + } else { + out.cycle_after = next_position_cycle_seq_; + out.signed_units_after = stage.incoming == PositionSide::SHORT + ? -stage.after_qty : stage.after_qty; + } + if (!std::isfinite(out.resulting_abs_notional) + || !std::isfinite(out.signed_units_after)) + return invalid_projection(Status::InvalidAccounting); return out; } diff --git a/src/engine_fills.cpp b/src/engine_fills.cpp index 2ee91d805..ef671c616 100644 --- a/src/engine_fills.cpp +++ b/src/engine_fills.cpp @@ -29,6 +29,20 @@ using namespace internal; namespace { +std::size_t source_opening_fragment_count(const std::vector& lots, + uint64_t incarnation) { + return static_cast(std::count_if(lots.begin(), lots.end(), + [&](const PyramidEntry& lot) { return lot.entry_incarnation == incarnation; })); +} + +bool source_opening_was_created(const std::vector& lots, + uint64_t incarnation, int64_t cycle_before, + int64_t cycle_after, std::size_t fragments_before) { + return !lots.empty() && lots.back().entry_incarnation == incarnation + && (cycle_after != cycle_before + || source_opening_fragment_count(lots, incarnation) == fragments_before + 1); +} + // A pass keeps identities and ordering hints, never borrowed vector elements. // The hint makes the unchanged-book path constant time; OCA erasure requires // re-resolution by incarnation. A reused label/priority cannot match this key. @@ -6429,6 +6443,10 @@ void BacktestEngine::apply_filled_order_to_state( || std::abs(position_qty_ - position_qty_before_fill) > kQtyEpsilon || pyramid_entries_.size() != pyramid_lots_before_fill || trades_.size() != trades_before; + const bool opening_fill_applied = !pyramid_entries_.empty() + && pyramid_entries_.back().entry_incarnation == order.incarnation + && (position_cycle_seq_ != position_cycle_before_fill + || pyramid_entries_.size() > pyramid_lots_before_fill); max_intraday_filled_orders_.outcome( primary_fill_applied ? compat::pine::FillOutcome::Committed @@ -6456,7 +6474,7 @@ void BacktestEngine::apply_filled_order_to_state( } if (primary_fill_applied) { - if (order.type == OrderType::MARKET && process_orders_on_close_ + if (opening_fill_applied && order.type == OrderType::MARKET && process_orders_on_close_ && order.created_bar == bar_index_ && order.created_position_side == PositionSide::FLAT && !placement_has_prior_close(order) @@ -6471,7 +6489,7 @@ void BacktestEngine::apply_filled_order_to_state( || (coof_scheduler_active_ && coof_cursor_is_bar_close_))) { pyramid_entries_.front().pooc_terminal_market_entry = true; } - if (order.type == OrderType::MARKET + if (opening_fill_applied && order.type == OrderType::MARKET && !process_orders_on_close_ && !calc_on_order_fills_ && !bar_magnifier_enabled_ && !coof_scheduler_active_ && !stream_warmup_mode_ && stream_phase_ == StreamPhase::IDLE @@ -6482,7 +6500,7 @@ void BacktestEngine::apply_filled_order_to_state( && position_entry_price_ == round_to_mintick(bar.open)) { pyramid_entries_.front().ordinary_market_open = true; } - if (order.type == OrderType::ENTRY + if (opening_fill_applied && order.type == OrderType::ENTRY && std::isfinite(order.legs.prices().stop_price) && std::isnan(order.legs.prices().limit_price) && !order.stop_limit_activated @@ -6964,6 +6982,13 @@ void BacktestEngine::apply_market_order_fill(PendingOrder& order, double fill_pr const Bar& bar, double& trail_best_path_state, bool later_same_tick_entry) { + const int64_t source_cycle_before = position_cycle_seq_; + const std::size_t source_fragments_before = + source_opening_fragment_count(pyramid_entries_, order.incarnation); + const auto new_source_opening = [&]() { + return source_opening_was_created(pyramid_entries_, order.incarnation, + source_cycle_before, position_cycle_seq_, source_fragments_before); + }; // design-market-entry-affordability: the entry leg was declined (at // placement or at fill) while an OPPOSITE position was live — execute the // reversal's closing leg only (rampatel BTC 2025-05-12 07:15Z: TV closed @@ -7009,7 +7034,7 @@ void BacktestEngine::apply_market_order_fill(PendingOrder& order, double fill_pr keep_mc_close_surplus ? -1 : order.qty_type, /*explicit_qty_prequantized=*/keep_mc_close_surplus, /*close_only=*/!keep_mc_close_surplus, order.incarnation); - if (keep_mc_close_surplus && !pyramid_entries_.empty()) + if (keep_mc_close_surplus && new_source_opening()) pyramid_entries_.back().entry_comment = order.comment; } trail_best_path_state = trail_best_price_; @@ -7086,27 +7111,10 @@ void BacktestEngine::apply_market_order_fill(PendingOrder& order, double fill_pr const double entry_fill = apply_fill_slippage(fill_price, order.is_long); if (std::isfinite(entry_fill) && add_qty > kQtyEpsilon) { - const double total_qty = position_qty_ + add_qty; - position_entry_price_ = - (position_entry_price_ * position_qty_ - + entry_fill * add_qty) / total_qty; - position_qty_ = total_qty; - ++position_entry_count_; - trail_best_price_ = entry_fill; - PyramidEntry lot{}; - lot.price = entry_fill; - lot.time = current_bar_.timestamp; - lot.qty = add_qty; - lot.entry_id = order.id; - lot.entry_bar_index = bar_index_; - lot.entry_comment = order.comment; - lot.entry_incarnation = order.incarnation; - lot.market_pyramid_add = true; - snapshot_entry_commission(lot); - pyramid_entries_.push_back(std::move(lot)); - if (stream_observe_actions_) stream_observe_entry(pyramid_entries_.back()); - id_unclosed_qty_[order.id] += add_qty; - cycle_filled_entry_ids_.insert(order.id); + const auto result = settle_source_opening( + requested, entry_fill, add_qty, order.id, order.comment, order.incarnation); + if (result.status == execution::Status::Applied && result.opened_units != 0.0) + pyramid_entries_.back().market_pyramid_add = true; } const double trail_best_after_fill = trail_best_price_; if (position_side_ == PositionSide::LONG) { @@ -7160,7 +7168,7 @@ void BacktestEngine::apply_market_order_fill(PendingOrder& order, double fill_pr throw std::logic_error("replacement cancellation receipt rejected"); } } - if (position_side_ == PositionSide::SHORT && !pyramid_entries_.empty()) + if (position_side_ == PositionSide::SHORT && new_source_opening()) pyramid_entries_.back().entry_comment = order.comment; const double trail_best_after_fill = trail_best_price_; if (position_side_ == PositionSide::LONG) @@ -7194,9 +7202,7 @@ void BacktestEngine::apply_market_order_fill(PendingOrder& order, double fill_pr order.incarnation); double trail_best_after_fill = trail_best_price_; // Set entry comment on the just-created pyramid entry - if (!pyramid_entries_.empty() - && (!paired_flat_market - || pyramid_entries_.back().entry_id == order.id)) { + if (new_source_opening()) { pyramid_entries_.back().entry_comment = order.comment; } // Update trail_best_price_ with intra-bar extremes for same-bar exit eval @@ -7226,6 +7232,9 @@ void BacktestEngine::apply_entry_order_fill(PendingOrder& order, double fill_pri double qty_before = position_qty_; int count_before = position_entry_count_; size_t trades_before_entry = trades_.size(); + const int64_t source_cycle_before = position_cycle_seq_; + const std::size_t source_fragments_before = + source_opening_fragment_count(pyramid_entries_, order.incarnation); // A pending priced (stop/limit) ENTRY that reaches its trigger while an // OPPOSITE position it did NOT open is live closes that position at the @@ -7354,7 +7363,10 @@ void BacktestEngine::apply_entry_order_fill(PendingOrder& order, double fill_pri bool was_priced_entry = priced_entry; if (did_execute) { double trail_best_after_fill = trail_best_price_; - if (!pyramid_entries_.empty()) pyramid_entries_.back().entry_comment = order.comment; + const bool new_source_opening = source_opening_was_created( + pyramid_entries_, order.incarnation, source_cycle_before, + position_cycle_seq_, source_fragments_before); + if (new_source_opening) pyramid_entries_.back().entry_comment = order.comment; // See apply_market_order_fill's matching guard: skip folding this // bar's pre-fill high/low into the trail when the fill happened AT // the bar's close (a POOC entry created and filled this same bar). @@ -7372,7 +7384,7 @@ void BacktestEngine::apply_entry_order_fill(PendingOrder& order, double fill_pri // Mask pre-fill bar extremes for the entry this fill created // (guard: back() really is this order's same-bar entry — a // close-only-opposite fill creates no new entry). - if (!pyramid_entries_.empty() + if (new_source_opening && pyramid_entries_.back().entry_bar_index == bar_index_ && pyramid_entries_.back().entry_id == order.id) { set_entry_fill_excursion_masks(pyramid_entries_.back(), bar, @@ -7729,29 +7741,12 @@ void BacktestEngine::apply_raw_order_fill(PendingOrder& order, double fill_price // Prefer the signal-time frozen quantity when the order carries one. double qty = !std::isnan(order.frozen_default_qty) ? order.frozen_default_qty : (std::isnan(order.qty) ? calc_qty(fill_price) : order.qty); - position_side_ = order.is_long ? PositionSide::LONG : PositionSide::SHORT; - position_cycle_seq_ = next_position_cycle_seq_++; - position_entry_price_ = fill_price; - // The shared post-dispatch hook queues the new fill's event. Clear any - // prior-cycle provenance first; RAW_ORDER opens do not route through - // open_fresh_position. - opening_obligations_.invalidate(); - position_entry_time_ = current_bar_.timestamp; - position_qty_ = qty; - position_entry_count_ = 1; - position_open_bar_ = bar_index_; - trail_best_price_ = fill_price; - pyramid_entries_.clear(); - id_unclosed_qty_.clear(); - cycle_filled_entry_ids_.clear(); - pyramid_entries_.push_back({fill_price, current_bar_.timestamp, qty, order.id, bar_index_}); - pyramid_entries_.back().entry_incarnation = order.incarnation; - snapshot_entry_commission(pyramid_entries_.back()); - if (stream_observe_actions_) stream_observe_entry(pyramid_entries_.back()); - id_unclosed_qty_[order.id] += qty; - cycle_filled_entry_ids_.insert(order.id); - bind_retained_exit_activations(); - if (!std::isnan(order.legs.prices().stop_price) || !std::isnan(order.legs.prices().limit_price)) { + const PositionSide requested = order.is_long ? PositionSide::LONG : PositionSide::SHORT; + const auto result = settle_source_opening( + requested, fill_price, qty, order.id, order.comment, order.incarnation); + if (result.status == execution::Status::Applied && result.opened_units != 0.0 + && (!std::isnan(order.legs.prices().stop_price) + || !std::isnan(order.legs.prices().limit_price))) { set_entry_fill_excursion_masks(pyramid_entries_.back(), current_bar_, fill_price); } trail_best_path_state = trail_best_price_; @@ -7794,23 +7789,14 @@ void BacktestEngine::apply_raw_order_fill(PendingOrder& order, double fill_price // Prefer the signal-time frozen quantity when the order carries one. double new_qty = !std::isnan(order.frozen_default_qty) ? order.frozen_default_qty : (std::isnan(order.qty) ? calc_qty(fill_price) : order.qty); - double total_qty = position_qty_ + new_qty; - position_entry_price_ = - (position_entry_price_ * position_qty_ + fill_price * new_qty) / total_qty; - position_qty_ = total_qty; - position_entry_count_++; - trail_best_price_ = fill_price; - pyramid_entries_.push_back({fill_price, current_bar_.timestamp, new_qty, order.id, bar_index_}); - pyramid_entries_.back().entry_incarnation = order.incarnation; - snapshot_entry_commission(pyramid_entries_.back()); - if (stream_observe_actions_) stream_observe_entry(pyramid_entries_.back()); + const auto result = settle_source_opening( + requested, fill_price, new_qty, order.id, order.comment, order.incarnation); // KI-62: flag same-direction MARKET adds (strategy.order path) so a // same-bar from_entry bracket exit can scratch them dur-0. - pyramid_entries_.back().market_pyramid_add = !is_priced_entry; - id_unclosed_qty_[order.id] += new_qty; - cycle_filled_entry_ids_.insert(order.id); - if (is_priced_entry) { - set_entry_fill_excursion_masks(pyramid_entries_.back(), current_bar_, fill_price); + if (result.status == execution::Status::Applied && result.opened_units != 0.0) { + pyramid_entries_.back().market_pyramid_add = !is_priced_entry; + if (is_priced_entry) + set_entry_fill_excursion_masks(pyramid_entries_.back(), current_bar_, fill_price); } } else { execute_market_exit(fill_price); diff --git a/src/engine_orders.cpp b/src/engine_orders.cpp index a43b680a6..0eb95db9d 100644 --- a/src/engine_orders.cpp +++ b/src/engine_orders.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,75 @@ namespace pineforge { using namespace internal; +namespace { +// Existing source FIFO endpoint policy; never a native quantity tolerance. +// Keep the R2 stop/whole-lot interpretation at 1e-10 in this adapter. +constexpr double kSourceFifoEndpointEpsilon = kQtyEpsilon; + +std::optional source_fifo_prefix_membership( + const std::vector& lots, double qty_limit, + int64_t cycle) { + if (cycle <= 0 || !std::isfinite(qty_limit) || qty_limit <= 0.0) + return std::nullopt; + + double qty_closed = 0.0; + size_t prefix_size = 0; + for (const auto& lot : lots) { + // Match the source's original accumulation and endpoint ordering. + // Once at the endpoint, even a tiny next sibling stays unselected. + if (qty_closed >= qty_limit - kSourceFifoEndpointEpsilon) break; + if (!std::isfinite(lot.qty) || lot.qty <= 0.0) return std::nullopt; + const double close_qty = std::min(lot.qty, qty_limit - qty_closed); + const double keep_qty = lot.qty - close_qty; + if (keep_qty > kSourceFifoEndpointEpsilon) return std::nullopt; + ++prefix_size; + qty_closed += close_qty; + } + if (prefix_size == 0 || prefix_size == lots.size()) return std::nullopt; + + execution::SelectedOpeningSet selection{cycle, {}}; + std::unordered_set included; + double selected_qty = 0.0; + for (size_t index = 0; index < prefix_size; ++index) { + const auto& lot = lots[index]; + if (lot.entry_incarnation == 0) return std::nullopt; + if (included.insert(lot.entry_incarnation).second) + selection.incarnations.push_back(lot.entry_incarnation); + selected_qty += lot.qty; + if (!std::isfinite(selected_qty)) return std::nullopt; + } + // An opening identity may have multiple physical fragments, but all of + // its live fragments must belong to this prefix. Otherwise use Reduce. + for (size_t index = prefix_size; index < lots.size(); ++index) { + if (included.count(lots[index].entry_incarnation) != 0) + return std::nullopt; + } + return selection; +} + +// Source predicates are resolved here, never retained by native settlement. +// Every fragment of an opening must agree with the selected source predicate. +template +std::vector source_opening_membership( + const std::vector& lots, Predicate selected) { + std::unordered_map membership; + std::vector incarnations; + for (const auto& lot : lots) { + const bool matches = selected(lot); + if (lot.entry_incarnation == 0) { + if (matches) + throw std::runtime_error("invalid resolved bound-close settlement: unowned opening"); + continue; + } + const auto [it, inserted] = membership.emplace(lot.entry_incarnation, matches); + if (!inserted && it->second != matches) + throw std::runtime_error("invalid resolved bound-close settlement: heterogeneous opening"); + if (inserted && matches) incarnations.push_back(lot.entry_incarnation); + } + return incarnations; +} +} // namespace + // Risk management + per-trade extreme tracking moved to engine_risk.cpp. @@ -25,6 +95,39 @@ double BacktestEngine::calc_qty_for_type(double fill_price, double qty_value, in if (std::isnan(qty_value)) { return calc_qty(fill_price); } + const double equity = qty_type == static_cast(QtyType::PERCENT_OF_EQUITY) + ? percent_commission_live_equity(round_to_mintick(current_bar_.close)) : 0.0; + return calc_qty_for_type_from_equity(fill_price, qty_value, qty_type, equity); +} + +double BacktestEngine::calc_default_qty_from_equity(double fill_price, double equity) const { + const double basis = round_to_mintick(fill_price); + switch (default_qty_type_) { + case QtyType::FIXED: + return apply_qty_step(default_qty_value_); + case QtyType::PERCENT_OF_EQUITY: { + // Source money precision and commission reservation stay exactly + // here for both live-equity and post-close projection callers. + if (tv_money_lot_sizing()) equity = tv_money_round(equity); + if (!std::isfinite(equity)) return 0.0; + const double cash = reserve_percent_commission( + equity * (default_qty_value_ / 100.0)) / active_account_currency_fx(); + if (!(std::isfinite(basis) && basis > 0.0)) return 0.0; + if (tv_money_lot_sizing()) + return tv_money_floor_lot(cash / (basis * syminfo_.pointvalue), qty_step_); + return apply_qty_step(cash / (basis * syminfo_.pointvalue)); + } + case QtyType::CASH: + return (std::isfinite(basis) && basis > 0.0) + ? apply_qty_step((default_qty_value_ / active_account_currency_fx()) + / (basis * syminfo_.pointvalue)) : 0.0; + } + return apply_qty_step(default_qty_value_); +} + +double BacktestEngine::calc_qty_for_type_from_equity( + double fill_price, double qty_value, int qty_type, double equity) const { + if (std::isnan(qty_value)) return calc_default_qty_from_equity(fill_price, equity); // qty_step_ lot-size flooring applies uniformly regardless of how the // caller's qty was derived — including this FIXED branch, which is the // common ``strategy.entry(qty=someComputedExpr)`` shape (e.g. a DCA base/ @@ -45,8 +148,6 @@ double BacktestEngine::calc_qty_for_type(double fill_price, double qty_value, in // tape pinned it (calc_qty carries the F / AAPL census). const double basis = round_to_mintick(fill_price); if (qty_type == static_cast(QtyType::PERCENT_OF_EQUITY)) { - const double equity = percent_commission_live_equity( - round_to_mintick(current_bar_.close)); if (!std::isfinite(equity)) return 0.0; double cash = reserve_percent_commission(equity * (qty_value / 100.0)); // Reject (qty 0) on a non-finite / non-positive fill price — a degenerate @@ -69,6 +170,24 @@ double BacktestEngine::calc_qty_for_type(double fill_price, double qty_value, in return apply_qty_step(qty_value); } +double BacktestEngine::source_reversal_qty( + double fill_price, double explicit_qty, int explicit_qty_type, + bool prequantized) const { + if (prequantized) return explicit_qty; + const bool needs_equity = std::isnan(explicit_qty) + ? default_qty_type_ == QtyType::PERCENT_OF_EQUITY + : explicit_qty_type == static_cast(QtyType::PERCENT_OF_EQUITY); + if (!needs_equity) + return calc_qty_for_type_from_equity(fill_price, explicit_qty, explicit_qty_type, 0.0); + const auto projection = project_native_settlement_v1( + execution::Flatten{}, execution::Fill{fill_price, {}, {}, 0}); + if (projection.status != execution::Status::Applied + && projection.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved reversal sizing projection"); + return calc_qty_for_type_from_equity( + fill_price, explicit_qty, explicit_qty_type, projection.realized_balance); +} + // Internal helper: execute a market entry (handles reverse-and-open). // @@ -102,8 +221,7 @@ void BacktestEngine::execute_market_entry(const std::string& id, bool is_long, d || (risk_direction_ == RiskDirection::SHORT_ONLY && is_long); if (is_opposite_entry && direction_blocked) { - double exit_fill = apply_slippage(fill_price, position_side_ == PositionSide::SHORT); - execute_market_exit(exit_fill); + execute_market_exit(fill_price); if (!paired_flat_market_transaction) purge_exit_orders(); return; } @@ -128,9 +246,10 @@ void BacktestEngine::execute_market_entry(const std::string& id, bool is_long, d } if (position_side_ == requested) { - add_to_pyramid_market(id, is_long, fill_price, explicit_qty, explicit_qty_type, - created_position_side, is_priced_entry, - entry_incarnation); + add_to_pyramid_market_with_qty_provenance( + id, is_long, fill_price, explicit_qty, explicit_qty_type, + created_position_side, is_priced_entry, explicit_qty_prequantized, + entry_incarnation); return; } @@ -147,9 +266,9 @@ void BacktestEngine::execute_market_entry(const std::string& id, bool is_long, d } if (later_same_tick_entry) { - sequential_same_tick_reversal_fill(id, is_long, fill_price, explicit_qty, - explicit_qty_type, - entry_incarnation); + sequential_same_tick_reversal_fill_with_qty_provenance( + id, is_long, fill_price, explicit_qty, explicit_qty_type, + explicit_qty_prequantized, entry_incarnation); return; } @@ -294,45 +413,34 @@ void BacktestEngine::record_range_end_close_trades() { // single from_entry id. See engine.hpp for the contract. Mirrors TradingView's // per-pyramid trade reporting: one Trade per drained slice. Returns total qty // drained so callers can assert / log if needed. +// Retained private ABI helper. Production close paths below use explicit +// source actions; this compatibility entry point also consumes the sole book. double BacktestEngine::fifo_drain(const std::string* from_entry, double qty_limit, double fill_price, bool was_long) { - double qty_closed = 0.0; - std::vector remaining; - for (auto& pe : pyramid_entries_) { - bool eligible = (from_entry == nullptr) || (pe.entry_id == *from_entry); - if (!eligible || qty_closed >= qty_limit - kQtyEpsilon) { - remaining.push_back(pe); - continue; - } - double close_qty = std::min(pe.qty, qty_limit - qty_closed); - double keep_qty = pe.qty - close_qty; - qty_closed += close_qty; - - emit_close_trade(pe, close_qty, fill_price, was_long); - - if (keep_qty > kQtyEpsilon) { - // Scale the accumulated USD excursion to the kept slice so the - // remaining entry's extremes stay consistent with its reduced - // qty (update_per_trade_extremes accumulates (diff) * pe.qty). - double keep_scale = keep_qty / pe.qty; - PyramidEntry kept = pe; - kept.qty = keep_qty; - kept.max_runup *= keep_scale; - kept.max_drawdown *= keep_scale; - // Realizing part of the lot consumes that part of its paid entry - // cost. A later rate/type/FX change cannot rewrite the payment. - kept.entry_commission_account = open_entry_commission(pe) - - allocated_entry_commission(pe, close_qty); - remaining.push_back(std::move(kept)); - } + (void)was_long; // physical orientation belongs to the authoritative book + const int pre_count = position_entry_count_; + execution::Result result; + if (from_entry) { + const auto incarnations = source_opening_membership(pyramid_entries_, + [&](const PyramidEntry& lot) { return lot.entry_id == *from_entry; }); + if (incarnations.empty()) return 0.0; + const execution::SelectedOpeningSet selection{position_cycle_seq_, incarnations}; + result = settle_execution_selected_with_lifecycle( + order_action::Reduce{qty_limit}, execution::Fill{fill_price, {}, {}, 0}, {}, selection); + } else { + result = settle_resolved_execution( + order_action::Reduce{qty_limit}, execution::Fill{fill_price, {}, {}, 0}); } - - pyramid_entries_ = std::move(remaining); - position_qty_ -= qty_closed; - return qty_closed; + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved compatibility drain settlement"); + // Old callers chose the later source slot policy themselves. Preserve + // that interface without retaining its former physical FIFO/fee loop. + if (result.status == execution::Status::Applied && position_side_ != PositionSide::FLAT) + position_entry_count_ = pre_count; + return result.closed_units; } - // Internal helper: execute a partial exit (reduce position by qty, create trade records) // TradingView creates individual trade records for each partial exit. void BacktestEngine::execute_partial_exit_qty( @@ -347,11 +455,21 @@ void BacktestEngine::execute_partial_exit_qty( bool is_buy = (position_side_ == PositionSide::SHORT); fill_price = apply_fill_slippage(fill_price, is_buy); - bool was_long = (position_side_ == PositionSide::LONG); - - // Close FIFO across all pyramid entries, creating trade records. - fifo_drain(/*from_entry=*/nullptr, qty_to_close, fill_price, was_long); - settle_position_after_partial_exit(cause); + const int pre_count = position_entry_count_; + execution::Action action = order_action::Reduce{qty_to_close}; + std::optional prefix; + if (std::abs(held) - qty_to_close <= kQtyEpsilon) action = execution::Flatten{}; + else prefix = source_fifo_prefix_membership( + pyramid_entries_, qty_to_close, position_cycle_seq_); + const execution::Fill fill{fill_price, {}, {}, 0}; + const auto result = prefix + ? settle_execution_selected_with_lifecycle(execution::Flatten{}, fill, {}, *prefix) + : settle_resolved_execution(action, fill); + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved partial-close settlement"); + if (result.status == execution::Status::Applied) + restore_source_partial_exit_slots(pre_count, cause); } @@ -360,6 +478,8 @@ void BacktestEngine::execute_partial_exit_qty( // seam. It neither consults a pyramiding cap nor invents a source-policy bit. // The caller supplies the lot's immutable label, identity and fill metadata; // accounting and stream observations still use the engine's sole lot ledger. +// Test-only compatibility construction seam; no production caller. Live +// source entries use settle_source_opening and the shared settlement owner. void BacktestEngine::append_same_side_fill(PyramidEntry lot) { snapshot_entry_commission(lot); const double total_qty = position_qty_ + lot.qty; @@ -408,22 +528,19 @@ void BacktestEngine::execute_partial_exit_by_entry(double fill_price, PositionReductionCause cause) { if (position_side_ == PositionSide::FLAT || pyramid_entries_.empty()) return; - bool is_buy = (position_side_ == PositionSide::SHORT); - fill_price = apply_fill_slippage(fill_price, is_buy); - bool was_long = (position_side_ == PositionSide::LONG); - - std::vector remaining; - for (auto& pe : pyramid_entries_) { - if (pe.entry_id == from_entry) { - emit_close_trade(pe, pe.qty, fill_price, was_long); - position_qty_ -= pe.qty; - } else { - remaining.push_back(pe); - } - } - - pyramid_entries_ = std::move(remaining); - settle_position_after_partial_exit(cause); + const auto incarnations = source_opening_membership(pyramid_entries_, + [&](const PyramidEntry& lot) { return lot.entry_id == from_entry; }); + if (incarnations.empty()) return; + const execution::SelectedOpeningSet selection{position_cycle_seq_, incarnations}; + fill_price = apply_fill_slippage(fill_price, position_side_ == PositionSide::SHORT); + const int pre_count = position_entry_count_; + const auto result = settle_execution_selected_with_lifecycle( + execution::Flatten{}, execution::Fill{fill_price, {}, {}, 0}, {}, selection); + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved bound-close settlement"); + if (result.status == execution::Status::Applied) + restore_source_partial_exit_slots(pre_count, cause); } @@ -437,12 +554,28 @@ void BacktestEngine::execute_partial_exit_by_entry_qty( if (position_side_ == PositionSide::FLAT || pyramid_entries_.empty()) return; if (!std::isfinite(qty_to_close) || qty_to_close <= kQtyEpsilon) return; - bool is_buy = (position_side_ == PositionSide::SHORT); - fill_price = apply_fill_slippage(fill_price, is_buy); - bool was_long = (position_side_ == PositionSide::LONG); - - fifo_drain(&from_entry, qty_to_close, fill_price, was_long); - settle_position_after_partial_exit(cause); + const auto incarnations = source_opening_membership(pyramid_entries_, + [&](const PyramidEntry& lot) { return lot.entry_id == from_entry; }); + if (incarnations.empty()) return; + double selected_qty = 0.0; + bool has_unselected_lots = false; + for (const auto& lot : pyramid_entries_) { + if (lot.entry_id == from_entry) selected_qty += lot.qty; + else has_unselected_lots = true; + } + const execution::SelectedOpeningSet selection{position_cycle_seq_, incarnations}; + execution::Action action = order_action::Reduce{qty_to_close}; + if (!has_unselected_lots && selected_qty - qty_to_close <= kQtyEpsilon) + action = execution::Flatten{}; + fill_price = apply_fill_slippage(fill_price, position_side_ == PositionSide::SHORT); + const int pre_count = position_entry_count_; + const auto result = settle_execution_selected_with_lifecycle( + action, execution::Fill{fill_price, {}, {}, 0}, {}, selection); + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved bound-close settlement"); + if (result.status == execution::Status::Applied) + restore_source_partial_exit_slots(pre_count, cause); } @@ -492,28 +625,26 @@ double BacktestEngine::cover_samebar_market_adds_on_exit(const PendingOrder& ord || !std::isnan(order.legs.prices().trail_price); if (!priced_bracket) return 0.0; - bool is_buy = (position_side_ == PositionSide::SHORT); - double slipped = apply_fill_slippage(fill_price, is_buy); - bool was_long = (position_side_ == PositionSide::LONG); - - std::vector remaining; - remaining.reserve(pyramid_entries_.size()); - double closed = 0.0; - for (auto& pe : pyramid_entries_) { - if (pe.market_pyramid_add - && pe.entry_bar_index == bar_index_ - && pe.entry_id == order.from_entry) { - emit_close_trade(pe, pe.qty, slipped, was_long); - closed += pe.qty; - } else { - remaining.push_back(pe); - } - } - if (closed <= kQtyEpsilon) return 0.0; // nothing covered - pyramid_entries_ = std::move(remaining); - position_qty_ -= closed; - settle_position_after_partial_exit(cause); - return closed; + const auto incarnations = source_opening_membership(pyramid_entries_, + [&](const PyramidEntry& lot) { + return lot.market_pyramid_add && lot.entry_bar_index == bar_index_ + && lot.entry_id == order.from_entry; + }); + if (incarnations.empty()) return 0.0; + const execution::SelectedOpeningSet selection{position_cycle_seq_, incarnations}; + const double slipped = apply_fill_slippage(fill_price, position_side_ == PositionSide::SHORT); + // This is a second Fill after primary restoration and R20, so capture its + // own current source slot count rather than reusing the primary snapshot. + const int pre_count = position_entry_count_; + const auto result = settle_execution_selected_with_lifecycle( + execution::Flatten{}, execution::Fill{slipped, order.id, order.comment, order.incarnation}, + {}, selection); + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved bound-close settlement"); + if (result.status == execution::Status::Applied) + restore_source_partial_exit_slots(pre_count, cause); + return result.closed_units; } @@ -851,6 +982,15 @@ void BacktestEngine::reset_position_state_to_flat() { consumed_partial_exit_ids_.clear(); } +void BacktestEngine::restore_source_partial_exit_slots( + int pre_count, PositionReductionCause cause) { + // Settlement owns quantities, average price, cycles, and physical dust. + // This adapter step restores only the source's occupied-slot policy. + if (position_side_ == PositionSide::FLAT || pyramid_entries_.empty()) return; + if (cause == PositionReductionCause::BRACKET_EXIT) { + position_entry_count_ = std::max(pre_count, static_cast(pyramid_entries_.size())); + } +} // After a partial exit potentially empties pyramid_entries_, either reset // position state to FLAT (no entries left or qty effectively zero) or @@ -959,10 +1099,32 @@ void BacktestEngine::unbind_exit_activations() { void BacktestEngine::open_fresh_position(PositionSide requested, double fill_price, double qty, const std::string& id, uint64_t entry_incarnation) { - PyramidEntry lot{fill_price, current_bar_.timestamp, qty, id, bar_index_}; - lot.entry_incarnation = entry_incarnation; - snapshot_entry_commission(lot); - open_quoted_position(requested, std::move(lot)); + if (position_side_ != PositionSide::FLAT) + throw std::runtime_error("invalid resolved fresh opening: position not flat"); + settle_source_opening(requested, fill_price, qty, id, {}, entry_incarnation); +} + +execution::Result BacktestEngine::settle_source_opening( + PositionSide requested, double fill_price, double qty, + const std::string& id, const std::string& comment, uint64_t incarnation) { + if ((requested != PositionSide::LONG && requested != PositionSide::SHORT) + || (position_side_ != PositionSide::FLAT && position_side_ != requested) + || !std::isfinite(qty) || qty < 0.0) + throw std::runtime_error("invalid resolved source opening"); + const std::size_t before_lots = pyramid_entries_.size(); + const auto result = settle_resolved_execution( + order_action::Transact{requested == PositionSide::LONG ? qty : -qty}, + execution::Fill{fill_price, id, comment, incarnation}); + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved source opening settlement"); + if (result.status == execution::Status::Applied + && (result.closed_units != 0.0 || result.opened_units == 0.0 + || pyramid_entries_.size() != before_lots + 1 + || pyramid_entries_.back().entry_incarnation != incarnation + || pyramid_entries_.back().qty != std::abs(result.opened_units))) + throw std::runtime_error("invalid resolved source opening lot provenance"); + return result; } void BacktestEngine::open_quoted_position(PositionSide requested, PyramidEntry lot) { @@ -1139,6 +1301,16 @@ void BacktestEngine::add_to_pyramid_market(const std::string& id, bool is_long, PositionSide created_position_side, bool is_priced_entry, uint64_t entry_incarnation) { + add_to_pyramid_market_with_qty_provenance( + id, is_long, fill_price, explicit_qty, explicit_qty_type, + created_position_side, is_priced_entry, false, entry_incarnation); +} + +void BacktestEngine::add_to_pyramid_market_with_qty_provenance( + const std::string& id, bool is_long, double fill_price, double explicit_qty, + int explicit_qty_type, PositionSide created_position_side, + bool is_priced_entry, bool explicit_qty_prequantized, + uint64_t entry_incarnation) { PositionSide requested = is_long ? PositionSide::LONG : PositionSide::SHORT; bool flat_armed_priced = is_priced_entry && created_position_side == PositionSide::FLAT; @@ -1150,26 +1322,19 @@ void BacktestEngine::add_to_pyramid_market(const std::string& id, bool is_long, && position_entry_count_ >= pyramiding_) { return; } - double new_qty = calc_qty_for_type(fill_price, explicit_qty, explicit_qty_type); + const double new_qty = explicit_qty_prequantized + ? explicit_qty : calc_qty_for_type(fill_price, explicit_qty, explicit_qty_type); // Zero-lot add safety net. The fill kernel (apply_filled_order_to_state's // zero-lot decline) consumes such an order before it reaches here; should // any path bypass that gate, never materialize a qty-0 pyramid lot nor // spend a pyramiding slot on it — TV does not place the order at all. if (!(new_qty > kQtyEpsilon)) return; - double total_qty = position_qty_ + new_qty; - position_entry_price_ = (position_entry_price_ * position_qty_ + fill_price * new_qty) / total_qty; - position_qty_ = total_qty; - position_entry_count_++; - trail_best_price_ = fill_price; - pyramid_entries_.push_back({fill_price, current_bar_.timestamp, new_qty, id, bar_index_}); - pyramid_entries_.back().entry_incarnation = entry_incarnation; - snapshot_entry_commission(pyramid_entries_.back()); - if (stream_observe_actions_) stream_observe_entry(pyramid_entries_.back()); + const auto result = settle_source_opening( + requested, fill_price, new_qty, id, {}, entry_incarnation); // KI-62: only a same-direction MARKET add is scratched by a same-bar // from_entry bracket exit; a priced pyramid add is not this collision. - pyramid_entries_.back().market_pyramid_add = !is_priced_entry; - id_unclosed_qty_[id] += new_qty; - cycle_filled_entry_ids_.insert(id); + if (result.status == execution::Status::Applied && result.opened_units != 0.0) + pyramid_entries_.back().market_pyramid_add = !is_priced_entry; } @@ -1268,56 +1433,23 @@ void BacktestEngine::flip_market_position_to(const std::string& id, bool is_long bool explicit_qty_prequantized, bool close_only, uint64_t entry_incarnation) { - // For the close we need exit slippage based on closing direction. - // Closing a long = sell (price - slip); closing a short = buy (price + slip). - // fill_price already has entry slippage applied; un-slip it before - // re-applying with the exit direction. - double raw_price = fill_price; - if (slippage_ != 0 && !current_fill_is_limit_) { - // LIMIT-triggered entry fills were never slipped (limit-or-better - // path), so there is no entry slip to back out for the close leg. - double slip = slippage_ * syminfo_mintick_; - raw_price = is_long ? (fill_price - slip) : (fill_price + slip); - } - // Flag-aware close leg: a limit-triggered flip's close leg follows the - // entry leg's limit semantics (unslipped, limit-or-better) for internal - // consistency with the sibling close_opposite_then_enter path — both - // legs land at the identical unslipped snapped price. No direct TV - // evidence yet (needs a limit-flip slippage>0 export) — corpus provably - // indifferent at slippage=0. - double exit_fill = apply_fill_slippage(raw_price, position_side_ == PositionSide::SHORT); - bool was_long = (position_side_ == PositionSide::LONG); - - // Emit one Trade per pyramid entry (matches TradingView reporting) - for (auto& pe : pyramid_entries_) { - emit_close_trade(pe, pe.qty, exit_fill, was_long); - } - - // The old lots are fully realized above. Clear their live position/PnL - // and entry-fee snapshots before sizing the incoming leg; otherwise a - // percent-typed reversal adds stale open PnL and debits the already- - // realized entry commission a second time. - reset_position_state_to_flat(); - - if (close_only) { - // Priced-entry reduce-only cases: either this order was armed during a - // prior cycle and flips a later opposite position, or its same-cycle - // frozen transaction exactly equals the grown live opposite position. - // In both cases close the whole position and stay flat; do NOT open. - // See apply_entry_order_fill's close_only_opposite predicates. - return; + // The incoming direction is also the closing direction. The caller has + // already resolved this one price; no un-slip/re-slip or second ticket. + const execution::Fill fill{fill_price, id, {}, entry_incarnation}; + double incoming = 0.0; + if (!close_only) { + incoming = source_reversal_qty( + fill_price, explicit_qty, explicit_qty_type, explicit_qty_prequantized); + if (!std::isfinite(incoming) || incoming < 0.0) + throw std::runtime_error("invalid resolved flip quantity"); } - - // Default-sized MARKET quantities are frozen and exchange-quantized at - // signal time. Every sibling dispatch path preserves that provenance; - // ordinary flips must not feed the frozen contracts through qty_step a - // second time (binary64 can turn 1.3410 into 1.3409 on the second floor). - // Explicit/FIXED quantities keep their normal single fill-side floor. - double new_qty = explicit_qty_prequantized - ? explicit_qty - : calc_qty_for_type(fill_price, explicit_qty, explicit_qty_type); - PositionSide requested = is_long ? PositionSide::LONG : PositionSide::SHORT; - open_fresh_position(requested, fill_price, new_qty, id, entry_incarnation); + const auto result = incoming > 0.0 + ? settle_reversal_with_lifecycle_v1( + execution::ReverseTo{is_long ? incoming : -incoming}, fill, {}) + : settle_resolved_execution(execution::Flatten{}, fill); + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved flip settlement"); } @@ -1369,39 +1501,29 @@ void BacktestEngine::sequential_same_tick_reversal_fill(const std::string& id, double explicit_qty, int explicit_qty_type, uint64_t entry_incarnation) { - // fill_price arrives entry-slipped (execute_market_entry applied entry - // slippage). The close leg needs EXIT slippage for the closing - // direction — un-slip, then re-apply, mirroring flip_market_position_to. - double raw_price = fill_price; - if (slippage_ != 0 && !current_fill_is_limit_) { - double slip = slippage_ * syminfo_mintick_; - raw_price = is_long ? (fill_price - slip) : (fill_price + slip); - } - double exit_fill = apply_fill_slippage(raw_price, position_side_ == PositionSide::SHORT); - bool was_long = (position_side_ == PositionSide::LONG); - double old_qty = position_qty_; - - // Close the ENTIRE opposite position — one row per pyramid lot, all - // tagged with THIS order's id by apply_filled_order_to_state (matches - // TV's single-row exit attribution to the first closing signal). - for (auto& pe : pyramid_entries_) { - emit_close_trade(pe, pe.qty, exit_fill, was_long); - } - reset_position_state_to_flat(); - - // Plain (non-augmented) sizing, computed after the close so - // percent-of-equity sees the realized PnL — the same equity basis - // flip_market_position_to uses. Only the portion that crosses zero - // opens a position; if the plain qty doesn't reach past the old - // position, the LAST same-tick entry (filling next from flat) owns - // the new position instead. - double tx_qty = calc_qty_for_type(fill_price, explicit_qty, explicit_qty_type); - double remainder = tx_qty - old_qty; - if (remainder > kQtyEpsilon) { - PositionSide requested = is_long ? PositionSide::LONG : PositionSide::SHORT; - open_fresh_position( - requested, fill_price, remainder, id, entry_incarnation); - } + sequential_same_tick_reversal_fill_with_qty_provenance( + id, is_long, fill_price, explicit_qty, explicit_qty_type, false, entry_incarnation); +} + +void BacktestEngine::sequential_same_tick_reversal_fill_with_qty_provenance( + const std::string& id, bool is_long, double fill_price, double explicit_qty, + int explicit_qty_type, bool explicit_qty_prequantized, uint64_t entry_incarnation) { + double held = 0.0; + for (const auto& lot : pyramid_entries_) held += lot.qty; + const double transaction = source_reversal_qty( + fill_price, explicit_qty, explicit_qty_type, explicit_qty_prequantized); + if (!std::isfinite(transaction) || transaction < 0.0) + throw std::runtime_error("invalid resolved sequential quantity"); + execution::Action action = execution::Flatten{}; + if (transaction - held > kQtyEpsilon) + action = order_action::Transact{is_long ? transaction : -transaction}; + // Class B closes the entire old book here; its later sibling remains a + // separate Fill. Empty lifecycle preserves the active pending iteration. + const auto result = settle_resolved_execution( + action, execution::Fill{fill_price, id, {}, entry_incarnation}); + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved sequential settlement"); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 632e43be3..5c18c009f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,7 +2,13 @@ set(TEST_SOURCES test_unbounded_margin_admission test_resolved_execution test_native_scoped_close + test_native_selected_settlement + test_native_settlement_projection + test_native_settlement_callers test_native_reversal_contract + test_native_reverse_to + test_source_exact_reversal + test_source_fifo_endpoints test_order_action test_native_order_core test_native_order_terminal @@ -295,6 +301,11 @@ set(TEST_SOURCES find_package(Threads REQUIRED) find_package(Python3 COMPONENTS Interpreter REQUIRED) +add_test(NAME test_cmake_version_source + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/test_cmake_version_source.py) +add_test(NAME test_ci_verify + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/test_ci_verify.py) + add_test( NAME test_aggregate_cpp_versions COMMAND ${Python3_EXECUTABLE} @@ -339,6 +350,28 @@ add_test( ) set_tests_properties(test_native_cpp_abi PROPERTIES TIMEOUT 240) +# The actual historical archive is prepared explicitly before CTest so the +# compatibility check itself stays offline and cannot silently use a stub. +set(PINEFORGE_SETTLEMENT_ABI_BASE_RECEIPT + "${PROJECT_BINARY_DIR}/settlement-abi-base/receipt.json" + CACHE FILEPATH "Prepared actual R2 ABI provider receipt") +set(PINEFORGE_SETTLEMENT_ABI_PRIOR_RECEIPT + "${PROJECT_BINARY_DIR}/settlement-abi-prior/receipt.json" + CACHE FILEPATH "Prepared actual 0e18690 ABI provider receipt") +add_test(NAME test_settlement_cpp_abi + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/check_settlement_cpp_abi.py + --compiler ${CMAKE_CXX_COMPILER} + --library $ + --include ${PROJECT_SOURCE_DIR}/include + --generated-include ${PROJECT_BINARY_DIR}/include + --base-receipt ${PINEFORGE_SETTLEMENT_ABI_BASE_RECEIPT} + --prior-receipt ${PINEFORGE_SETTLEMENT_ABI_PRIOR_RECEIPT} + --receipt ${PROJECT_BINARY_DIR}/settlement-abi-receipt.json + ${_pf_script_cpp_abi_flags}) +set_tests_properties(test_settlement_cpp_abi PROPERTIES TIMEOUT 600) +add_test(NAME test_settlement_cpp_abi_tooling + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/test_settlement_cpp_abi.py) + add_test( NAME test_derive_corpus_feeds COMMAND ${Python3_EXECUTABLE} diff --git a/tests/fixtures/settlement_cpp_abi/0e18690/manifest.json b/tests/fixtures/settlement_cpp_abi/0e18690/manifest.json new file mode 100644 index 000000000..b38140744 --- /dev/null +++ b/tests/fixtures/settlement_cpp_abi/0e18690/manifest.json @@ -0,0 +1,277 @@ +{ + "commit": "0e18690db3fb6bb4705841c2a86936dfa206380a", + "files": { + "include/pineforge/bar.hpp": { + "bytes": 655, + "git_blob": "836f73daab382e9c8d5a6d03a942a921e39b756f", + "sha256": "665c2528dca64ac9474de622dfd7ffdb540ff91316f70317e39aada158d1e0de" + }, + "include/pineforge/broker_events.hpp": { + "bytes": 4343, + "git_blob": "2972d53c7b039af266f4da161301c01eec54c16d", + "sha256": "4f3d9948a4d1604750901fa3982d6ea49bd50dd15a86c4e5d690b83d1f5f23c0" + }, + "include/pineforge/color.hpp": { + "bytes": 1253, + "git_blob": "a4d54f37a1c9d8d10ff5423aca4d42814a666446", + "sha256": "29eb9d4473e4dd488f959fb9f53999656e3feb6aa25255287e80855ecabe96d6" + }, + "include/pineforge/compat/pine/exit_activation.hpp": { + "bytes": 3017, + "git_blob": "a62927dd936e839160671cee665de908cb7f5dc3", + "sha256": "9190382359099021870d260718ad593c567024603b016294d1b113c278ddd73d" + }, + "include/pineforge/compat/pine/exit_lifecycle.hpp": { + "bytes": 882, + "git_blob": "c78f64b82adaf18e013e017a633b9b9b6b48f22d", + "sha256": "2fcfce8c914424404767bdd7b2fc6ea30decbc3f5f5a6123cac9080e2b9bd8d2" + }, + "include/pineforge/compat/pine/frozen_market_instruction.hpp": { + "bytes": 3201, + "git_blob": "8890ad48c5e4485b9d5a65a5bdd7f7f876e39c24", + "sha256": "e67a4b470e82086a089d3bf0b8996f88ab0fd5d86046682d6cb19b2ed64922c8" + }, + "include/pineforge/compat/pine/intraday_cap.hpp": { + "bytes": 10469, + "git_blob": "0aba76b6a78251df33e92f19290473adecf5a61e", + "sha256": "0ac0eab34625ab9a974037ff8c0217603a01deb81599350a4b24f7bf48ac0cbd" + }, + "include/pineforge/compat/pine/intraday_order_budget.hpp": { + "bytes": 3878, + "git_blob": "730d941fffa3939e375489f40a50901ccea9ec87", + "sha256": "43f7700f9c66c630ab8a9d8fd899eecb03c2fd9aabff11d2dc2b7fa85f92981b" + }, + "include/pineforge/compat/pine/market_admission.hpp": { + "bytes": 1057, + "git_blob": "33cf82e5e53ae870c3665ede69d7cde2e9695752", + "sha256": "83ce322f8864a569e197493cd2bcca23aa226997d3508caaa6a783ff1bb97b03" + }, + "include/pineforge/compat/pine/order_birth.hpp": { + "bytes": 841, + "git_blob": "aa48c054f613b65ee5ded14fde41643f807347bd", + "sha256": "e819f5c4202911e5e592f4472925a15c07f7300fabcfecf4f214348e5ee605f8" + }, + "include/pineforge/compat/pine/order_priority.hpp": { + "bytes": 1577, + "git_blob": "86135165fd3d83fc4638a2295dda769024836517", + "sha256": "e88ffcf28bf3fac95bc439b3d99a3944fc9c36ce0d3892fd365d25f9f3f920e4" + }, + "include/pineforge/compat/pine/reservation_expansion.hpp": { + "bytes": 759, + "git_blob": "1115a6593f5fabadf722b14305a213b07afb6c85", + "sha256": "7ad4195af574df40a600b748399f00c41cafa9441bfdcb5b90898ec8a4b577dd" + }, + "include/pineforge/drawing.hpp": { + "bytes": 17150, + "git_blob": "d04f12bb214b82c5d0b174824c25ff9f148ab090", + "sha256": "1635cfc9e65a91369cc1491a64dc564944b6828333af62e3ecd80edb5ba74183" + }, + "include/pineforge/engine.hpp": { + "bytes": 312399, + "git_blob": "68cc254148529095b909b98a0764b3bbbbbd23ab", + "sha256": "7b9b0e5f4152e688ae80574cc99c770e26940e8885bbb19787d5b085d2fe21bd" + }, + "include/pineforge/execution.hpp": { + "bytes": 4294, + "git_blob": "f270b7c64544ee9188c22752d5e41eaa78bf01d1", + "sha256": "e6e2440118f38a0eba5b75a0fb0cf03675f92939c154aaf8dc0837653a113895" + }, + "include/pineforge/execution_close_scope.hpp": { + "bytes": 714, + "git_blob": "d4633524495566d68e7fba8e3244fa32f8bcc01e", + "sha256": "b319a176f7d3d0c668f2a1b5c607e3b2077088221e9196a1d490aa6cd548f0b6" + }, + "include/pineforge/execution_close_selection.hpp": { + "bytes": 517, + "git_blob": "1a6e9e89444e5730d923356b290beddb121df2d8", + "sha256": "efb4c90fd058f71ef896e32c9e08db4400b79e7d138c1c5b5f6eda3275646c6e" + }, + "include/pineforge/execution_consumer.hpp": { + "bytes": 3010, + "git_blob": "19e99314462c9c3729953b0f343abd4aa0bf1496", + "sha256": "ffd09417d1f1e8aab32794673d6802616c5055b7976101e49416c283b64856e0" + }, + "include/pineforge/execution_projection.hpp": { + "bytes": 1835, + "git_blob": "68abfa45dfc387243b66cb72a270b191ed7c894b", + "sha256": "464f5f4a98cc83d78cbf769c2cc2c7e80e6934cf72ff24e0dd0496f4790d85f1" + }, + "include/pineforge/exit_leg_lifecycle.hpp": { + "bytes": 22622, + "git_blob": "91492ec22e05d7b6d9327f60ff5e239ae6906fcd", + "sha256": "9d785271511a350114adc7cf5bcb1600cf569f0a10559a8671b7e4b9cb9dc89a" + }, + "include/pineforge/generic_matrix.hpp": { + "bytes": 25438, + "git_blob": "8694e5577d56221498cca844aa14cddb9a15e84f", + "sha256": "9c94a877b6f9e5965604b03f572ceea2ea3680de586d69ef0627c78208624150" + }, + "include/pineforge/leg_activation.hpp": { + "bytes": 1353, + "git_blob": "4160c841782800853bb574219b3dad2ca8aa0cc6", + "sha256": "859d75d0ae90ec3cc109765da066b37c3bbbff9c7c840368fbdd05dc7e178f3f" + }, + "include/pineforge/live_parser.h": { + "bytes": 3870, + "git_blob": "0d00afe0586477c69dea35a1bfef72f3acb491a3", + "sha256": "f1308086a66fe6a0636b8b2fc44c3fcb6b0c21ed0311654297ecc8ea5e959e1e" + }, + "include/pineforge/log.hpp": { + "bytes": 496, + "git_blob": "6cc4c35ad1a902f05022fb11357dff86281b5301", + "sha256": "df5db36e01ddaa66c44c1b378e6b0c3b05e0c732c59fbcb5401e3606585d25a3" + }, + "include/pineforge/magnifier.hpp": { + "bytes": 3241, + "git_blob": "1bbc910a869d162900d7610fa0bd9a88776b7b0f", + "sha256": "c0ed220b6df054b44596a434d4f48b1c3e960ca329bb8644a405a608175e06ff" + }, + "include/pineforge/map.hpp": { + "bytes": 14476, + "git_blob": "5dd59cbaad9b93138738d3f92b8c457c0ac92a7d", + "sha256": "772a4f6c79ea01af6ad057e361e821d79a7d9884e4aff8bac5a2f77565e55b9f" + }, + "include/pineforge/market_admission.hpp": { + "bytes": 10221, + "git_blob": "70be6af6b556bf77cab6c9f823bb3492cf7c8294", + "sha256": "22914434a2c82f761b98c1b5a2ff9365c89bb802b584cdda2a3c30c5986ec6fe" + }, + "include/pineforge/market_driver.hpp": { + "bytes": 5374, + "git_blob": "3e6c68fb491156520640b1095dfce057395375a3", + "sha256": "f89b8b49127dff19f88199da50db3defbcd891c2a25cc777f595d7f277a7cd3c" + }, + "include/pineforge/math.hpp": { + "bytes": 1316, + "git_blob": "9ec09d539dd0ba732449424e6a8c80c3f5454888", + "sha256": "be95fa75d1e8d288ed49d62fa56257efe99435be7ba357fbb93cc36c0dc9eda3" + }, + "include/pineforge/matrix.hpp": { + "bytes": 4868, + "git_blob": "22b29448fb60dded446543b1940c1a1466986133", + "sha256": "f463ab76db92436fd818839c390e69462a794e07413d99fb49af4ed1b67bca6f" + }, + "include/pineforge/metrics.hpp": { + "bytes": 1353, + "git_blob": "f468f485bbdd0512536e96390d2b411228c2f517", + "sha256": "9e35101f16c2717dbfde428ecc515802dd758056e080fd45ee6411e9d0ead5be" + }, + "include/pineforge/na.hpp": { + "bytes": 1064, + "git_blob": "e98505aadc68a09cbe5a14aacf591a799a15ecd2", + "sha256": "0e8c256fcdeda94a0855458fa3b8c4ad4f9864dc2aecce969934f6bea82fd0a5" + }, + "include/pineforge/native_calendar.hpp": { + "bytes": 19589, + "git_blob": "742ecfabc3764d626617c2762d4b574d46d8ff46", + "sha256": "5495cb007509937616255a9615dcc92c49e5115692c8375ef93b594a36e55860" + }, + "include/pineforge/native_host.hpp": { + "bytes": 10304, + "git_blob": "58573006ad056ecaa773dc79d22ca1375d51fac8", + "sha256": "9b7236e5946d1e5d2355cd6a72a02d987ab6b215acf2155ebce89771c8015808" + }, + "include/pineforge/native_order.hpp": { + "bytes": 36944, + "git_blob": "238475d372cc87b91d163c6c1a378f10751d444e", + "sha256": "3881f6b1ebecb95801e5ea6f4acb5a5a8203d94322ffdc36663d69b4966b7c08" + }, + "include/pineforge/native_order_identity.hpp": { + "bytes": 1948, + "git_blob": "c0a6835c8fdb5c3b331ad519439af12946e34cf2", + "sha256": "6f6d63c1a1f54499f04a137d0b13ea343aaf94ffe090058a62c854311345ac69" + }, + "include/pineforge/native_run_spec.hpp": { + "bytes": 5518, + "git_blob": "1b5ed883a71ed772e9f115415ab5c7acb1f920f0", + "sha256": "e7be6d930c4eed13d9fa2358f529c98e5ad37b75ad72013f1ba65513cabb8da1" + }, + "include/pineforge/order_action.hpp": { + "bytes": 3470, + "git_blob": "d992472cbefefdf8e29704a21f04614d68d39a58", + "sha256": "7d76ddc3f6ba76216d5feb34cdf28908e33a92a56757a707ac6bd94dca315001" + }, + "include/pineforge/order_birth.hpp": { + "bytes": 4833, + "git_blob": "ed4a4038e9d079c92b4ebd777a5f22f1f1256b0c", + "sha256": "0eadb3e2a92248fbc8e6ba8f94285b15b0e83d7a0396f5904b40c216619604e4" + }, + "include/pineforge/order_cancellation.hpp": { + "bytes": 7172, + "git_blob": "a4f11409f19c7a85347879ecfd942bd7bae5c2eb", + "sha256": "389eb3659f121351ad4b53b936d3a2c79f343ecdb6c0c4bb0bc13777596b90ee" + }, + "include/pineforge/order_priority.hpp": { + "bytes": 694, + "git_blob": "3e9fa5ee4121971a7962db8a60859aba6fb9ec95", + "sha256": "25bb7898972153b044675245db230c1171644856ef904f4dd02353b8968d433b" + }, + "include/pineforge/pending_order_mirror.hpp": { + "bytes": 21978, + "git_blob": "fc08bfd7328718de1c658bdcea1826c2fbc9eebe", + "sha256": "7921179f690dbb6f00ee8ab6ca71b3174fcc42a61ebc6786c3727c7f85a576a4" + }, + "include/pineforge/pine_float_compare.hpp": { + "bytes": 2893, + "git_blob": "a3de0e9a2ef3517098be500880fda71bdcfe2260", + "sha256": "3d389142b8e4fbd899c6923c11a27447920a1c9e9db0338ff9abc8a9f4a06d1c" + }, + "include/pineforge/pineforge.h": { + "bytes": 74362, + "git_blob": "31db9f0b9f8ac5f33041917f7d599daff91013c8", + "sha256": "e4c735a1e39411db4156ac13a50cee10c554039b20f33e3c2299cd73be6a6c3e" + }, + "include/pineforge/position_close_obligation.hpp": { + "bytes": 1060, + "git_blob": "09576ee0f2a1d49becc46f5701f4369cf8f87bf6", + "sha256": "6f6ad0aaca952dda7a9386cb78e57705b3494364dbd8970b0a051fe99d05b777" + }, + "include/pineforge/quantity_intent.hpp": { + "bytes": 3054, + "git_blob": "41262fee5aa2db4032fc008542c573032563629b", + "sha256": "c4cc0f1cd5d6ed1f248825712a5ba4b4f5a808b787c05c58971934b89d770b90" + }, + "include/pineforge/reservation_expansion.hpp": { + "bytes": 2434, + "git_blob": "b4302c9eed97c3409d9911fd6cecdc64fa0aad86", + "sha256": "27e514249f62aa7353ac084a6e59e25e338a867fe4d67ce4646446a1cf9f9dab" + }, + "include/pineforge/series.hpp": { + "bytes": 3439, + "git_blob": "61baaef2c351d37b0d6a5eb179e91ce670eb83c3", + "sha256": "e7709f67df9046c9eba163c2dbe91f8161084a955265ccf6b163355856c0afde" + }, + "include/pineforge/session_time.hpp": { + "bytes": 10337, + "git_blob": "af3a4472d852e0965947c006d0b6ad85304539e7", + "sha256": "f58c94bfae948e08fa21e52f20eae3940b31e398d83b800baf3a31f3a5fde669" + }, + "include/pineforge/str_utils.hpp": { + "bytes": 820, + "git_blob": "d44059f10b41872ea57a1abd9c246fadb3b7b3eb", + "sha256": "58e052a040da94f3824ec36db05b37dfe9e2219a32e843b56998fb1f0efe1393" + }, + "include/pineforge/ta.hpp": { + "bytes": 35301, + "git_blob": "3baeef801f44b938c16137fca5881784646760f9", + "sha256": "7221324be971ef6de425253656327395d8a1ffd081b3c1eef0c8c5b96f51e6b6" + }, + "include/pineforge/timeframe.hpp": { + "bytes": 29534, + "git_blob": "3d1bc9d495ac7e5b39ea5bae57c9d9afe580192b", + "sha256": "5e4c312da463fdcff55e0887188523c8bbf85a2ae62f3317a2b23f22fe0161e2" + }, + "include/pineforge/version.h.in": { + "bytes": 893, + "git_blob": "27118bd81741597e187e3a49d04ac201f9508011", + "sha256": "9b9e81d5e0998441a82dbe15a4495182fdfe126baea1cf26bdaaf51e491ad99f" + }, + "include/pineforge/window_sum.hpp": { + "bytes": 2828, + "git_blob": "1ea86860f6cb901fe79ff89ef46f1c77ce7e2b8c", + "sha256": "b5519f95d57d46e2a8045d380c1e1e1427ce744f4d53ee3a04ebfdaf942d3f68" + } + }, + "schemaVersion": "pineforge-settlement-abi-base-headers/v1", + "tree": "c29545d262bf6dcc22e46aa3382422f2e9c68378" +} diff --git a/tests/fixtures/settlement_cpp_abi/README.md b/tests/fixtures/settlement_cpp_abi/README.md new file mode 100644 index 000000000..f9352c416 --- /dev/null +++ b/tests/fixtures/settlement_cpp_abi/README.md @@ -0,0 +1,118 @@ +# Settlement C++ ABI proof + +`e60e571/manifest.json` pins the complete 52-file public header closure of +R2 commit `e60e57156a54bb1c74c0c0a4f5a338fd924c046d`, tree +`e209c89035ec884774f8f3e68e1dc118f26aaf51`. Each entry records bytes, SHA-256 +and Git blob identity. Binary libraries are **not** checked into this fixture. + +`0e18690/manifest.json` similarly pins the complete 54-file header closure of +R3 commit `0e18690db3fb6bb4705841c2a86936dfa206380a`, tree +`c29545d262bf6dcc22e46aa3382422f2e9c68378`. This provider has selected settlement +and account projection, and predates the four `ReverseTo` methods. Both real +historical providers remain required; the selected-method rejection uses R2. + +The new settlement checker supplements `check_native_cpp_abi.py` and +`check_script_cpp_abi.py`; retain those existing checks and their old epoch +and UBSan RTTI controls. No generated pairing executable is run. + +## Prepare the real historical providers explicitly + +The shared `ci_verify.py` profiles use `PINEFORGE_VERSION_SOURCE=FILE`, so +verification version identity is independent of tags or checkout depth. They +fetch each pinned old commit without tags only when it is missing and reuse +matching prepared archives on subsequent runs. For manual preparation, configure +the current build first, then fetch each old object if needed, outside CTest: + +```sh +git fetch --no-tags --depth=1 origin e60e57156a54bb1c74c0c0a4f5a338fd924c046d +python3 scripts/prepare_settlement_cpp_abi_base.py \ + --source-repo . --current-build build \ + --output build/settlement-abi-base --jobs 4 +git fetch --no-tags --depth=1 origin 0e18690db3fb6bb4705841c2a86936dfa206380a +python3 scripts/prepare_settlement_cpp_abi_base.py \ + --source-repo . --current-build build \ + --output build/settlement-abi-prior --jobs 4 \ + --commit 0e18690db3fb6bb4705841c2a86936dfa206380a \ + --tree c29545d262bf6dcc22e46aa3382422f2e9c68378 \ + --header-manifest tests/fixtures/settlement_cpp_abi/0e18690/manifest.json +``` + +For the sanitizer lane, replace the build paths with `build-asan`; for the +native-runner lane use `build-live`. The preparation script copies the actual +current CMake compiler/build-type/flags/sanitizer settings, Eigen package or +already populated FetchContent source, and platform options. It builds each full pinned static library +with tests, tutorial, runner and corpus disabled. Source comes only from +immutable local Git objects. Dependencies must already be available in the current build; its downloaded +Eigen source is reused when needed. FetchContent stays disconnected. The script itself performs no Git/dependency +network fetch and runs no test or strategy binary. + +Every preparation output must be new. Keep an existing receipt only for the +same compiler/configuration; the checker rejects mismatches. The receipt +binds archive/header bytes, actual compiler implementation/version/target, +configuration, compilation database and source identity. The version header +is generated from the archived base's VERSION fallback; it is retained and +hashed, not mislabeled as a live Git checkout descriptor. + +## Check the integrated current archive + +```sh +python3 scripts/check_settlement_cpp_abi.py \ + --compiler /path/to/current/c++ --library build/lib/libpineforge.a \ + --include include --generated-include build/include \ + --base-receipt build/settlement-abi-base/receipt.json \ + --prior-receipt build/settlement-abi-prior/receipt.json \ + --receipt build/settlement-abi-receipt.json +``` + +Pass `--extra-flag=-fsanitize=address,undefined` for an instrumented archive, +as the existing CMake ABI guards do. The compiler must match the current +build and both prepared providers. A preserved Mac Release archive cannot stand +in for a Linux, Debug or sanitizer provider. Missing artifacts fail with the preparation +command; no skip, stub or implicit network fallback is available. + +The checker: + +* authenticates both historical header inventories and all three archives, checks old defined symbols, + and requires current archive freshness against all `src`, `include`, + `cmake` files and `CMakeLists.txt`; +* freezes execution aggregate field/order/type/status encodings, Action and CloseScope + alternatives, native header contracts, engine storage declarations and + virtual methods; compares actual compiler-emitted offsets/alignment/type + sizes for 251 engine data members plus returned structures; separately freezes + selected/projection headers and compares their layouts against 0e18690; +* compiles each old/new caller before any link result is interpreted; +* links old Book/singleton/lifecycle, old private F8/F11 wrappers and old + return-only host event callers against both full providers; +* links the six new selected/projection methods and new private provenance + helpers against the current provider, then rejects those same objects + against the real R2 provider for every required method; +* links selected/projection callers built with 0e18690 headers against both + 0e18690 and current archives, plus current selected callers against 0e18690; +* links one caller for all four reversal methods against the current archive, + then rejects that same object against real 0e18690 with each exact method + name and `reverse_to_v1::ReverseTo` parameter namespace; +* compiles wrong projection `_v2`/return-namespace and wrong selection-domain + callers, then requires rejection at those exact symbols. Return namespace + is not demanded in ordinary projection method mangling; +* rejects synthetic reversal `_v2` methods and `reverse_to_v2::ReverseTo` + parameters against current, supplementing the real historical rejection; +* refuses unrelated undefined symbols, retains full compiler/link logs and + records `executedBinaries: 0`. + +Private/protected member access uses explicit test-only `-fno-access-control` +on the pairing TUs. Product visibility is unchanged. Callers reinterpret an +unused command-line pointer and are link-only; **never execute them**. +The compiler's non-standard-layout `offsetof` extension is used only for +same-toolchain ABI observation, with its specific diagnostic suppressed. + +Three development modes return labeled partial receipts: +`--base-only` checks old-provider controls; `--old-rejections-only` checks +new public declarations against the real old provider before the new archive +exists; `--public-only` runs all public pairings before new private provenance +helpers are integrated. These partial modes do not include the reversal matrix +and may omit `--prior-receipt`. None returns `status: passed` or +`newArchivePairingComplete: true`. Do not use these modes in CI acceptance. + +The offline Python refusal tests are run by +`python3 scripts/test_settlement_cpp_abi.py -v`; they are tooling checks, not +substitutes for the actual compiler/archive matrix. diff --git a/tests/fixtures/settlement_cpp_abi/e60e571/manifest.json b/tests/fixtures/settlement_cpp_abi/e60e571/manifest.json new file mode 100644 index 000000000..92cf74ab8 --- /dev/null +++ b/tests/fixtures/settlement_cpp_abi/e60e571/manifest.json @@ -0,0 +1,267 @@ +{ + "commit": "e60e57156a54bb1c74c0c0a4f5a338fd924c046d", + "files": { + "include/pineforge/bar.hpp": { + "bytes": 655, + "git_blob": "836f73daab382e9c8d5a6d03a942a921e39b756f", + "sha256": "665c2528dca64ac9474de622dfd7ffdb540ff91316f70317e39aada158d1e0de" + }, + "include/pineforge/broker_events.hpp": { + "bytes": 4343, + "git_blob": "2972d53c7b039af266f4da161301c01eec54c16d", + "sha256": "4f3d9948a4d1604750901fa3982d6ea49bd50dd15a86c4e5d690b83d1f5f23c0" + }, + "include/pineforge/color.hpp": { + "bytes": 1253, + "git_blob": "a4d54f37a1c9d8d10ff5423aca4d42814a666446", + "sha256": "29eb9d4473e4dd488f959fb9f53999656e3feb6aa25255287e80855ecabe96d6" + }, + "include/pineforge/compat/pine/exit_activation.hpp": { + "bytes": 3017, + "git_blob": "a62927dd936e839160671cee665de908cb7f5dc3", + "sha256": "9190382359099021870d260718ad593c567024603b016294d1b113c278ddd73d" + }, + "include/pineforge/compat/pine/exit_lifecycle.hpp": { + "bytes": 882, + "git_blob": "c78f64b82adaf18e013e017a633b9b9b6b48f22d", + "sha256": "2fcfce8c914424404767bdd7b2fc6ea30decbc3f5f5a6123cac9080e2b9bd8d2" + }, + "include/pineforge/compat/pine/frozen_market_instruction.hpp": { + "bytes": 3201, + "git_blob": "8890ad48c5e4485b9d5a65a5bdd7f7f876e39c24", + "sha256": "e67a4b470e82086a089d3bf0b8996f88ab0fd5d86046682d6cb19b2ed64922c8" + }, + "include/pineforge/compat/pine/intraday_cap.hpp": { + "bytes": 10469, + "git_blob": "0aba76b6a78251df33e92f19290473adecf5a61e", + "sha256": "0ac0eab34625ab9a974037ff8c0217603a01deb81599350a4b24f7bf48ac0cbd" + }, + "include/pineforge/compat/pine/intraday_order_budget.hpp": { + "bytes": 3878, + "git_blob": "730d941fffa3939e375489f40a50901ccea9ec87", + "sha256": "43f7700f9c66c630ab8a9d8fd899eecb03c2fd9aabff11d2dc2b7fa85f92981b" + }, + "include/pineforge/compat/pine/market_admission.hpp": { + "bytes": 1057, + "git_blob": "33cf82e5e53ae870c3665ede69d7cde2e9695752", + "sha256": "83ce322f8864a569e197493cd2bcca23aa226997d3508caaa6a783ff1bb97b03" + }, + "include/pineforge/compat/pine/order_birth.hpp": { + "bytes": 841, + "git_blob": "aa48c054f613b65ee5ded14fde41643f807347bd", + "sha256": "e819f5c4202911e5e592f4472925a15c07f7300fabcfecf4f214348e5ee605f8" + }, + "include/pineforge/compat/pine/order_priority.hpp": { + "bytes": 1577, + "git_blob": "86135165fd3d83fc4638a2295dda769024836517", + "sha256": "e88ffcf28bf3fac95bc439b3d99a3944fc9c36ce0d3892fd365d25f9f3f920e4" + }, + "include/pineforge/compat/pine/reservation_expansion.hpp": { + "bytes": 759, + "git_blob": "1115a6593f5fabadf722b14305a213b07afb6c85", + "sha256": "7ad4195af574df40a600b748399f00c41cafa9441bfdcb5b90898ec8a4b577dd" + }, + "include/pineforge/drawing.hpp": { + "bytes": 17150, + "git_blob": "d04f12bb214b82c5d0b174824c25ff9f148ab090", + "sha256": "1635cfc9e65a91369cc1491a64dc564944b6828333af62e3ecd80edb5ba74183" + }, + "include/pineforge/engine.hpp": { + "bytes": 312105, + "git_blob": "3624e74ddb4785db78dbabb216664aa938452f0f", + "sha256": "7cbc440c20f1b4c7b24c95ea30463eef509a11dc2131a484c5e8a752db48e31d" + }, + "include/pineforge/execution.hpp": { + "bytes": 4294, + "git_blob": "f270b7c64544ee9188c22752d5e41eaa78bf01d1", + "sha256": "e6e2440118f38a0eba5b75a0fb0cf03675f92939c154aaf8dc0837653a113895" + }, + "include/pineforge/execution_close_scope.hpp": { + "bytes": 583, + "git_blob": "65294d9f25468ccf3e4c79861beb96b8037b56d5", + "sha256": "daba6e221a19d3d3cd1db248a2861414ef91648c8330709917067ab32b2ebdb4" + }, + "include/pineforge/execution_consumer.hpp": { + "bytes": 3010, + "git_blob": "19e99314462c9c3729953b0f343abd4aa0bf1496", + "sha256": "ffd09417d1f1e8aab32794673d6802616c5055b7976101e49416c283b64856e0" + }, + "include/pineforge/exit_leg_lifecycle.hpp": { + "bytes": 22622, + "git_blob": "91492ec22e05d7b6d9327f60ff5e239ae6906fcd", + "sha256": "9d785271511a350114adc7cf5bcb1600cf569f0a10559a8671b7e4b9cb9dc89a" + }, + "include/pineforge/generic_matrix.hpp": { + "bytes": 25438, + "git_blob": "8694e5577d56221498cca844aa14cddb9a15e84f", + "sha256": "9c94a877b6f9e5965604b03f572ceea2ea3680de586d69ef0627c78208624150" + }, + "include/pineforge/leg_activation.hpp": { + "bytes": 1353, + "git_blob": "4160c841782800853bb574219b3dad2ca8aa0cc6", + "sha256": "859d75d0ae90ec3cc109765da066b37c3bbbff9c7c840368fbdd05dc7e178f3f" + }, + "include/pineforge/live_parser.h": { + "bytes": 3870, + "git_blob": "0d00afe0586477c69dea35a1bfef72f3acb491a3", + "sha256": "f1308086a66fe6a0636b8b2fc44c3fcb6b0c21ed0311654297ecc8ea5e959e1e" + }, + "include/pineforge/log.hpp": { + "bytes": 496, + "git_blob": "6cc4c35ad1a902f05022fb11357dff86281b5301", + "sha256": "df5db36e01ddaa66c44c1b378e6b0c3b05e0c732c59fbcb5401e3606585d25a3" + }, + "include/pineforge/magnifier.hpp": { + "bytes": 3241, + "git_blob": "1bbc910a869d162900d7610fa0bd9a88776b7b0f", + "sha256": "c0ed220b6df054b44596a434d4f48b1c3e960ca329bb8644a405a608175e06ff" + }, + "include/pineforge/map.hpp": { + "bytes": 14476, + "git_blob": "5dd59cbaad9b93138738d3f92b8c457c0ac92a7d", + "sha256": "772a4f6c79ea01af6ad057e361e821d79a7d9884e4aff8bac5a2f77565e55b9f" + }, + "include/pineforge/market_admission.hpp": { + "bytes": 10221, + "git_blob": "70be6af6b556bf77cab6c9f823bb3492cf7c8294", + "sha256": "22914434a2c82f761b98c1b5a2ff9365c89bb802b584cdda2a3c30c5986ec6fe" + }, + "include/pineforge/market_driver.hpp": { + "bytes": 5374, + "git_blob": "3e6c68fb491156520640b1095dfce057395375a3", + "sha256": "f89b8b49127dff19f88199da50db3defbcd891c2a25cc777f595d7f277a7cd3c" + }, + "include/pineforge/math.hpp": { + "bytes": 1316, + "git_blob": "9ec09d539dd0ba732449424e6a8c80c3f5454888", + "sha256": "be95fa75d1e8d288ed49d62fa56257efe99435be7ba357fbb93cc36c0dc9eda3" + }, + "include/pineforge/matrix.hpp": { + "bytes": 4868, + "git_blob": "22b29448fb60dded446543b1940c1a1466986133", + "sha256": "f463ab76db92436fd818839c390e69462a794e07413d99fb49af4ed1b67bca6f" + }, + "include/pineforge/metrics.hpp": { + "bytes": 1353, + "git_blob": "f468f485bbdd0512536e96390d2b411228c2f517", + "sha256": "9e35101f16c2717dbfde428ecc515802dd758056e080fd45ee6411e9d0ead5be" + }, + "include/pineforge/na.hpp": { + "bytes": 1064, + "git_blob": "e98505aadc68a09cbe5a14aacf591a799a15ecd2", + "sha256": "0e8c256fcdeda94a0855458fa3b8c4ad4f9864dc2aecce969934f6bea82fd0a5" + }, + "include/pineforge/native_calendar.hpp": { + "bytes": 19589, + "git_blob": "742ecfabc3764d626617c2762d4b574d46d8ff46", + "sha256": "5495cb007509937616255a9615dcc92c49e5115692c8375ef93b594a36e55860" + }, + "include/pineforge/native_host.hpp": { + "bytes": 10304, + "git_blob": "58573006ad056ecaa773dc79d22ca1375d51fac8", + "sha256": "9b7236e5946d1e5d2355cd6a72a02d987ab6b215acf2155ebce89771c8015808" + }, + "include/pineforge/native_order.hpp": { + "bytes": 36944, + "git_blob": "238475d372cc87b91d163c6c1a378f10751d444e", + "sha256": "3881f6b1ebecb95801e5ea6f4acb5a5a8203d94322ffdc36663d69b4966b7c08" + }, + "include/pineforge/native_order_identity.hpp": { + "bytes": 1948, + "git_blob": "c0a6835c8fdb5c3b331ad519439af12946e34cf2", + "sha256": "6f6d63c1a1f54499f04a137d0b13ea343aaf94ffe090058a62c854311345ac69" + }, + "include/pineforge/native_run_spec.hpp": { + "bytes": 5518, + "git_blob": "1b5ed883a71ed772e9f115415ab5c7acb1f920f0", + "sha256": "e7be6d930c4eed13d9fa2358f529c98e5ad37b75ad72013f1ba65513cabb8da1" + }, + "include/pineforge/order_action.hpp": { + "bytes": 3470, + "git_blob": "d992472cbefefdf8e29704a21f04614d68d39a58", + "sha256": "7d76ddc3f6ba76216d5feb34cdf28908e33a92a56757a707ac6bd94dca315001" + }, + "include/pineforge/order_birth.hpp": { + "bytes": 4833, + "git_blob": "ed4a4038e9d079c92b4ebd777a5f22f1f1256b0c", + "sha256": "0eadb3e2a92248fbc8e6ba8f94285b15b0e83d7a0396f5904b40c216619604e4" + }, + "include/pineforge/order_cancellation.hpp": { + "bytes": 7172, + "git_blob": "a4f11409f19c7a85347879ecfd942bd7bae5c2eb", + "sha256": "389eb3659f121351ad4b53b936d3a2c79f343ecdb6c0c4bb0bc13777596b90ee" + }, + "include/pineforge/order_priority.hpp": { + "bytes": 694, + "git_blob": "3e9fa5ee4121971a7962db8a60859aba6fb9ec95", + "sha256": "25bb7898972153b044675245db230c1171644856ef904f4dd02353b8968d433b" + }, + "include/pineforge/pending_order_mirror.hpp": { + "bytes": 21978, + "git_blob": "fc08bfd7328718de1c658bdcea1826c2fbc9eebe", + "sha256": "7921179f690dbb6f00ee8ab6ca71b3174fcc42a61ebc6786c3727c7f85a576a4" + }, + "include/pineforge/pine_float_compare.hpp": { + "bytes": 2893, + "git_blob": "a3de0e9a2ef3517098be500880fda71bdcfe2260", + "sha256": "3d389142b8e4fbd899c6923c11a27447920a1c9e9db0338ff9abc8a9f4a06d1c" + }, + "include/pineforge/pineforge.h": { + "bytes": 74362, + "git_blob": "31db9f0b9f8ac5f33041917f7d599daff91013c8", + "sha256": "e4c735a1e39411db4156ac13a50cee10c554039b20f33e3c2299cd73be6a6c3e" + }, + "include/pineforge/position_close_obligation.hpp": { + "bytes": 1060, + "git_blob": "09576ee0f2a1d49becc46f5701f4369cf8f87bf6", + "sha256": "6f6ad0aaca952dda7a9386cb78e57705b3494364dbd8970b0a051fe99d05b777" + }, + "include/pineforge/quantity_intent.hpp": { + "bytes": 3054, + "git_blob": "41262fee5aa2db4032fc008542c573032563629b", + "sha256": "c4cc0f1cd5d6ed1f248825712a5ba4b4f5a808b787c05c58971934b89d770b90" + }, + "include/pineforge/reservation_expansion.hpp": { + "bytes": 2434, + "git_blob": "b4302c9eed97c3409d9911fd6cecdc64fa0aad86", + "sha256": "27e514249f62aa7353ac084a6e59e25e338a867fe4d67ce4646446a1cf9f9dab" + }, + "include/pineforge/series.hpp": { + "bytes": 3439, + "git_blob": "61baaef2c351d37b0d6a5eb179e91ce670eb83c3", + "sha256": "e7709f67df9046c9eba163c2dbe91f8161084a955265ccf6b163355856c0afde" + }, + "include/pineforge/session_time.hpp": { + "bytes": 10337, + "git_blob": "af3a4472d852e0965947c006d0b6ad85304539e7", + "sha256": "f58c94bfae948e08fa21e52f20eae3940b31e398d83b800baf3a31f3a5fde669" + }, + "include/pineforge/str_utils.hpp": { + "bytes": 820, + "git_blob": "d44059f10b41872ea57a1abd9c246fadb3b7b3eb", + "sha256": "58e052a040da94f3824ec36db05b37dfe9e2219a32e843b56998fb1f0efe1393" + }, + "include/pineforge/ta.hpp": { + "bytes": 35301, + "git_blob": "3baeef801f44b938c16137fca5881784646760f9", + "sha256": "7221324be971ef6de425253656327395d8a1ffd081b3c1eef0c8c5b96f51e6b6" + }, + "include/pineforge/timeframe.hpp": { + "bytes": 29534, + "git_blob": "3d1bc9d495ac7e5b39ea5bae57c9d9afe580192b", + "sha256": "5e4c312da463fdcff55e0887188523c8bbf85a2ae62f3317a2b23f22fe0161e2" + }, + "include/pineforge/version.h.in": { + "bytes": 893, + "git_blob": "27118bd81741597e187e3a49d04ac201f9508011", + "sha256": "9b9e81d5e0998441a82dbe15a4495182fdfe126baea1cf26bdaaf51e491ad99f" + }, + "include/pineforge/window_sum.hpp": { + "bytes": 2828, + "git_blob": "1ea86860f6cb901fe79ff89ef46f1c77ce7e2b8c", + "sha256": "b5519f95d57d46e2a8045d380c1e1e1427ce744f4d53ee3a04ebfdaf942d3f68" + } + }, + "schemaVersion": "pineforge-settlement-abi-base-headers/v1", + "tree": "e209c89035ec884774f8f3e68e1dc118f26aaf51" +} diff --git a/tests/test_native_reverse_to.cpp b/tests/test_native_reverse_to.cpp new file mode 100644 index 000000000..5ff60becd --- /dev/null +++ b/tests/test_native_reverse_to.cpp @@ -0,0 +1,524 @@ +// Literal tests of the actual resolved ReverseTo seam; no strategy/tape loop. +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge; +namespace x = pineforge::execution; +using ReverseTo = x::reverse_to_v1::ReverseTo; + +namespace { +int checks = 0, failures = 0; +const char* scenario = "setup"; +struct Abort {}; +#define CHECK(value) do { ++checks; if (!(value)) { ++failures; \ + std::printf("FAIL %s:%d %s\n", scenario, __LINE__, #value); } } while (0) +#define REQUIRE(value) do { const bool ok_ = bool(value); CHECK(ok_); \ + if (!ok_) throw Abort{}; } while (0) + +uint64_t bits(double value) { + uint64_t result; + static_assert(sizeof(result) == sizeof(value)); + std::memcpy(&result, &value, sizeof(result)); + return result; +} +void exact(double actual, double expected) { + if (bits(actual) != bits(expected)) + std::printf(" actual=%.17g (%016llx) expected=%.17g (%016llx)\n", + actual, static_cast(bits(actual)), + expected, static_cast(bits(expected))); + CHECK(bits(actual) == bits(expected)); +} +void near(double actual, double expected) { + const bool ok = std::isfinite(actual) && std::isfinite(expected) + && std::abs(actual - expected) <= 1e-12 * std::max(1.0, std::abs(expected)); + if (!ok) std::printf(" actual=%.17g expected=%.17g\n", actual, expected); + CHECK(ok); +} + +struct Book final : BacktestEngine { + Book() { + initial_capital_ = 1000; + commission_type_ = CommissionType::CASH_PER_ORDER; + commission_value_ = 0; + syminfo_.pointvalue = 1; + account_currency_fx_ = 1; + stream_observe_actions_ = true; + current_bar_ = {100, 130, 70, 100, 1, 1736121600000LL}; + bar_index_ = 3; + } + void on_bar(const Bar&) override {} + x::PhysicalExecutionContext context() const { + // Native execution coordinates intentionally differ from chart state. + return {1736121660000LL, 7, {}, {}}; + } + x::SettlementInspection inspect(double target, const x::Fill& fill) const { + return inspect_native_reversal_v1(ReverseTo{target}, fill); + } + x::AccountEffectProjection project(double target, const x::Fill& fill) const { + return project_native_reversal_v1(ReverseTo{target}, fill); + } + x::Result reverse(double target, const x::Fill& fill) { + return settle_native_reversal_at_v1(ReverseTo{target}, fill, context()); + } + x::Result reverse_with(double target, const x::Fill& fill, + const x::LifecycleEffects& effects) { + return settle_reversal_with_lifecycle_v1(ReverseTo{target}, fill, effects); + } + x::Result flow(const x::Action& action, const x::Fill& fill) { + return settle_native_execution_at(action, fill, context()); + } + void open(double quantity, double price, uint64_t incarnation, + std::optional paid = 0.0) { + const auto result = flow(order_action::Transact{quantity}, + {price, "seed-" + std::to_string(incarnation), "historical", incarnation, paid}); + REQUIRE(result.status == x::Status::Applied); + } + void schedule(CommissionType kind, double value, double pointvalue = 1, + double fx = 1) { + commission_type_ = kind; + commission_value_ = value; + syminfo_.pointvalue = pointvalue; + account_currency_fx_ = fx; + } + const auto& lots() const { return pyramid_entries_; } + auto& lots() { return pyramid_entries_; } + const auto& rows() const { return trades_; } + size_t actions() const { return stream_order_actions_.size(); } + double balance() const { return initial_capital_ + net_profit_sum_; } + double net() const { return net_profit_sum_; } + double marked(double price) const { return marked_equity(price); } + double quantity() const { + return position_side_ == PositionSide::SHORT ? -position_qty_ : position_qty_; + } + int64_t cycle() const { return position_cycle_seq_; } + int64_t next_cycle() const { return next_position_cycle_seq_; } + uint64_t next_order() const { return next_order_incarnation_; } + uint64_t event_sequence() const { return exit_leg_event_seq_; } + void next_cycle(int64_t value) { next_position_cycle_seq_ = value; } + void corrupt_side(PositionSide value) { position_side_ = value; } + void initial(double value) { initial_capital_ = value; } + void exhaust_stream() { stream_action_sequence_ = UINT64_MAX; } + void exhaust_wins() { win_trades_count_ = std::numeric_limits::max(); } + void exhaust_lifecycle() { exit_leg_event_seq_ = UINT64_MAX; } + void pending_exit() { + PendingOrder order; + order.type = OrderType::EXIT; + order.id = "retained"; + order.from_entry = "seed-11"; + order.incarnation = 700; + order.created_seq = 700; + order.legs.attach(order.incarnation, position_cycle_seq_); + pending_orders_.push_back(std::move(order)); + } + size_t pending_count() const { return pending_orders_.size(); } + const PendingOrder* pending_data() const { return pending_orders_.data(); } +}; + +x::Fill fill(double price = 100, std::optional ticket = {}) { + return {price, "reverse", "resolved once", 90, ticket}; +} +struct Snapshot { + uint64_t broker, stream, next_order, event_sequence; + int64_t cycle, next_cycle; + size_t lots, rows, actions, pending; + double balance, quantity; + const PendingOrder* pending_data; + explicit Snapshot(const Book& book) + : broker(book.broker_state_hash()), stream(book.stream_state_hash()), + next_order(book.next_order()), event_sequence(book.event_sequence()), + cycle(book.cycle()), next_cycle(book.next_cycle()), lots(book.lots().size()), + rows(book.rows().size()), actions(book.actions()), pending(book.pending_count()), + balance(book.balance()), quantity(book.quantity()), pending_data(book.pending_data()) {} + void unchanged(const Book& book) const { + CHECK(book.broker_state_hash() == broker); + CHECK(book.stream_state_hash() == stream); + CHECK(book.next_order() == next_order && book.event_sequence() == event_sequence); + CHECK(book.cycle() == cycle && book.next_cycle() == next_cycle); + CHECK(book.lots().size() == lots && book.rows().size() == rows); + CHECK(book.actions() == actions && book.pending_count() == pending); + CHECK(book.pending_data() == pending_data); + exact(book.balance(), balance); + exact(book.quantity(), quantity); + } +}; + +x::AccountEffectProjection agree_and_apply(Book& book, double target, const x::Fill& f) { + const Snapshot before(book); + const auto inspection = book.inspect(target, f); + before.unchanged(book); + const auto projection = book.project(target, f); + before.unchanged(book); + const auto repeated = book.project(target, f); + before.unchanged(book); + REQUIRE(inspection.status == x::Status::Applied); + REQUIRE(projection.status == x::Status::Applied); + REQUIRE(repeated.status == x::Status::Applied); + exact(inspection.opened_units, target); + exact(projection.opened_units, target); + exact(projection.signed_units_after, target); + exact(inspection.resulting_abs_units, std::abs(target)); + exact(projection.resulting_abs_units, std::abs(target)); + CHECK(inspection.resulting_lot_count == 1 && projection.resulting_lot_count == 1); + CHECK(inspection.would_open && projection.would_open); + CHECK(inspection.incoming_short == (target < 0)); + CHECK(projection.incoming_short == (target < 0)); + exact(inspection.closed_units, projection.closed_units); + exact(inspection.current_ticket, projection.current_ticket); + exact(inspection.resulting_abs_notional, projection.resulting_abs_notional); + exact(repeated.realized_balance, projection.realized_balance); + CHECK(projection.cycle_after == before.next_cycle); + CHECK(repeated.cycle_after == before.next_cycle); + + const auto result = book.reverse(target, f); + REQUIRE(result.status == x::Status::Applied); + REQUIRE(book.lots().size() == 1); + exact(result.closed_units, projection.closed_units); + exact(result.opened_units, target); + exact(result.current_ticket, projection.current_ticket); + exact(book.lots()[0].qty, std::abs(target)); + exact(book.quantity(), target); + exact(book.balance(), projection.realized_balance); + exact(book.marked(f.price), projection.marked_equity); + exact(book.lots()[0].entry_commission_account, projection.remaining_entry_cost); + CHECK(book.cycle() == projection.cycle_after && book.next_cycle() == before.next_cycle + 1); + CHECK(result.first_trade_index == before.rows); + CHECK(result.closed_trade_count == before.lots); + CHECK(result.opened_lot_incarnation == f.incarnation); + CHECK(book.rows().size() == before.rows + before.lots); + CHECK(book.actions() == before.actions + before.lots + 1); + CHECK(book.lots()[0].entry_incarnation == f.incarnation); + CHECK(book.lots()[0].entry_id == f.id && book.lots()[0].entry_comment == f.comment); + exact(book.lots()[0].price, f.price); + CHECK(book.lots()[0].time == book.context().effective_time_ms); + CHECK(book.lots()[0].entry_bar_index == book.context().interval_index); + for (size_t index = before.rows; index < book.rows().size(); ++index) { + CHECK(book.rows()[index].exit_time == book.context().effective_time_ms); + CHECK(book.rows()[index].exit_bar_index == book.context().interval_index); + exact(book.rows()[index].exit_price, f.price); + CHECK(book.rows()[index].exit_id == f.id && book.rows()[index].exit_comment == f.comment); + } + return projection; +} + +void exact_target(double sign) { + scenario = "ReverseTo exact non-dyadic target and one cycle per reversal"; + Book book; + book.open(sign, 100, 11); + const auto projection = agree_and_apply(book, -sign * .1, fill(100 + sign * 10)); + REQUIRE(book.rows().size() == 1); + exact(book.rows()[0].qty, 1); + near(book.rows()[0].pnl, 10); + near(projection.realized_balance, 1010); + CHECK(bits(book.lots()[0].qty) == UINT64_C(0x3fb999999999999a)); + agree_and_apply(book, sign * .2, fill(100)); + CHECK(book.cycle() == 3 && book.next_cycle() == 4); +} + +void non_dyadic_roster(double sign) { + scenario = "ReverseTo closes each non-dyadic roster member in order"; + Book book; + book.open(sign * .1, 100, 11); + book.open(sign * .2, 100, 12); + book.open(sign * .3, 100, 13); + agree_and_apply(book, -sign * .1, fill()); + REQUIRE(book.rows().size() == 3); + const double quantities[] = {.1, .2, .3}; + for (size_t index = 0; index < 3; ++index) { + CHECK(book.rows()[index].entry_incarnation == 11 + index); + exact(book.rows()[index].qty, quantities[index]); + } +} + +void absorbed_quantity(double sign, bool tiny_target) { + scenario = tiny_target ? "large held and absorbed tiny target remains representable" + : "tiny held and absorbed large target remains representable"; + const double held = tiny_target ? 1e16 : .1; + const double target = tiny_target ? .1 : 1e16; + Book book; + book.open(sign * held, 1, 11); + agree_and_apply(book, -sign * target, fill(1)); + REQUIRE(book.rows().size() == 1); + exact(book.rows()[0].qty, held); +} + +void tiny_roster_member(double sign) { + scenario = "ReverseTo whole-book close retains a tiny physical roster member"; + Book book; + book.open(sign * 1e16, 1, 11); + // Construct the valid physical roster directly: a Transact add intentionally + // cannot express this absorbed member. ReverseTo must still close both lots. + PyramidEntry tiny{1, 1736121600000LL, .1, "tiny", 3}; + tiny.entry_incarnation = 12; + tiny.entry_commission_account = 0; + book.lots().push_back(tiny); + agree_and_apply(book, -sign * .2, fill(1)); + REQUIRE(book.rows().size() == 2); + CHECK(book.rows()[0].entry_incarnation == 11 && book.rows()[1].entry_incarnation == 12); + exact(book.rows()[0].qty, 1e16); + exact(book.rows()[1].qty, .1); +} + +void one_ticket(double sign, CommissionType type, double value, + std::optional explicit_ticket, double ticket, + double first_commission, double second_commission, + double opening_paid, double balance, double equity) { + scenario = "ReverseTo one current ticket conserves historical paid entry costs"; + Book book; + book.open(sign * 2, 120 - sign * 20, 11, 4); + book.open(sign * 3, 120 - sign * 10, 12, 6); + book.schedule(type, value); + const auto projection = agree_and_apply(book, -sign * 5, fill(120, explicit_ticket)); + REQUIRE(book.rows().size() == 2); + near(projection.current_ticket, ticket); + near(book.rows()[0].commission, first_commission); + near(book.rows()[1].commission, second_commission); + near(book.lots()[0].entry_commission_account, opening_paid); + near(book.balance(), balance); + near(book.marked(120), equity); + // Existing paid costs are 10; one current ticket is shared by all effects. + near(book.rows()[0].commission + book.rows()[1].commission + + book.lots()[0].entry_commission_account, 10 + ticket); +} + +void historical_fx(double sign) { + scenario = "ReverseTo percentage fee uses current FX without repricing paid entry costs"; + Book book; + book.schedule(CommissionType::PERCENT, 1, 2, 2); + book.open(sign * 2, 120 - sign * 20, 11, std::nullopt); + book.open(sign * 3, 120 - sign * 10, 12, std::nullopt); + const double first_paid = sign > 0 ? 8 : 11.2; + const double second_paid = sign > 0 ? 13.2 : 15.6; + near(book.lots()[0].entry_commission_account, first_paid); + near(book.lots()[1].entry_commission_account, second_paid); + book.schedule(CommissionType::PERCENT, 1, 2, 3); + const auto projection = agree_and_apply(book, -sign * 5, fill(120)); + REQUIRE(book.rows().size() == 2); + near(projection.current_ticket, 72); + near(book.rows()[0].commission, first_paid + 14.4); + near(book.rows()[1].commission, second_paid + 21.6); + near(book.rows()[0].pnl, sign > 0 ? 217.6 : 214.4); + near(book.rows()[1].pnl, sign > 0 ? 145.2 : 142.8); + near(projection.remaining_entry_cost, 36); + near(projection.realized_balance, sign > 0 ? 1362.8 : 1357.2); + near(projection.marked_equity, sign > 0 ? 1326.8 : 1321.2); +} + +void invalid_inspection(const x::SettlementInspection& value, x::Status status) { + CHECK(value.status == status); + CHECK(value.closed_units == 0 && value.opened_units == 0 && value.resulting_abs_units == 0); + CHECK(value.resulting_lot_count == 0 && value.resulting_abs_notional == 0 && value.current_ticket == 0); + CHECK(!value.would_open && !value.incoming_short); +} +void refused(Book& book, double target, const x::Fill& f, x::Status status) { + const Snapshot before(book); + invalid_inspection(book.inspect(target, f), status); + before.unchanged(book); + const auto projection = book.project(target, f); + CHECK(projection.status == status); + CHECK(projection.closed_units == 0 && projection.opened_units == 0 && projection.resulting_abs_units == 0); + CHECK(projection.resulting_lot_count == 0 && projection.resulting_abs_notional == 0 && projection.current_ticket == 0); + CHECK(!projection.would_open && !projection.incoming_short); + CHECK(projection.realized_balance == 0 && projection.remaining_entry_cost == 0 && projection.marked_equity == 0); + CHECK(projection.cycle_after == 0 && projection.signed_units_after == 0); + before.unchanged(book); + const auto result = book.reverse(target, f); + CHECK(result.status == status); + CHECK(result.closed_units == 0 && result.opened_units == 0 && result.current_ticket == 0); + CHECK(result.first_trade_index == 0 && result.closed_trade_count == 0 && result.opened_lot_incarnation == 0); + before.unchanged(book); +} + +void quantity_and_book_refusals(double sign) { + scenario = "ReverseTo has no NoEffect quantity or side case"; + Book book; + book.open(sign, 100, 11); + for (double target : {0.0, -0.0, std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity(), -std::numeric_limits::infinity()}) + refused(book, target, fill(), x::Status::InvalidQuantity); + refused(book, sign * .1, fill(), x::Status::InvalidCloseTarget); + Book flat; + refused(flat, -sign * .1, fill(), x::Status::InvalidCloseTarget); + for (double invalid : {std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity()}) { + refused(book, -sign * .1, fill(invalid), x::Status::InvalidPrice); + refused(book, -sign * .1, fill(100, invalid), x::Status::InvalidAccounting); + } + scenario = "ReverseTo invalid physical book refuses before mutation"; + for (double quantity : {-1.0, 0.0, std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity()}) { + Book invalid; + invalid.open(sign, 100, 11); + invalid.lots()[0].qty = quantity; + refused(invalid, -sign * .1, fill(), x::Status::InvalidBook); + } + Book invalid_price; + invalid_price.open(sign, 100, 11); + invalid_price.lots()[0].price = std::numeric_limits::infinity(); + refused(invalid_price, -sign * .1, fill(), x::Status::InvalidBook); + Book invalid_side; + invalid_side.open(sign, 100, 11); + invalid_side.corrupt_side(static_cast(99)); + refused(invalid_side, -sign * .1, fill(), x::Status::InvalidBook); + Book absent_roster; + absent_roster.open(sign, 100, 11); + absent_roster.lots().clear(); + refused(absent_roster, -sign * .1, fill(), x::Status::InvalidBook); +} + +void overflow_refusals(double sign) { + scenario = "ReverseTo finite gross overflow precedes commission quote"; + Book gross; + const double large = std::numeric_limits::max() * .75; + gross.open(sign * large, 0, 11); + // A fee quote would itself be invalid. Quantity overflow must win first. + gross.schedule(CommissionType::CASH_PER_ORDER, std::numeric_limits::quiet_NaN()); + refused(gross, -sign * large, fill(0), x::Status::UnrepresentableQuantity); + + scenario = "ReverseTo finite component fees cannot overflow the aggregate ticket"; + Book ticket; + ticket.open(sign, 0, 11); + ticket.open(sign, 0, 12); + ticket.schedule(CommissionType::CASH_PER_CONTRACT, std::numeric_limits::max() / 2); + refused(ticket, -sign, fill(0), x::Status::InvalidAccounting); + + scenario = "ReverseTo overflowing held roster is invalid book"; + Book held; + held.open(sign * large, 0, 11); + auto copy = held.lots()[0]; + copy.entry_incarnation = 12; + held.lots().push_back(copy); + refused(held, -sign, fill(0), x::Status::InvalidBook); +} + +template void throws_atomically(Book& book, F call) { + const Snapshot before(book); + bool threw = false; + try { call(); } catch (const std::overflow_error&) { threw = true; } + CHECK(threw); + before.unchanged(book); +} +void exception_preflight() { + scenario = "ReverseTo cycle projection peeks without consuming exhausted sequence"; + for (int64_t next : {int64_t{0}, int64_t{-1}, std::numeric_limits::max()}) { + Book book; + book.open(1, 100, 11); + book.next_cycle(next); + const Snapshot before(book); + CHECK(book.inspect(-.1, fill()).status == x::Status::Applied); + before.unchanged(book); + throws_atomically(book, [&] { (void)book.project(-.1, fill()); }); + throws_atomically(book, [&] { (void)book.reverse(-.1, fill()); }); + } + scenario = "ReverseTo exhausted close counter refuses before close or opening"; + Book counter; + counter.open(1, 100, 11); + counter.exhaust_wins(); + throws_atomically(counter, [&] { (void)counter.reverse(-.1, fill(110)); }); + scenario = "ReverseTo exhausted stream refuses before close or opening"; + Book stream; + stream.open(1, 100, 11); + stream.exhaust_stream(); + throws_atomically(stream, [&] { (void)stream.reverse(-.1, fill()); }); + scenario = "ReverseTo exhausted pending lifecycle refuses atomically"; + Book lifecycle; + lifecycle.open(1, 100, 11); + lifecycle.pending_exit(); + lifecycle.exhaust_lifecycle(); + throws_atomically(lifecycle, [&] { (void)lifecycle.reverse(-.1, fill()); }); +} + +void invalid_lifecycle() { + scenario = "ReverseTo lifecycle wrapper validates batch before financial effects"; + Book book; + book.open(1, 100, 11); + book.pending_exit(); + x::LifecycleEffects effects; + effects.removals.push_back({999, 999, {}, 0}); + const Snapshot before(book); + const auto result = book.reverse_with(-.1, fill(), effects); + CHECK(result.status == x::Status::InvalidLifecycle); + CHECK(result.closed_units == 0 && result.opened_units == 0 && result.current_ticket == 0); + before.unchanged(book); +} + +void fresh_commit() { + scenario = "ReverseTo projection is read-only and does not authorize a stale close"; + Book book; + book.open(1, 100, 11); + const auto earlier = book.project(-.1, fill(110)); + REQUIRE(earlier.status == x::Status::Applied); + exact(earlier.closed_units, 1); + REQUIRE(book.flow(order_action::Reduce{.5}, fill(110, 0)).status == x::Status::Applied); + const auto fresh = agree_and_apply(book, -.1, fill(110)); + exact(fresh.closed_units, .5); + CHECK(fresh.closed_units != earlier.closed_units); + const Snapshot after(book); + CHECK(book.reverse(-.1, fill(110)).status == x::Status::InvalidCloseTarget); + after.unchanged(book); +} + +void sequential_balance() { + scenario = "ReverseTo projection preserves sequential realized PnL association"; + Book book; + book.initial(1); + const double high = 10000000000000100.0; + book.open(1, high, 11); + REQUIRE(book.flow(x::Flatten{}, fill(100, 0)).status == x::Status::Applied); + exact(book.net(), -1e16); + book.open(1, 100, 12); + book.open(.5, high - 2, 13); + const auto projection = agree_and_apply(book, -.1, fill(high, 0)); + REQUIRE(book.rows().size() == 3); + exact(book.rows()[1].pnl, 1e16); + exact(book.rows()[2].pnl, 1); + exact(book.net(), 1); + exact(projection.realized_balance, 2); + exact(projection.marked_equity, 2); +} + +template void run(F call) { + try { call(); } + catch (const Abort&) {} + catch (const std::exception& error) { + ++failures; + std::printf("FAIL %s exception: %s\n", scenario, error.what()); + } +} +} // namespace + +int main() { + static_assert(std::variant_size_v == 3); + for (double sign : {1.0, -1.0}) { + run([&] { exact_target(sign); }); + run([&] { non_dyadic_roster(sign); }); + run([&] { absorbed_quantity(sign, true); }); + run([&] { absorbed_quantity(sign, false); }); + run([&] { tiny_roster_member(sign); }); + run([&] { one_ticket(sign, CommissionType::CASH_PER_ORDER, 10, {}, 10, 6, 9, 5, 1055, 1050); }); + run([&] { one_ticket(sign, CommissionType::CASH_PER_CONTRACT, 2, {}, 20, 8, 12, 10, 1050, 1040); }); + run([&] { one_ticket(sign, CommissionType::CASH_PER_ORDER, 99, 0, 0, 4, 6, 0, 1060, 1060); }); + run([&] { one_ticket(sign, CommissionType::CASH_PER_ORDER, 99, -10, -10, 2, 3, -5, 1065, 1070); }); + run([&] { one_ticket(sign, CommissionType::CASH_PER_ORDER, -10, {}, -10, 2, 3, -5, 1065, 1070); }); + run([&] { historical_fx(sign); }); + run([&] { quantity_and_book_refusals(sign); }); + run([&] { overflow_refusals(sign); }); + } + run(exception_preflight); + run(invalid_lifecycle); + run(fresh_commit); + run(sequential_balance); + std::printf("%s native ReverseTo: %d checks, %d failures\n", + failures ? "FAIL" : "PASS", checks, failures); + return failures ? 1 : 0; +} diff --git a/tests/test_native_selected_settlement.cpp b/tests/test_native_selected_settlement.cpp new file mode 100644 index 000000000..4c45c6265 --- /dev/null +++ b/tests/test_native_selected_settlement.cpp @@ -0,0 +1,171 @@ +// R3 selected-set owner: literal economics on the actual settlement seam. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge; +namespace x = pineforge::execution; +namespace { +int checks=0, failures=0; +const char* scenario="setup"; +struct Abort {}; +#define CHECK(v) do { ++checks; if(!(v)){++failures;std::printf("FAIL %s:%d %s\n",scenario,__LINE__,#v);} } while(0) +#define REQUIRE(v) do { bool ok_=bool(v);CHECK(ok_);if(!ok_)throw Abort{}; } while(0) +void near(double a,double b) { + const bool ok=std::isfinite(a)&&std::isfinite(b)&&std::abs(a-b)<=1e-12*std::max(1.0,std::abs(b)); + if(!ok)std::printf(" actual=%.17g expected=%.17g\n",a,b);CHECK(ok); +} +struct Book final : BacktestEngine { + Book(){initial_capital_=1000;commission_type_=CommissionType::CASH_PER_ORDER;commission_value_=6; + syminfo_.pointvalue=1;account_currency_fx_=1;stream_observe_actions_=true; + current_bar_={100,100,100,100,1,1736121600000LL};bar_index_=7;} + void on_bar(const Bar&) override {} + x::PhysicalExecutionContext context() const {return {current_bar_.timestamp,bar_index_,{}, {}};} + void open(double q,double price,uint64_t id,const char* label,double paid=0){ + const auto r=settle_native_execution_at(order_action::Transact{q},x::Fill{price,label,"",id,paid},context()); + REQUIRE(r.status==x::Status::Applied); + } + void schedule(CommissionType kind,double value,double pv=1,double fx=1){ + commission_type_=kind;commission_value_=value;syminfo_.pointvalue=pv;account_currency_fx_=fx; + } + x::SettlementInspection inspect(const x::Action& a,const x::Fill& f,const x::SelectedOpeningSet& s)const{ + return inspect_native_settlement_selected(a,f,s); + } + x::AccountEffectProjection project(const x::Action& a,const x::Fill& f,const x::SelectedOpeningSet& s)const{ + return project_native_settlement_selected_v1(a,f,s); + } + x::Result settle(const x::Action& a,const x::Fill& f,const x::SelectedOpeningSet& s){ + return settle_native_execution_selected_at(a,f,context(),s); + } + int64_t cycle()const{return position_cycle_seq_;} + int64_t next_cycle()const{return next_position_cycle_seq_;} + uint64_t next_order()const{return next_order_incarnation_;} + const auto& lots()const{return pyramid_entries_;} + const auto& rows()const{return trades_;} + size_t actions()const{return stream_order_actions_.size();} + double qty()const{return position_qty_;} + double net()const{return net_profit_sum_;} +}; +x::Fill fill(double price,std::optional ticket={}){return {price,"selected-close","",900,ticket};} + +void invalid_sets(double sign){ + scenario="selected invalid atomicity";Book b;b.open(sign,100,11,"A",2);b.open(sign*3,100,22,"B",6); + const uint64_t hash=b.broker_state_hash(),stream=b.stream_state_hash(); + const auto actions=b.actions();const auto next=b.next_cycle();const auto order=b.next_order(); + const std::vector bad={{b.cycle(),{}},{b.cycle(),{0}}, + {b.cycle(),{11,11}},{b.cycle(),{11,999}},{0,{11}},{-1,{11}},{b.cycle()+1,{11}}}; + for(const auto& selector:bad){ + const auto original=selector.incarnations; + CHECK(b.inspect(x::Flatten{},fill(100),selector).status==x::Status::InvalidCloseTarget); + CHECK(b.project(x::Flatten{},fill(100),selector).status==x::Status::InvalidCloseTarget); + CHECK(b.settle(x::Flatten{},fill(100),selector).status==x::Status::InvalidCloseTarget); + CHECK(selector.incarnations==original); + CHECK(b.broker_state_hash()==hash&&b.stream_state_hash()==stream); + CHECK(b.next_cycle()==next&&b.next_order()==order&&b.actions()==actions&&b.rows().empty()); + } + const x::SelectedOpeningSet valid{b.cycle(),{22,11}}; + CHECK(b.inspect(order_action::Transact{sign},fill(100),valid).status==x::Status::InvalidCloseTarget); + CHECK(b.project(order_action::Transact{sign},fill(100),valid).status==x::Status::InvalidCloseTarget); + CHECK(b.settle(order_action::Transact{sign},fill(100),valid).status==x::Status::InvalidCloseTarget); + CHECK(b.settle(order_action::Reduce{0},fill(100),valid).status==x::Status::NoEffect); + for(double fee:{1.0,-1.0}) + CHECK(b.settle(order_action::Reduce{0},fill(100,fee),valid).status==x::Status::InvalidAccounting); + CHECK(b.broker_state_hash()==hash&&b.stream_state_hash()==stream); +} + +void unequal_fee_attribution(double sign,std::optional total,double first_fee,double last_fee){ + scenario="selected FIFO and unequal one-ticket attribution";Book b; + b.open(sign,100,11,"A");b.open(sign*2,105,22,"untouched",2);b.open(sign*3,100,33,"C"); + const PyramidEntry untouched=b.lots()[1]; + x::SelectedOpeningSet selected{b.cycle(),{33,11}};const auto ids=selected.incarnations; + const auto before=b.broker_state_hash(); + const auto q=b.inspect(x::Flatten{},fill(100+sign*10,total),selected); + REQUIRE(q.status==x::Status::Applied);CHECK(q.closed_units==4&&q.resulting_lot_count==1); + near(q.current_ticket,first_fee+last_fee);CHECK(b.broker_state_hash()==before); + const auto r=b.settle(x::Flatten{},fill(100+sign*10,total),selected); + REQUIRE(r.status==x::Status::Applied);CHECK(r.closed_units==4&&r.closed_trade_count==2); + CHECK(selected.incarnations==ids);REQUIRE(b.rows().size()==2&&b.lots().size()==1); + CHECK(b.rows()[0].entry_incarnation==11&&b.rows()[1].entry_incarnation==33); + near(b.rows()[0].commission,first_fee);near(b.rows()[1].commission,last_fee); + near(b.rows()[0].pnl,10-first_fee);near(b.rows()[1].pnl,30-last_fee); + near(b.net(),40-first_fee-last_fee); + CHECK(b.lots()[0].entry_incarnation==untouched.entry_incarnation); + CHECK(b.lots()[0].price==untouched.price&&b.lots()[0].qty==untouched.qty); + CHECK(b.lots()[0].time==untouched.time&&b.lots()[0].entry_commission_account==untouched.entry_commission_account); + CHECK(b.cycle()==selected.cycle); +} + +void fragments_and_schedule_change(double sign){ + scenario="selected legal fragments and historical fee residue";Book b; + b.schedule(CommissionType::PERCENT,10); + b.open(sign,100,11,"A-fragment-1",2);b.open(sign*3,100,22,"other",3); + b.open(sign*2,100,11,"A-fragment-2",4); + b.schedule(CommissionType::CASH_PER_ORDER,6); + const x::SelectedOpeningSet selection{b.cycle(),{11}}; + const auto r=b.settle(order_action::Reduce{1.5},fill(100+sign*10),selection); + REQUIRE(r.status==x::Status::Applied);CHECK(r.closed_units==1.5&&r.closed_trade_count==2); + REQUIRE(b.rows().size()==2&&b.lots().size()==2); + CHECK(b.rows()[0].entry_incarnation==11&&b.rows()[1].entry_incarnation==11); + near(b.rows()[0].qty,1);near(b.rows()[1].qty,.5); + // Current shares4/2 plus historical2/1. A schedule change cannot reprice the past. + near(b.rows()[0].commission,6);near(b.rows()[1].commission,3); + near(b.rows()[0].pnl,4);near(b.rows()[1].pnl,2); + CHECK(b.lots()[0].entry_incarnation==22);near(b.lots()[0].qty,3);near(b.lots()[0].entry_commission_account,3); + CHECK(b.lots()[1].entry_incarnation==11);near(b.lots()[1].qty,1.5);near(b.lots()[1].entry_commission_account,3); +} + +void per_unit_and_negative_price_percent(double sign){ + scenario="selected cash-per-unit with scalar FX";Book units; + units.schedule(CommissionType::CASH_PER_CONTRACT,2,3,2); + units.open(sign,100,11,"A");units.open(sign*3,100,22,"B"); + const auto u=units.settle(x::Flatten{},fill(100+sign*10),{units.cycle(),{22,11}}); + REQUIRE(u.status==x::Status::Applied);near(u.current_ticket,8); + near(units.rows()[0].commission,2);near(units.rows()[1].commission,6);near(units.net(),232); + scenario="selected absolute-notional percent at negative price";Book percent; + percent.schedule(CommissionType::PERCENT,1,2,3); + percent.open(sign,-100,11,"A");percent.open(sign*3,-100,22,"B"); + const auto p=percent.settle(x::Flatten{},fill(-100+sign*10),{percent.cycle(),{22,11}}); + REQUIRE(p.status==x::Status::Applied); + const double one=sign>0?5.4:6.6; + near(p.current_ticket,one*4);near(percent.rows()[0].commission,one);near(percent.rows()[1].commission,one*3); + near(percent.net(),240-one*4); +} + +void interior_dust(double sign){ + scenario="selected interior positive residue";Book b;b.schedule(CommissionType::CASH_PER_ORDER,0); + b.open(sign,100,11,"A",6);b.open(sign*3,100,22,"B",0); + const double q=1.0-5e-12;const double residue=1.0-q; + const x::SelectedOpeningSet selection{b.cycle(),{11}}; + const auto r=b.settle(order_action::Reduce{q},fill(100+sign*10),selection); + REQUIRE(r.status==x::Status::Applied);CHECK(r.closed_units==q&&r.closed_trade_count==1); + REQUIRE(b.lots().size()==2);CHECK(b.lots()[0].entry_incarnation==11&&b.lots()[0].qty==residue); + CHECK(b.lots()[0].qty>0&&b.lots()[0].entry_commission_account>0); + CHECK(b.lots()[0].entry_commission_account==6.0-6.0*q); + CHECK(b.lots()[1].entry_incarnation==22&&b.lots()[1].qty==3); + CHECK(b.qty()==b.lots()[0].qty+b.lots()[1].qty);CHECK(b.cycle()==selection.cycle); + CHECK(b.rows()[0].qty==q);near(b.rows()[0].commission,6*q); +} + +templatevoid run(F fn){try{fn();}catch(const Abort&){}catch(const std::exception& e){++failures;std::printf("FAIL %s exception %s\n",scenario,e.what());}} +} +int main(){ + for(double sign:{1.0,-1.0}){ + run([&]{invalid_sets(sign);}); + run([&]{unequal_fee_attribution(sign,std::nullopt,1.5,4.5);}); + run([&]{unequal_fee_attribution(sign,12,3,9);}); + run([&]{unequal_fee_attribution(sign,0,0,0);}); + run([&]{unequal_fee_attribution(sign,-6,-1.5,-4.5);}); + run([&]{fragments_and_schedule_change(sign);}); + run([&]{per_unit_and_negative_price_percent(sign);}); + run([&]{interior_dust(sign);}); + } + std::printf("%s native selected settlement: %d checks %d failures\n",failures?"FAIL":"PASS",checks,failures); + return failures?1:0; +} diff --git a/tests/test_native_settlement_callers.cpp b/tests/test_native_settlement_callers.cpp new file mode 100644 index 000000000..923bab0a0 --- /dev/null +++ b/tests/test_native_settlement_callers.cpp @@ -0,0 +1,251 @@ +// R3 live adapter helpers: actual source selection/sizing followed by the one book owner. +// Private member access uses the same explicit-instantiation pattern as R3a tests. +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge; +namespace x=pineforge::execution; +namespace { +int checks=0,failures=0;const char* scenario="setup"; +struct Abort{}; +#define CHECK(v) do{++checks;if(!(v)){++failures;std::printf("FAIL %s:%d %s\n",scenario,__LINE__,#v);}}while(0) +#define REQUIRE(v) do{bool ok_=bool(v);CHECK(ok_);if(!ok_)throw Abort{};}while(0) +void near(double a,double b){const bool ok=std::isfinite(a)&&std::isfinite(b)&&std::abs(a-b)<=1e-12*std::max(1.0,std::abs(b)); + if(!ok)std::printf(" actual=%.17g expected=%.17g\n",a,b);CHECK(ok);} +templatestruct Access{friend auto access(Tag){return Member;}}; +#define ACCESS(Tag,Method) struct Tag{friend auto access(Tag);}; template struct Access +ACCESS(Partial,execute_partial_exit_qty); +ACCESS(Percent,execute_partial_exit); +ACCESS(ByEntry,execute_partial_exit_by_entry); +ACCESS(ByQty,execute_partial_exit_by_entry_qty); +ACCESS(ByPercent,execute_partial_exit_by_entry_percent); +ACCESS(Scratch,cover_samebar_market_adds_on_exit); +ACCESS(Flip,flip_market_position_to); +ACCESS(Sequential,sequential_same_tick_reversal_fill); +ACCESS(Add,add_to_pyramid_market); +ACCESS(Entry,execute_market_entry); +ACCESS(Raw,apply_raw_order_fill); +ACCESS(Exit,apply_exit_order_fill); +ACCESS(Range,record_range_end_close_trades); +#undef ACCESS +templatestruct Args; +templatestruct Args{using tuple=std::tuple;}; +using Cause=std::tuple_element_t<2,typename Args::tuple>; +constexpr Cause Script=static_cast(0),Bracket=static_cast(1),Margin=static_cast(2); + +struct Book final:BacktestEngine{ + Book(){initial_capital_=1000;commission_type_=CommissionType::CASH_PER_ORDER;commission_value_=6; + syminfo_.pointvalue=1;syminfo_.mintick=.01;syminfo_mintick_=.01;account_currency_fx_=1; + pyramiding_=100;qty_step_=0;slippage_=0;stream_observe_actions_=true;bar(3,100);} + void on_bar(const Bar&)override{} + void bar(int index,double close){bar_index_=index;current_bar_={close,close+20,close-20,close,1,1736121600000LL+index*60000};} + void open(double q,double price,uint64_t inc,const char* label,double paid=0){ + x::PhysicalExecutionContext c{current_bar_.timestamp,bar_index_,{}, {}}; + REQUIRE(settle_native_execution_at(order_action::Transact{q},x::Fill{price,label,"",inc,paid},c).status==x::Status::Applied); + } + void partial(double p,double q,Cause c=Script){(this->*access(Partial{}))(p,q,c);} + void percent(double p,double q,Cause c=Script){(this->*access(Percent{}))(p,q,c);} + void by_qty(double p,const char* id,double q,Cause c=Script){(this->*access(ByQty{}))(p,std::string(id),q,c);} + void by_all(double p,const char* id,Cause c=Script){(this->*access(ByEntry{}))(p,std::string(id),c);} + void by_percent(double p,const char* id,double q,Cause c=Script){(this->*access(ByPercent{}))(p,std::string(id),q,c);} + double scratch(const PendingOrder& o,double p,Cause c=Bracket){return (this->*access(Scratch{}))(o,p,c);} + void flip(bool buy,double p,double q,int type,bool frozen=false,bool close_only=false){ + (this->*access(Flip{}))(std::string("flip"),buy,p,q,type,frozen,close_only,90); + } + void sequential(bool buy,double p,double q,int type){(this->*access(Sequential{}))(std::string("sequential"),buy,p,q,type,91);} + void old_add(bool buy,double p,double q){(this->*access(Add{}))(std::string("old-add"),buy,p,q,-1,position_side_,false,92);} + void dispatch(bool buy,double p,double q,bool later,bool frozen){ + (this->*access(Entry{}))(std::string("dispatch"),buy,p,q,-1,position_side_,false,false,0,bar_index_,later,false,frozen,93); + } + void raw(PendingOrder& o,double p){double trail=trail_best_price_;int closed_bar=-1;uint64_t closed_inc=0;bool closed_long=false; + (this->*access(Raw{}))(o,p,trail,closed_bar,closed_inc,closed_long);} + void exit(PendingOrder& o,double p){int closed_bar=-1;uint64_t closed_inc=0;bool closed_long=false; + (this->*access(Exit{}))(o,p,closed_bar,closed_inc,closed_long);} + void range(){record_equity_point(current_bar_.timestamp);(this->*access(Range{}))();} + void freeze(){freeze_script_position_view();} void unfreeze(){clear_script_position_view();} + double source_position()const{return signed_position_size();} + double size50(double p)const{return calc_qty_for_type(p,50,static_cast(QtyType::PERCENT_OF_EQUITY));} + x::AccountEffectProjection flatten_quote(double p)const{return project_native_settlement_v1(x::Flatten{},x::Fill{p,"quote","",90,{}});} + double physical()const{return position_side_==PositionSide::SHORT?-position_qty_:position_qty_;} + double qty()const{return position_qty_;}double balance()const{return initial_capital_+net_profit_sum_;} + double marked(double p)const{return marked_equity(p);}int64_t cycle()const{return position_cycle_seq_;} + int64_t next_cycle()const{return next_position_cycle_seq_;}int slots()const{return position_entry_count_;} + void slots(int n){position_entry_count_=n;}void step(double q){qty_step_=q;} + void fee(double q){commission_value_=q;}void pyramid(int n){pyramiding_=n;} + void default_percent(double q){default_qty_type_=QtyType::PERCENT_OF_EQUITY;default_qty_value_=q;} + void financial_scale(double pv,double fx){syminfo_.pointvalue=pv;account_currency_fx_=fx;} + void slip(int n,double tick){slippage_=n;syminfo_mintick_=tick;syminfo_.mintick=tick;} + void block_opposite(bool held_long){risk_direction_=held_long?RiskDirection::LONG_ONLY:RiskDirection::SHORT_ONLY;} + auto& lots(){return pyramid_entries_;}const auto& lots()const{return pyramid_entries_;} + const auto& rows()const{return trades_;}const auto& actions()const{return stream_order_actions_;} + double ledger(const char* id)const{auto at=id_unclosed_qty_.find(id);return at==id_unclosed_qty_.end()?0:at->second;} + void retain_exit(){PendingOrder o;o.id="retained";o.from_entry="old";o.type=OrderType::EXIT;o.incarnation=999;o.created_seq=999;pending_orders_.push_back(std::move(o));} + size_t pending()const{return pending_orders_.size();}const PendingOrder* pending_data()const{return pending_orders_.data();} +}; + +void by_entry_fragments(double sign){ + scenario="caller distinct incarnation construction across fragments";Book b; + b.open(sign,100,11,"A");b.open(sign*2,100,11,"A");b.open(sign*3,100,22,"Other");b.open(sign,100,33,"A"); + const auto other=b.lots()[2];b.by_qty(100+sign*10,"A",2.5); + REQUIRE(b.rows().size()==2);CHECK(b.rows()[0].entry_incarnation==11&&b.rows()[1].entry_incarnation==11); + near(b.rows()[0].qty,1);near(b.rows()[1].qty,1.5); + near(b.rows()[0].commission,2.4);near(b.rows()[1].commission,3.6); + REQUIRE(b.lots().size()==3);near(b.lots()[0].qty,.5);CHECK(b.lots()[1].entry_incarnation==22); + CHECK(b.lots()[1].qty==other.qty&&b.lots()[1].price==other.price&&b.lots()[1].entry_commission_account==other.entry_commission_account); + b.by_all(100+sign*10,"A");REQUIRE(b.lots().size()==1);CHECK(b.lots()[0].entry_incarnation==22); + near(b.physical(),sign*3);CHECK(b.rows().size()==4); + near(b.rows()[2].commission,2);near(b.rows()[3].commission,4); +} + +void refused_provenance(bool zero,bool scratch){ + scenario=scratch?"caller scratch heterogeneous provenance":"caller bound-close invalid provenance";Book b; + b.open(1,100,zero?0:11,"A");b.open(1,100,zero?22:11,"Other"); + b.open(3,100,33,"untouched"); + if(scratch)b.lots()[0].market_pyramid_add=true; + const auto hash=b.broker_state_hash();const auto action_count=b.actions().size();bool threw=false; + try{if(scratch){PendingOrder o;o.from_entry="A";o.legs.set_stop_price(99);(void)b.scratch(o,110);} + else b.by_all(110,"A");}catch(const std::runtime_error& e){threw=true;CHECK(std::string(e.what()).size()>0);} + CHECK(threw);CHECK(b.broker_state_hash()==hash&&b.rows().empty()&&b.actions().size()==action_count); + near(b.qty(),5); +} + +void caller_dust_and_percent(double sign){ + scenario="caller interior dust must not become selected Flatten";Book b;b.fee(0); + b.open(sign,100,11,"A",6);b.open(sign*3,100,22,"B"); + const double amount=1.0-5e-12;b.by_qty(100+sign*10,"A",amount); + REQUIRE(b.lots().size()==2);CHECK(b.rows()[0].qty==amount); + CHECK(b.lots()[0].qty==1.0-amount&&b.lots()[0].qty>0); + CHECK(b.lots()[0].entry_commission_account==6.0-6.0*amount); + CHECK(b.lots()[1].entry_incarnation==22&&b.lots()[1].qty==3); + CHECK(b.qty()==b.lots()[0].qty+b.lots()[1].qty); + scenario="caller percent quantity floor leaves real remainder";Book p;p.fee(0);p.open(sign*5.4103,100,11,"A");p.step(.0001); + p.percent(100,50);REQUIRE(p.rows().size()==1&&p.lots().size()==1); + near(p.rows()[0].qty,2.7051);near(p.qty(),2.7052); +} + +void flips_and_sequential(double held_sign){ + scenario="F7 held plus new size versus F8 total quantity"; + const bool buy=held_sign<0;const double price=100+held_sign*10; + Book flip;flip.open(held_sign*3,100,11,"old");flip.retain_exit();const auto* pending=flip.pending_data(); + flip.flip(buy,price,1,-1,true);REQUIRE(flip.lots().size()==1&&flip.rows().size()==1); + near(flip.physical(),-held_sign);near(flip.rows()[0].qty,3);CHECK(flip.rows()[0].exit_price==price&&flip.lots()[0].price==price); + near(flip.rows()[0].commission+flip.lots()[0].entry_commission_account,6); + CHECK(flip.pending()==1&&flip.pending_data()==pending); + Book small;small.open(held_sign*3,100,11,"old");small.retain_exit();const auto* keep=small.pending_data(); + small.sequential(buy,price,1,-1);CHECK(small.lots().empty()&&small.cycle()==0&&small.rows().size()==1); + CHECK(small.pending()==1&&small.pending_data()==keep);near(small.rows()[0].commission,6); + Book cross;cross.open(held_sign*3,100,11,"old");cross.sequential(buy,price,4,-1); + REQUIRE(cross.lots().size()==1);near(cross.physical(),-held_sign);near(cross.rows()[0].qty,3); + near(cross.rows()[0].commission+cross.lots()[0].entry_commission_account,6); + Book close;close.open(held_sign*3,100,11,"old");close.flip(buy,price,99,-1,true,true); + CHECK(close.lots().empty()&&close.cycle()==0&&close.rows().size()==1); +} + +void projected_percent_flip(bool default_quantity){ + scenario=default_quantity?"F7 default percent consumes close-only quote":"F7 explicit percent consumes close-only quote"; + Book b;b.open(1,100,11,"old");b.open(3,100,22,"old");b.default_percent(50); + const auto hash=b.broker_state_hash();const auto q=b.flatten_quote(110); + REQUIRE(q.status==x::Status::Applied);CHECK(b.broker_state_hash()==hash); + near(q.realized_balance,1034);near(q.current_ticket,6);near(b.size50(110),4.5454545454545459); + b.flip(false,110,default_quantity?std::numeric_limits::quiet_NaN():50, + default_quantity?-1:static_cast(QtyType::PERCENT_OF_EQUITY)); + REQUIRE(b.lots().size()==1&&b.rows().size()==2); + near(b.physical(),-4.7000000000000002);near(b.rows()[0].commission,.68965517241379315); + near(b.rows()[1].commission,2.0689655172413794);near(b.lots()[0].entry_commission_account,3.2413793103448274); + near(b.balance(),1037.2413793103448);near(b.marked(110),1034); +} + +void frozen_quantity_provenance(double sign){ + scenario="F8 frozen off-grid quantity versus retained old wrapper";Book b;b.open(-sign*3,100,11,"old");b.step(1); + b.dispatch(sign>0,100,4.25,true,true);REQUIRE(b.lots().size()==1);CHECK(b.physical()==sign*1.25); + Book old;old.open(-sign*3,100,11,"old");old.step(1);old.sequential(sign>0,100,4.25,-1); + near(old.physical(),sign); // Original no-provenance wrapper intentionally floors4.25 to4. + scenario="F11 frozen same-side add versus retained old wrapper";Book add;add.open(sign*3,100,11,"old");add.step(1); + add.dispatch(sign>0,100,1.25,false,true);REQUIRE(add.lots().size()==2);CHECK(add.lots().back().qty==1.25); + CHECK(add.lots().back().market_pyramid_add);CHECK(add.lots().back().entry_incarnation==93); + CHECK(add.ledger("dispatch")==1.25);near(add.physical(),sign*4.25); + Book wrapper;wrapper.open(sign*3,100,11,"old");wrapper.step(1);wrapper.old_add(sign>0,100,1.25); + REQUIRE(wrapper.lots().size()==2);near(wrapper.lots().back().qty,1); + scenario="F7 cash quantity keeps price pointvalue and FX basis";Book cash;cash.financial_scale(2,2); + cash.open(-sign*3,100,11,"old");cash.flip(sign>0,100,1000,static_cast(QtyType::CASH)); + REQUIRE(cash.lots().size()==1);near(cash.physical(),sign*2.5); + CHECK(cash.lots()[0].price==100&&cash.rows()[0].exit_price==100); + near(cash.rows()[0].commission+cash.lots()[0].entry_commission_account,6); +} + +void raw_cycles_and_noeffect(double sign){ + scenario="RAW one cycle allocator and after-Applied stamps";Book b; + PendingOrder o;o.type=OrderType::RAW_ORDER;o.id="raw";o.is_long=sign>0;o.qty=1.25;o.incarnation=11; + o.created_position_side=PositionSide::FLAT;b.raw(o,100); + REQUIRE(b.lots().size()==1);CHECK(b.cycle()==1&&b.next_cycle()==2);CHECK(b.ledger("raw")==1.25); + PendingOrder add=o;add.id="raw-add";add.incarnation=22;add.created_position_side=sign>0?PositionSide::LONG:PositionSide::SHORT; + b.raw(add,100);REQUIRE(b.lots().size()==2);CHECK(b.cycle()==1&&b.next_cycle()==2); + CHECK(b.lots().back().market_pyramid_add&&b.lots().back().entry_incarnation==22); + CHECK(b.ledger("raw-add")==1.25);near(b.lots().back().entry_commission_account,6); + b.open(sign,100,33,"unrelated-native-lot");REQUIRE(!b.lots().back().market_pyramid_add); + const auto rows=b.rows().size(),actions=b.actions().size(),lots=b.lots().size();const auto last=b.lots().back(); + b.old_add(sign>0,100,0);CHECK(b.lots().size()==lots&&b.rows().size()==rows&&b.actions().size()==actions); + CHECK(b.lots().back().entry_incarnation==last.entry_incarnation&&b.lots().back().market_pyramid_add==last.market_pyramid_add); +} + +void source_slots_and_scratch(){ + scenario="helper-local bracket/script/margin slot restoration"; + for(Cause cause:{Bracket,Script,Margin}){Book b;b.open(1,100,11,"A");b.open(2,100,22,"B");b.slots(7); + b.partial(110,1,cause);CHECK(b.lots().size()==1);CHECK(b.slots()==(cause==Bracket?7:1));} + scenario="R20 release remains after bracket slot restoration";Book unique;unique.pyramid(2); + unique.bar(0,100);unique.open(1,100,11,"A");unique.bar(1,100);unique.open(1,100,22,"B");unique.bar(3,110); + PendingOrder o;o.type=OrderType::EXIT;o.id="X";o.from_entry="A";o.qty=1;o.qty_percent=50;o.incarnation=99;o.created_seq=99; + o.created_bar=2;o.created_position_side=PositionSide::LONG;o.created_position_cycle_seq=unique.cycle(); + o.quantity_request.request(QuantityIntent::units(1));o.quantity_request.reserve(1,2);o.legs.set_limit_price(110); + unique.exit(o,110);REQUIRE(unique.lots().size()==1);CHECK(unique.lots()[0].entry_incarnation==22&&unique.slots()==1); + scenario="KI62 second one-ticket fill after primary bracket";Book scratch; + scratch.bar(1,100);scratch.open(1,100,55,"A");scratch.bar(3,110); + scratch.open(1,100,11,"A");scratch.open(2,100,11,"A");scratch.open(4,100,22,"Other"); + scratch.lots()[1].market_pyramid_add=true;scratch.lots()[2].market_pyramid_add=true;scratch.slots(5); + PendingOrder bracket;bracket.type=OrderType::EXIT;bracket.id="scratch-bracket";bracket.from_entry="A"; + bracket.qty=1;bracket.qty_percent=12.5;bracket.incarnation=100;bracket.created_seq=100; + bracket.quantity_request.request(QuantityIntent::units(1));bracket.quantity_request.reserve(1,8); + bracket.legs.set_limit_price(110); + scratch.exit(bracket,110);REQUIRE(scratch.rows().size()==3&&scratch.lots().size()==1); + CHECK(scratch.rows()[0].entry_incarnation==55&&scratch.rows()[1].entry_incarnation==11&&scratch.rows()[2].entry_incarnation==11); + near(scratch.rows()[0].commission,6);near(scratch.rows()[1].commission,2);near(scratch.rows()[2].commission,4); + CHECK(scratch.rows()[1].entry_bar_index==scratch.rows()[1].exit_bar_index); + CHECK(scratch.rows()[2].entry_bar_index==scratch.rows()[2].exit_bar_index); + CHECK(scratch.slots()==5&&scratch.lots()[0].entry_incarnation==22); + const auto hash=scratch.broker_state_hash();CHECK(scratch.scratch(bracket,110,Bracket)==0);CHECK(scratch.broker_state_hash()==hash); +} + +void direction_blocked_one_slip(double sign){ + scenario="direction-blocked F1 resolves slippage once";Book b;b.open(sign,100,11,"old");b.slip(2,.25);b.block_opposite(sign>0); + b.dispatch(sign<0,100,1,false,false);REQUIRE(b.rows().size()==1&&b.lots().empty()); + CHECK(b.rows()[0].exit_price==100-sign*.5);near(b.rows()[0].pnl,-6.5); +} + +void nonphysical_observations(){ + scenario="range-end and frozen source view do not settle exposure";Book b;b.open(1,100,11,"A",2);b.open(3,100,22,"B",6); + const auto lots=b.lots();const auto rows=b.rows().size(),actions=b.actions().size();const auto cycle=b.cycle(),next=b.next_cycle(); + const double balance=b.balance();b.freeze();CHECK(b.source_position()==4);CHECK(b.physical()==4);b.unfreeze(); + b.bar(4,110);b.range();CHECK(b.rows().size()==rows&&b.actions().size()==actions); + CHECK(b.report_trade_count()==2&&b.get_report_trade(0).open_at_end&&b.get_report_trade(1).open_at_end); + CHECK(b.cycle()==cycle&&b.next_cycle()==next&&b.balance()==balance&&b.lots().size()==lots.size()); + for(size_t i=0;ivoid run(F fn){try{fn();}catch(const Abort&){}catch(const std::exception& e){++failures;std::printf("FAIL %s exception %s\n",scenario,e.what());}} +} +int main(){ + for(double sign:{1.0,-1.0}){run([&]{by_entry_fragments(sign);});run([&]{caller_dust_and_percent(sign);}); + run([&]{flips_and_sequential(sign);});run([&]{frozen_quantity_provenance(sign);}); + run([&]{raw_cycles_and_noeffect(sign);});run([&]{direction_blocked_one_slip(sign);});} + run([]{refused_provenance(true,false);});run([]{refused_provenance(false,false);});run([]{refused_provenance(false,true);}); + run([]{projected_percent_flip(false);});run([]{projected_percent_flip(true);});run(source_slots_and_scratch);run(nonphysical_observations); + std::printf("%s native settlement callers: %d checks %d failures\n",failures?"FAIL":"PASS",checks,failures); + return failures?1:0; +} diff --git a/tests/test_native_settlement_projection.cpp b/tests/test_native_settlement_projection.cpp new file mode 100644 index 000000000..d3f86d4c7 --- /dev/null +++ b/tests/test_native_settlement_projection.cpp @@ -0,0 +1,233 @@ +// Non-applying R3 account projection against the actual allocation/settlement owner. +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge; +namespace x=pineforge::execution; +namespace { +int checks=0,failures=0; +const char* scenario="setup"; +struct Abort{}; +#define CHECK(v) do{++checks;if(!(v)){++failures;std::printf("FAIL %s:%d %s\n",scenario,__LINE__,#v);}}while(0) +#define REQUIRE(v) do{bool ok_=bool(v);CHECK(ok_);if(!ok_)throw Abort{};}while(0) +void near(double a,double b){bool ok=std::isfinite(a)&&std::isfinite(b)&&std::abs(a-b)<=1e-12*std::max(1.0,std::abs(b)); + if(!ok)std::printf(" actual=%.17g expected=%.17g\n",a,b);CHECK(ok);} +struct Book final:BacktestEngine{ + Book(){initial_capital_=1000;commission_type_=CommissionType::CASH_PER_ORDER;commission_value_=6; + syminfo_.pointvalue=1;account_currency_fx_=1;stream_observe_actions_=true; + current_bar_={100,100,100,100,1,1736121600000LL};bar_index_=3;} + void on_bar(const Bar&)override{} + x::PhysicalExecutionContext context()const{return {current_bar_.timestamp,bar_index_,{}, {}};} + x::Result settle(const x::Action& a,const x::Fill& f){return settle_native_execution_at(a,f,context());} + x::Result settle(const x::Action& a,const x::Fill& f,const x::SelectedOpeningSet& s){return settle_native_execution_selected_at(a,f,context(),s);} + x::AccountEffectProjection project(const x::Action& a,const x::Fill& f)const{return project_native_settlement_v1(a,f);} + x::AccountEffectProjection project(const x::Action& a,const x::Fill& f,x::CloseScope s)const{return project_native_settlement_scoped_v1(a,f,s);} + x::AccountEffectProjection project(const x::Action& a,const x::Fill& f,const x::SelectedOpeningSet& s)const{return project_native_settlement_selected_v1(a,f,s);} + x::SettlementInspection inspect(const x::Action& a,const x::Fill& f)const{return inspect_native_settlement(a,f);} + void open(double q,double p,uint64_t inc,double paid=0){REQUIRE(settle(order_action::Transact{q},x::Fill{p,"seed","",inc,paid}).status==x::Status::Applied);} + double balance()const{return initial_capital_+net_profit_sum_;} + double marked(double p)const{return marked_equity(p);} + double paid()const{double v=0;for(const auto& lot:pyramid_entries_)v+=lot.entry_commission_account;return v;} + double signed_qty()const{return position_side_==PositionSide::SHORT?-position_qty_:position_qty_;} + double net()const{return net_profit_sum_;} + int64_t cycle()const{return position_cycle_seq_;} + int64_t next_cycle()const{return next_position_cycle_seq_;} + uint64_t next_order()const{return next_order_incarnation_;} + size_t lots()const{return pyramid_entries_.size();} + size_t rows()const{return trades_.size();} + size_t actions()const{return stream_order_actions_.size();} + const Trade& row(size_t n)const{return trades_[n];} + void initial(double value){initial_capital_=value;} + void cash_fee(double value){commission_value_=value;} + void schedule(CommissionType kind,double value){commission_type_=kind;commission_value_=value;} + void force_next_cycle(int64_t value){next_position_cycle_seq_=value;} + void account_boundary(double initial,double net){initial_capital_=initial;net_profit_sum_=net;} +}; +x::Fill fill(double price,uint64_t inc=90,std::optional fee={}){return {price,"projection","",inc,fee};} +struct Snapshot{ + uint64_t broker,stream,next_order;int64_t cycle,next_cycle;size_t lots,rows,actions; + explicit Snapshot(const Book& b):broker(b.broker_state_hash()),stream(b.stream_state_hash()),next_order(b.next_order()), + cycle(b.cycle()),next_cycle(b.next_cycle()),lots(b.lots()),rows(b.rows()),actions(b.actions()){} + void unchanged(const Book& b)const{CHECK(b.broker_state_hash()==broker&&b.stream_state_hash()==stream); + CHECK(b.next_order()==next_order&&b.cycle()==cycle&&b.next_cycle()==next_cycle); + CHECK(b.lots()==lots&&b.rows()==rows&&b.actions()==actions);} +}; +void quote_facts(const x::AccountEffectProjection& q,x::Status status,double closed,double opened, + double abs_units,size_t lots,double notional,double ticket,double balance, + double paid,double marked,int64_t cycle,double signed_units){ + CHECK(q.status==status);near(q.closed_units,closed);near(q.opened_units,opened); + near(q.resulting_abs_units,abs_units);CHECK(q.resulting_lot_count==lots);near(q.resulting_abs_notional,notional); + near(q.current_ticket,ticket);near(q.realized_balance,balance);near(q.remaining_entry_cost,paid); + near(q.marked_equity,marked);CHECK(q.cycle_after==cycle);near(q.signed_units_after,signed_units); + CHECK(q.would_open==(opened!=0));CHECK(q.incoming_short==(status==x::Status::Applied&&opened<0)); +} +void matches_commit(const Book& b,const x::AccountEffectProjection& q,const x::Result& r,double mark){ + CHECK(r.status==q.status);near(r.closed_units,q.closed_units);near(r.opened_units,q.opened_units); + near(r.current_ticket,q.current_ticket);near(b.balance(),q.realized_balance);near(b.marked(mark),q.marked_equity); + near(b.paid(),q.remaining_entry_cost);near(b.signed_qty(),q.signed_units_after); + CHECK(b.lots()==q.resulting_lot_count&&b.cycle()==q.cycle_after); +} + +void opening_add_flip(double sign){ + scenario="projection flat opening cost once";Book flat; + const auto f=fill(100);const Snapshot before(flat); + const auto q=flat.project(order_action::Transact{sign*2},f);before.unchanged(flat); + quote_facts(q,x::Status::Applied,0,sign*2,2,1,200,6,1000,6,994,1,sign*2); + matches_commit(flat,q,flat.settle(order_action::Transact{sign*2},f),100);CHECK(flat.next_cycle()==2); + + scenario="projection same-side add cycle and paid costs";Book add;add.open(sign,100,11,2); + const double price=100+sign*10;const Snapshot add_before(add); + const auto a=add.project(order_action::Transact{sign*3},fill(price));add_before.unchanged(add); + quote_facts(a,x::Status::Applied,0,sign*3,4,2,4*price,6,1000,8,1002,1,sign*4); + matches_commit(add,a,add.settle(order_action::Transact{sign*3},fill(price)),price);CHECK(add.next_cycle()==2); + + scenario="projection crossing cost allocation and fresh cycle";Book flip;flip.open(-sign,100,11,2); + const double crossed=100+sign*10;const Snapshot flip_before(flip); + const auto p=flip.project(order_action::Transact{sign*3},fill(crossed));flip_before.unchanged(flip); + quote_facts(p,x::Status::Applied,1,sign*2,2,1,2*crossed,6,986,4,982,2,sign*2); + matches_commit(flip,p,flip.settle(order_action::Transact{sign*3},fill(crossed)),crossed);CHECK(flip.next_cycle()==3); +} + +void partial_and_selected_flatten(double sign){ + scenario="projection partial Book close";Book b;b.open(sign,100,11,2);b.open(sign*3,100+sign*10,22,6); + const double price=100+sign*20;const Snapshot before(b); + const auto q=b.project(order_action::Reduce{1},fill(price));before.unchanged(b); + quote_facts(q,x::Status::Applied,1,0,3,1,3*price,6,1012,6,1036,1,sign*3); + matches_commit(b,q,b.settle(order_action::Reduce{1},fill(price)),price); + + scenario="projection selected Flatten leaves sibling";Book selected;selected.open(sign,100,11,2);selected.open(sign*3,100+sign*10,22,6); + const x::SelectedOpeningSet one{selected.cycle(),{22}};const Snapshot original(selected); + const auto set=selected.project(x::Flatten{},fill(price),one); + const auto singleton=selected.project(x::Flatten{},fill(price),x::OpeningExposure{22,selected.cycle()});original.unchanged(selected); + quote_facts(set,x::Status::Applied,3,0,1,1,price,6,1018,2,1036,1,sign); + CHECK(singleton.status==set.status);near(singleton.realized_balance,set.realized_balance); + near(singleton.marked_equity,set.marked_equity);near(singleton.remaining_entry_cost,set.remaining_entry_cost); + matches_commit(selected,set,selected.settle(x::Flatten{},fill(price),one),price); + + scenario="projection selected Flatten entire roster";Book all;all.open(sign,100,11,2);all.open(sign*3,100+sign*10,22,6); + const x::SelectedOpeningSet both{all.cycle(),{22,11}};const Snapshot all_before(all); + const auto p=all.project(x::Flatten{},fill(price),both);all_before.unchanged(all); + quote_facts(p,x::Status::Applied,4,0,0,0,0,6,1036,0,1036,0,0); + matches_commit(all,p,all.settle(x::Flatten{},fill(price),both),price); +} + +void invalid_quote(const x::AccountEffectProjection& p,x::Status status){ + CHECK(p.status==status);CHECK(p.closed_units==0&&p.opened_units==0&&p.resulting_abs_units==0); + CHECK(p.resulting_lot_count==0&&p.resulting_abs_notional==0&&p.current_ticket==0); + CHECK(!p.would_open&&!p.incoming_short);CHECK(p.realized_balance==0&&p.remaining_entry_cost==0&&p.marked_equity==0); + CHECK(p.cycle_after==0&&p.signed_units_after==0); +} +void invalid_inspect(const x::SettlementInspection& p,x::Status status){ + CHECK(p.status==status);CHECK(p.closed_units==0&&p.opened_units==0&&p.resulting_abs_units==0); + CHECK(p.resulting_lot_count==0&&p.resulting_abs_notional==0&&p.current_ticket==0); + CHECK(!p.would_open&&!p.incoming_short); +} +void invalid_result(const x::Result& r,x::Status status){ + CHECK(r.status==status);CHECK(r.closed_units==0&&r.opened_units==0&&r.current_ticket==0); + CHECK(r.first_trade_index==0&&r.closed_trade_count==0&&r.opened_lot_incarnation==0); +} +void noeffect_and_invalid(double sign){ + scenario="projection valid NoEffect live and empty";Book b;b.open(sign*2,100,11,6); + const double mark=100+sign*10;const Snapshot before(b); + const auto p=b.project(order_action::Reduce{0},fill(mark));before.unchanged(b); + quote_facts(p,x::Status::NoEffect,0,0,2,1,2*mark,0,1000,6,1014,1,sign*2); + CHECK(b.project(order_action::Reduce{0},fill(mark,90,0)).status==x::Status::NoEffect); + for(double fee:{1.0,-1.0})invalid_quote(b.project(order_action::Reduce{0},fill(mark,90,fee)),x::Status::InvalidAccounting); + invalid_quote(b.project(order_action::Reduce{-1},fill(mark)),x::Status::InvalidQuantity); + invalid_quote(b.project(x::Flatten{},fill(std::numeric_limits::quiet_NaN())),x::Status::InvalidPrice); + invalid_quote(b.project(x::Flatten{},fill(mark),x::SelectedOpeningSet{b.cycle(),{999}}),x::Status::InvalidCloseTarget); + before.unchanged(b); + Book empty;const Snapshot flat(empty);const auto e=empty.project(x::Flatten{},fill(100));flat.unchanged(empty); + quote_facts(e,x::Status::NoEffect,0,0,0,0,0,0,1000,0,1000,0,0); + scenario="projection added account overflow is not inspection status"; + Book huge;huge.account_boundary(std::numeric_limits::max(),std::numeric_limits::max()); + const Snapshot boundary(huge);CHECK(huge.inspect(order_action::Reduce{0},fill(100)).status==x::Status::NoEffect); + invalid_quote(huge.project(order_action::Reduce{0},fill(100)),x::Status::InvalidAccounting);boundary.unchanged(huge); +} + +void explicit_ticket_projection(double ticket){ + scenario="projection explicit ticket waiver or rebate";Book b;b.open(1,100,11);b.open(3,100,22); + const Snapshot before(b);const auto p=b.project(x::Flatten{},fill(110,90,ticket));before.unchanged(b); + quote_facts(p,x::Status::Applied,4,0,0,0,0,ticket,1040-ticket,0,1040-ticket,0,0); + matches_commit(b,p,b.settle(x::Flatten{},fill(110,90,ticket)),110); +} + +void accumulation_order(bool paid_cost){ + scenario=paid_cost?"projection per-lot marked-equity association":"projection exact commit-order balance"; + Book b;b.initial(1);b.cash_fee(0);const double high=10000000000000100.0; + b.open(1,high,1);REQUIRE(b.settle(x::Flatten{},fill(100,90,0)).status==x::Status::Applied); + CHECK(b.net()==-1e16); + b.open(1,100,2,paid_cost?1:0);b.open(.5,high-2,3); + const Snapshot before(b); + const auto unchanged=b.project(order_action::Reduce{0},fill(high,90,0));before.unchanged(b); + CHECK(unchanged.status==x::Status::NoEffect);CHECK(unchanged.realized_balance==-1e16); + CHECK(unchanged.marked_equity==1);CHECK(unchanged.remaining_entry_cost==(paid_cost?1:0)); + CHECK(unchanged.marked_equity==b.marked(high)); + const auto p=b.project(x::Flatten{},fill(high,90,0));before.unchanged(b); + CHECK(p.status==x::Status::Applied&&p.realized_balance==2&&p.marked_equity==2); + const size_t first=b.rows();const auto r=b.settle(x::Flatten{},fill(high,90,0)); + REQUIRE(r.status==x::Status::Applied&&b.rows()==first+2); + CHECK(b.row(first).pnl==1e16&&b.row(first+1).pnl==1); + CHECK(b.balance()==2&&b.net()==1);matches_commit(b,p,r,high); +} + +void quote_is_not_authority(){ + scenario="projection selection is revalidated at commit";Book b;b.open(1,100,11);b.open(3,110,22); + const x::SelectedOpeningSet selected{b.cycle(),{11}};const Snapshot before(b); + const auto quote=b.project(x::Flatten{},fill(120),selected);REQUIRE(quote.status==x::Status::Applied);before.unchanged(b); + REQUIRE(b.settle(x::Flatten{},fill(120),selected).status==x::Status::Applied); + const Snapshot changed(b);CHECK(b.settle(x::Flatten{},fill(120),selected).status==x::Status::InvalidCloseTarget);changed.unchanged(b); + CHECK(b.lots()==1&&b.signed_qty()==3); + Book smaller;smaller.open(3,100,11);const x::SelectedOpeningSet live{smaller.cycle(),{11}}; + const auto old=smaller.project(x::Flatten{},fill(110),live);CHECK(old.closed_units==3);near(old.realized_balance,1024); + REQUIRE(smaller.settle(order_action::Reduce{1},fill(110)).status==x::Status::Applied); + const auto fresh=smaller.settle(x::Flatten{},fill(110),live);REQUIRE(fresh.status==x::Status::Applied); + CHECK(fresh.closed_units==2&&fresh.closed_units!=old.closed_units);near(smaller.balance(),1018); +} + +void finite_row_infinite_ticket(){ + // Three qty-1 lots @0 with paid 0; per-contract DBL_MAX/2 Flatten at DBL_MAX/2 + // keeps each row PnL finite 0 while 3*(DBL_MAX/2) overflows the current ticket. + scenario="finite row fee cannot apply infinite ticket"; + Book b;b.open(1,0,11,0);b.open(1,0,22,0);b.open(1,0,33,0); + REQUIRE(b.lots()==3&&b.signed_qty()==3&&b.paid()==0&&b.rows()==0&&b.net()==0); + const auto live=b.inspect(x::Flatten{},fill(0)); + REQUIRE(live.status==x::Status::Applied);near(live.current_ticket,6);CHECK(live.closed_units==3); + b.schedule(CommissionType::CASH_PER_CONTRACT,std::numeric_limits::max()/2.0); + const double price=std::numeric_limits::max()/2.0;const Snapshot before(b); + invalid_inspect(b.inspect(x::Flatten{},fill(price)),x::Status::InvalidAccounting);before.unchanged(b); + invalid_quote(b.project(x::Flatten{},fill(price)),x::Status::InvalidAccounting);before.unchanged(b); + invalid_result(b.settle(x::Flatten{},fill(price)),x::Status::InvalidAccounting);before.unchanged(b); + CHECK(b.lots()==3&&b.rows()==0&&b.actions()==before.actions&&b.signed_qty()==3&&b.paid()==0&&b.net()==0); +} + +void cycle_peek_exhaustion(){ + scenario="projection cycle exhaustion without consumption"; + for(int64_t next:{int64_t{0},int64_t{-1},std::numeric_limits::max()}){ + Book flat;flat.force_next_cycle(next);const Snapshot before(flat);bool threw=false; + try{(void)flat.project(order_action::Transact{1},fill(100));}catch(const std::overflow_error&){threw=true;} + CHECK(threw);before.unchanged(flat); + } + Book live;live.open(1,100,11);live.force_next_cycle(std::numeric_limits::max());const Snapshot existing(live); + const auto add=live.project(order_action::Transact{1},fill(100)); + CHECK(add.status==x::Status::Applied&&add.cycle_after==live.cycle()); + const auto zero=live.project(order_action::Reduce{0},fill(100));CHECK(zero.status==x::Status::NoEffect); + existing.unchanged(live); +} +templatevoid run(F fn){try{fn();}catch(const Abort&){}catch(const std::exception& e){++failures;std::printf("FAIL %s exception %s\n",scenario,e.what());}} +} +int main(){ + for(double sign:{1.0,-1.0}){run([&]{opening_add_flip(sign);});run([&]{partial_and_selected_flatten(sign);});run([&]{noeffect_and_invalid(sign);});} + run([]{explicit_ticket_projection(0);});run([]{explicit_ticket_projection(-6);}); + run([]{accumulation_order(false);});run([]{accumulation_order(true);});run(quote_is_not_authority); + run(finite_row_infinite_ticket);run(cycle_peek_exhaustion); + std::printf("%s native settlement projection: %d checks %d failures\n",failures?"FAIL":"PASS",checks,failures); + return failures?1:0; +} diff --git a/tests/test_source_exact_reversal.cpp b/tests/test_source_exact_reversal.cpp new file mode 100644 index 000000000..a28863dd4 --- /dev/null +++ b/tests/test_source_exact_reversal.cpp @@ -0,0 +1,275 @@ +// Literal calls to the real F7 and F8 adapters. No run(), tape or strategy loop. +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge; +namespace x = pineforge::execution; +namespace { +int checks = 0, failures = 0; +const char* scenario = "setup"; +struct Abort {}; +#define CHECK(value) do { ++checks; if (!(value)) { ++failures; \ + std::printf("FAIL %s:%d %s\n", scenario, __LINE__, #value); } } while (0) +#define REQUIRE(value) do { const bool ok_ = bool(value); CHECK(ok_); \ + if (!ok_) throw Abort{}; } while (0) + +template struct Access { + friend auto access(Tag) { return Member; } +}; +struct FlipTag { friend auto access(FlipTag); }; +struct SequentialTag { friend auto access(SequentialTag); }; +template struct Access; +template struct Access; + +uint64_t bits(double value) { + uint64_t result; + static_assert(sizeof(result) == sizeof(value)); + std::memcpy(&result, &value, sizeof(result)); + return result; +} +void exact(double actual, double expected) { + if (bits(actual) != bits(expected)) + std::printf(" actual=%.17g (%016llx) expected=%.17g (%016llx)\n", + actual, static_cast(bits(actual)), + expected, static_cast(bits(expected))); + CHECK(bits(actual) == bits(expected)); +} +void near(double actual, double expected) { + const bool ok = std::isfinite(actual) && std::isfinite(expected) + && std::abs(actual - expected) <= 1e-12 * std::max(1.0, std::abs(expected)); + if (!ok) std::printf(" actual=%.17g expected=%.17g\n", actual, expected); + CHECK(ok); +} + +struct Book final : BacktestEngine { + Book() { + initial_capital_ = 1000; + commission_type_ = CommissionType::CASH_PER_ORDER; + commission_value_ = 0; + slippage_ = 0; + qty_step_ = 0; + pyramiding_ = 100; + syminfo_.pointvalue = 1; + syminfo_.mintick = .01; + syminfo_mintick_ = .01; + account_currency_fx_ = 1; + stream_observe_actions_ = true; + current_bar_ = {100, 130, 70, 110, 1, 1736121660000LL}; + bar_index_ = 7; + } + void on_bar(const Bar&) override {} + void open(double quantity, double price, uint64_t incarnation) { + const x::PhysicalExecutionContext context{1736121600000LL, 6, {}, {}}; + REQUIRE(settle_native_execution_at(order_action::Transact{quantity}, + x::Fill{price, "old", "historical", incarnation, 0}, context).status == x::Status::Applied); + } + void flip(bool buy, double price, double quantity, int type = -1, + bool frozen = true, bool close_only = false) { + (this->*access(FlipTag{}))(std::string("flip"), buy, price, quantity, + type, frozen, close_only, 90); + } + void sequential(bool buy, double price, double transaction) { + (this->*access(SequentialTag{}))(std::string("sequential"), buy, price, + transaction, -1, 91); + } + void fee(double value) { commission_value_ = value; } + void step(double value) { qty_step_ = value; } + void scale(double pointvalue, double fx) { + syminfo_.pointvalue = pointvalue; + account_currency_fx_ = fx; + } + void default_percent(double value) { + default_qty_type_ = QtyType::PERCENT_OF_EQUITY; + default_qty_value_ = value; + } + void already_resolved_slippage() { slippage_ = 99; } + void retain_exit() { + PendingOrder order; + order.id = "retained"; + order.from_entry = "old"; + order.type = OrderType::EXIT; + order.incarnation = 700; + order.created_seq = 700; + order.legs.attach(order.incarnation, position_cycle_seq_); + pending_orders_.push_back(std::move(order)); + } + const auto& lots() const { return pyramid_entries_; } + const auto& rows() const { return trades_; } + const auto& pending() const { return pending_orders_; } + double quantity() const { + return position_side_ == PositionSide::SHORT ? -position_qty_ : position_qty_; + } + double balance() const { return initial_capital_ + net_profit_sum_; } + double marked(double price) const { return marked_equity(price); } + int64_t cycle() const { return position_cycle_seq_; } + int64_t next_cycle() const { return next_position_cycle_seq_; } + size_t actions() const { return stream_order_actions_.size(); } +}; + +void exact_f7_and_unchanged_f8(double held_sign) { + scenario = "F7 preserves exact requested target; F8 preserves transaction remainder"; + const bool buy = held_sign < 0; + const double price = 100 + held_sign * 10; + Book flip; + flip.open(held_sign, 100, 11); + flip.retain_exit(); + const auto* pending = flip.pending().data(); + const auto next_cycle = flip.next_cycle(); + const auto actions = flip.actions(); + flip.already_resolved_slippage(); + flip.step(1); // Frozen source quantity must not be floored a second time. + flip.flip(buy, price, .1); + REQUIRE(flip.lots().size() == 1 && flip.rows().size() == 1); + CHECK(bits(flip.lots()[0].qty) == UINT64_C(0x3fb999999999999a)); + exact(flip.quantity(), -held_sign * .1); + exact(flip.rows()[0].qty, 1); + near(flip.rows()[0].pnl, 10); + exact(flip.rows()[0].exit_price, price); + exact(flip.lots()[0].price, price); + CHECK(flip.rows()[0].entry_incarnation == 11); + CHECK(flip.lots()[0].entry_incarnation == 90 && flip.lots()[0].entry_id == "flip"); + CHECK(flip.rows()[0].exit_id == "flip"); + CHECK(flip.rows()[0].exit_time == 1736121660000LL && flip.rows()[0].exit_bar_index == 7); + CHECK(flip.lots()[0].time == 1736121660000LL && flip.lots()[0].entry_bar_index == 7); + CHECK(flip.cycle() == next_cycle && flip.next_cycle() == next_cycle + 1); + CHECK(flip.actions() == actions + 2); + REQUIRE(flip.pending().size() == 1); + CHECK(flip.pending().data() == pending && flip.pending()[0].incarnation == 700); + CHECK(flip.pending()[0].legs.target().owner == flip.cycle()); + + Book class_c; + class_c.open(held_sign, 100, 11); + class_c.sequential(buy, price, 1.1); + REQUIRE(class_c.lots().size() == 1 && class_c.rows().size() == 1); + // These are the old and current F8 witness bits, not exact source Q bits. + CHECK(bits(class_c.lots()[0].qty) == UINT64_C(0x3fb99999999999a0)); + CHECK(bits(class_c.lots()[0].qty) != bits(.1)); + exact(class_c.rows()[0].qty, 1); + near(class_c.rows()[0].pnl, 10); + + Book class_b; + class_b.open(held_sign, 100, 11); + class_b.sequential(buy, price, .1); + REQUIRE(class_b.lots().empty() && class_b.rows().size() == 1); + exact(class_b.rows()[0].qty, 1); + CHECK(class_b.cycle() == 0); +} + +void f7_non_dyadic_roster(double sign) { + scenario = "F7 closes non-dyadic roster and opens exact target with one ticket"; + Book book; + book.open(sign * .1, 100, 11); + book.open(sign * .2, 100, 12); + book.open(sign * .3, 100, 13); + book.fee(6); + book.flip(sign < 0, 100, .1); + REQUIRE(book.rows().size() == 3 && book.lots().size() == 1); + exact(book.lots()[0].qty, .1); + const double quantities[] = {.1, .2, .3}; + double paid = book.lots()[0].entry_commission_account; + for (size_t index = 0; index < 3; ++index) { + CHECK(book.rows()[index].entry_incarnation == 11 + index); + exact(book.rows()[index].qty, quantities[index]); + paid += book.rows()[index].commission; + } + near(paid, 6); + near(book.marked(100), 994); +} + +void f7_absorbed_quantities(double sign, bool tiny_target) { + scenario = tiny_target ? "F7 accepts tiny target absorbed by old held quantity" + : "F7 accepts large target that absorbs old held quantity"; + const double held = tiny_target ? 1e16 : .1; + const double quantity = tiny_target ? .1 : 1e16; + Book book; + book.open(sign * held, 1, 11); + book.flip(sign < 0, 1, quantity); + REQUIRE(book.lots().size() == 1 && book.rows().size() == 1); + exact(book.lots()[0].qty, quantity); + exact(book.rows()[0].qty, held); +} + +void f7_zero_and_close_only(double sign) { + for (bool close_only : {false, true}) { + scenario = close_only ? "F7 close-only stays Flatten" : "F7 resolved zero stays Flatten"; + Book book; + book.open(sign, 100, 11); + book.retain_exit(); + const auto* pending = book.pending().data(); + const auto next_cycle = book.next_cycle(); + book.fee(6); + book.flip(sign < 0, 100 + sign * 10, close_only ? 99 : 0, -1, true, close_only); + REQUIRE(book.rows().size() == 1 && book.lots().empty()); + exact(book.rows()[0].qty, 1); + near(book.rows()[0].commission, 6); + CHECK(book.cycle() == 0 && book.next_cycle() == next_cycle); + REQUIRE(book.pending().size() == 1); + CHECK(book.pending().data() == pending && book.pending()[0].legs.target().owner == 0); + } +} + +void f7_cash_size(double sign) { + scenario = "F7 cash sizing preserves pointvalue and current FX conversion"; + Book book; + book.scale(2, 2); + book.open(sign * 3, 100, 11); + book.fee(6); + book.flip(sign < 0, 100, 1000, static_cast(QtyType::CASH), false); + REQUIRE(book.lots().size() == 1 && book.rows().size() == 1); + exact(book.quantity(), -sign * 2.5); + near(book.rows()[0].commission + book.lots()[0].entry_commission_account, 6); + near(book.marked(100), 994); +} + +void f7_projected_percent(bool use_default) { + scenario = use_default ? "F7 default percent keeps Flatten-projected sizing" + : "F7 explicit percent keeps Flatten-projected sizing"; + Book book; + book.open(1, 100, 11); + book.open(3, 100, 12); + book.fee(6); + book.default_percent(50); + book.flip(false, 110, use_default ? std::numeric_limits::quiet_NaN() : 50, + use_default ? -1 : static_cast(QtyType::PERCENT_OF_EQUITY), false); + REQUIRE(book.lots().size() == 1 && book.rows().size() == 2); + // Closing 4 @110 first quotes a balance of 1034 for sizing. Half at110 is4.7. + exact(book.lots()[0].qty, 4.7000000000000002); + near(book.rows()[0].commission, .68965517241379315); + near(book.rows()[1].commission, 2.0689655172413794); + near(book.lots()[0].entry_commission_account, 3.2413793103448274); + near(book.balance(), 1037.2413793103448); + near(book.marked(110), 1034); +} + +template void run(F call) { + try { call(); } + catch (const Abort&) {} + catch (const std::exception& error) { + ++failures; + std::printf("FAIL %s exception: %s\n", scenario, error.what()); + } +} +} // namespace + +int main() { + for (double sign : {1.0, -1.0}) { + run([&] { exact_f7_and_unchanged_f8(sign); }); + run([&] { f7_non_dyadic_roster(sign); }); + run([&] { f7_absorbed_quantities(sign, true); }); + run([&] { f7_absorbed_quantities(sign, false); }); + run([&] { f7_zero_and_close_only(sign); }); + run([&] { f7_cash_size(sign); }); + } + run([] { f7_projected_percent(false); }); + run([] { f7_projected_percent(true); }); + std::printf("%s source exact reversal: %d checks, %d failures\n", + failures ? "FAIL" : "PASS", checks, failures); + return failures ? 1 : 0; +} diff --git a/tests/test_source_fifo_endpoints.cpp b/tests/test_source_fifo_endpoints.cpp new file mode 100644 index 000000000..1a096394f --- /dev/null +++ b/tests/test_source_fifo_endpoints.cpp @@ -0,0 +1,509 @@ +// Literal source FIFO endpoint calls paired with the actual native Reduce owner. +// No BacktestEngine::run(), generated strategy, tape, corpus or grading loop. +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge; +namespace x = pineforge::execution; + +namespace { +int checks = 0, failures = 0; +const char* scenario = "setup"; +struct Abort {}; +#define CHECK(value) do { ++checks; if (!(value)) { ++failures; \ + std::printf("FAIL %s:%d %s\n", scenario, __LINE__, #value); } } while (0) +#define REQUIRE(value) do { ++checks; if (!(value)) { ++failures; \ + std::printf("FAIL %s:%d %s\n", scenario, __LINE__, #value); throw Abort{}; } } while (0) + +template struct Access { friend auto access(Tag) { return Member; } }; +#define ACCESS(Tag, Member) struct Tag { friend auto access(Tag); }; \ + template struct Access +ACCESS(Partial, execute_partial_exit_qty); +ACCESS(ByQuantity, execute_partial_exit_by_entry_qty); +ACCESS(Drain, fifo_drain); +ACCESS(ComputeClose, compute_close_target_qty); +ACCESS(ImmediateClose, execute_immediate_close); +ACCESS(ExitFill, apply_exit_order_fill); +#undef ACCESS +template struct Args; +template struct Args { using tuple = std::tuple; }; +using Cause = std::tuple_element_t<2, typename Args::tuple>; +constexpr Cause Script = static_cast(0), Bracket = static_cast(1), Margin = static_cast(2); + +uint64_t bits(double value) { + uint64_t result; + static_assert(sizeof(result) == sizeof(value)); + std::memcpy(&result, &value, sizeof(result)); + return result; +} +void exact(double actual, double expected) { + if (bits(actual) != bits(expected)) + std::printf(" actual=%.17g (%016llx) expected=%.17g (%016llx)\n", + actual, static_cast(bits(actual)), + expected, static_cast(bits(expected))); + CHECK(bits(actual) == bits(expected)); +} +void near(double actual, double expected) { + const bool ok = std::isfinite(actual) && std::isfinite(expected) + && std::abs(actual - expected) <= 1e-12 * std::max(1.0, std::abs(expected)); + if (!ok) std::printf(" actual=%.17g expected=%.17g\n", actual, expected); + CHECK(ok); +} + +struct Book final : BacktestEngine { + Book() { + initial_capital_ = 1000; + commission_type_ = CommissionType::CASH_PER_ORDER; + commission_value_ = 0; + syminfo_.pointvalue = 1; + syminfo_.mintick = .01; + syminfo_mintick_ = .01; + account_currency_fx_ = 1; + qty_step_ = 0; + slippage_ = 0; + pyramiding_ = 100; + stream_observe_actions_ = true; + bar(1, 100); + } + void on_bar(const Bar&) override {} + void bar(int index, double price) { + current_bar_ = {price, price + 20, price - 20, price, 1, 1736121600000LL + index * 60000}; + bar_index_ = index; + } + x::PhysicalExecutionContext context() const { return {current_bar_.timestamp, bar_index_, {}, {}}; } + void open(double quantity, double price, uint64_t incarnation, const char* label, + std::optional paid = 0.0) { + REQUIRE(settle_native_execution_at(order_action::Transact{quantity}, + x::Fill{price, label, "historical", incarnation, paid}, context()).status == x::Status::Applied); + } + void source(double quantity, double price = 100, Cause cause = Script) { + (this->*access(Partial{}))(price, quantity, cause); + } + void native(double quantity, double price = 100) { + REQUIRE(settle_native_execution_at(order_action::Reduce{quantity}, + x::Fill{price, {}, {}, 0}, context()).status == x::Status::Applied); + } + void by_entry(double quantity, double price = 100) { + (this->*access(ByQuantity{}))(price, std::string("prefix"), quantity, Script); + } + void compatibility_drain(double quantity, double price = 100) { + (void)(this->*access(Drain{}))(nullptr, quantity, price, position_side_ == PositionSide::LONG); + } + void close_logical(const char* id) { + const double na = std::numeric_limits::quiet_NaN(); + double matching = 0, quantity = 0, retired = 0; + bool all = false; + REQUIRE((this->*access(ComputeClose{}))(std::string(id), na, na, false, + matching, quantity, all, retired)); + exact(matching, .8); + exact(quantity, .8); + CHECK(!all && retired == 0); + (this->*access(ImmediateClose{}))(std::string(id), std::string("funded close"), + quantity, matching, false, true, false, false, false); + } + void exit(PendingOrder& order, double price) { + int closed_bar = -1; + uint64_t closed_incarnation = 0; + bool closed_long = false; + (this->*access(ExitFill{}))(order, price, closed_bar, closed_incarnation, closed_long); + CHECK(closed_bar == -1 && closed_incarnation == 0 && !closed_long); + } + void schedule(CommissionType kind, double value, double pointvalue = 1, double fx = 1) { + commission_type_ = kind; + commission_value_ = value; + syminfo_.pointvalue = pointvalue; + account_currency_fx_ = fx; + } + void funded(const char* id, double quantity) { id_unclosed_qty_[id] = quantity; } + bool funded(const char* id) const { return id_unclosed_qty_.count(id) != 0; } + void slots(int count) { position_entry_count_ = count; } + int slots() const { return position_entry_count_; } + void slippage(int ticks, double tick) { slippage_ = ticks; syminfo_mintick_ = tick; syminfo_.mintick = tick; } + const auto& lots() const { return pyramid_entries_; } + auto& lots() { return pyramid_entries_; } + const auto& rows() const { return trades_; } + size_t actions() const { return stream_order_actions_.size(); } + int64_t cycle() const { return position_cycle_seq_; } + int64_t next_cycle() const { return next_position_cycle_seq_; } + void cycle_for_fixture(int64_t value) { position_cycle_seq_ = value; } + double quantity() const { return position_qty_; } + double balance() const { return initial_capital_ + net_profit_sum_; } + int index() const { return bar_index_; } + int64_t timestamp() const { return current_bar_.timestamp; } + uint64_t broker_fills() const { return broker_fill_event_seq_; } +}; + +void same_lot(const PyramidEntry& actual, const PyramidEntry& expected) { + exact(actual.qty, expected.qty); + exact(actual.price, expected.price); + exact(actual.entry_commission_account, expected.entry_commission_account); + exact(actual.max_runup, expected.max_runup); + exact(actual.max_drawdown, expected.max_drawdown); + exact(actual.entry_path_position, expected.entry_path_position); + CHECK(actual.time == expected.time && actual.entry_bar_index == expected.entry_bar_index); + CHECK(actual.entry_id == expected.entry_id && actual.entry_comment == expected.entry_comment); + CHECK(actual.entry_incarnation == expected.entry_incarnation); + CHECK(actual.skip_entry_bar_high == expected.skip_entry_bar_high); + CHECK(actual.skip_entry_bar_low == expected.skip_entry_bar_low); + CHECK(actual.market_pyramid_add == expected.market_pyramid_add); + CHECK(actual.bracket_slot_shadowed == expected.bracket_slot_shadowed); + CHECK(actual.ordinary_market_open == expected.ordinary_market_open); + CHECK(actual.pooc_terminal_market_entry == expected.pooc_terminal_market_entry); + CHECK(actual.ordinary_stop_open == expected.ordinary_stop_open); +} +void same_row(const Trade& actual, const Trade& expected) { + exact(actual.qty, expected.qty); + exact(actual.entry_price, expected.entry_price); + exact(actual.exit_price, expected.exit_price); + exact(actual.pnl, expected.pnl); + exact(actual.pnl_pct, expected.pnl_pct); + exact(actual.commission, expected.commission); + exact(actual.max_runup, expected.max_runup); + exact(actual.max_drawdown, expected.max_drawdown); + CHECK(actual.entry_time == expected.entry_time && actual.exit_time == expected.exit_time); + CHECK(actual.entry_bar_index == expected.entry_bar_index && actual.exit_bar_index == expected.exit_bar_index); + CHECK(actual.entry_incarnation == expected.entry_incarnation && actual.is_long == expected.is_long); + CHECK(actual.entry_id == expected.entry_id && actual.exit_id == expected.exit_id); + CHECK(actual.entry_comment == expected.entry_comment && actual.exit_comment == expected.exit_comment); + CHECK(actual.exit_from_bracket == expected.exit_from_bracket && actual.open_at_end == expected.open_at_end); +} +void same_native_effects(const Book& source, const Book& native) { + REQUIRE(source.rows().size() == native.rows().size()); + REQUIRE(source.lots().size() == native.lots().size()); + for (size_t index = 0; index < source.rows().size(); ++index) same_row(source.rows()[index], native.rows()[index]); + for (size_t index = 0; index < source.lots().size(); ++index) same_lot(source.lots()[index], native.lots()[index]); + exact(source.quantity(), native.quantity()); + exact(source.balance(), native.balance()); + CHECK(source.cycle() == native.cycle() && source.next_cycle() == native.next_cycle()); + CHECK(source.actions() == native.actions()); +} +void seed(Book& book, double sign, const std::vector& identities = {11, 12, 13}, + const char* sibling_label = "sibling") { + REQUIRE(identities.size() == 3); + book.open(sign * .7, 100, identities[0], "prefix", 7); + book.bar(2, 100); + book.open(sign * .1, 100, identities[1], "prefix", 1); + book.bar(3, 100); + book.open(sign, 100, identities[2], sibling_label, 17); + book.lots().back().max_runup = .123; + book.lots().back().max_drawdown = .456; + book.lots().back().entry_path_position = .75; + book.lots().back().skip_entry_bar_low = true; + book.bar(7, 100); +} + +void endpoint(double sign, double request) { + scenario = "source complete FIFO endpoint closes exact prefix and preserves sibling"; + Book book; + seed(book, sign); + book.schedule(CommissionType::CASH_PER_ORDER, 6); + const auto sibling = book.lots()[2]; + const auto cycle = book.cycle(), next_cycle = book.next_cycle(); + const auto actions = book.actions(); + const auto time = book.timestamp(); + book.source(request); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 1); + exact(book.rows()[0].qty, .7); + exact(book.rows()[1].qty, .1); + CHECK(book.rows()[0].entry_incarnation == 11 && book.rows()[1].entry_incarnation == 12); + same_lot(book.lots()[0], sibling); + exact(book.quantity(), 1); + CHECK(book.cycle() == cycle && book.next_cycle() == next_cycle); + CHECK(book.actions() == actions + 2); + CHECK(book.timestamp() == time && book.index() == 7); + CHECK(book.slots() == 1); + for (const auto& row : book.rows()) { + CHECK(row.exit_time == time && row.exit_bar_index == 7); + exact(row.exit_price, 100); + } + near(book.rows()[0].commission + book.rows()[1].commission, 14); // paid8 + one current6 + near(book.balance(), 986); +} + +void reduce_control(double sign, double request) { + scenario = "real interior source quantity retains original native Reduce effects"; + Book source, native; + seed(source, sign); + seed(native, sign); + source.schedule(CommissionType::CASH_PER_ORDER, 6); + native.schedule(CommissionType::CASH_PER_ORDER, 6); + source.source(request); + native.native(request); + same_native_effects(source, native); + if (request < .8) { + REQUIRE(source.lots().size() == 2 && source.rows().size() == 2); + CHECK(source.lots()[0].qty > 1e-10 && source.lots()[0].entry_incarnation == 12); + exact(source.lots()[1].qty, 1); + } else { + REQUIRE(source.lots().size() == 1 && source.rows().size() == 3); + CHECK(source.rows()[2].qty > 0 && source.rows()[2].entry_incarnation == 13); + CHECK(source.lots()[0].qty < 1); + } +} + +void native_spill_and_scope_walls(double sign) { + scenario = "native Reduce and retained compatibility drain preserve exact spill"; + Book native, drain; + seed(native, sign); + seed(drain, sign); + native.native(.8); + drain.compatibility_drain(.8); + same_native_effects(drain, native); + REQUIRE(native.rows().size() == 3 && native.lots().size() == 1); + exact(native.rows()[2].qty, 1.1102230246251565e-16); + exact(native.lots()[0].qty, .99999999999999989); + CHECK(native.rows()[2].entry_incarnation == 13); + + scenario = "entry-scoped interior dust stays outside source FIFO translation"; + Book scoped; + scoped.open(sign, 100, 11, "prefix", 6); + scoped.open(sign * 3, 100, 12, "sibling", 17); + const auto sibling = scoped.lots()[1]; + const double request = 1 - 5e-11; + scoped.by_entry(request); + REQUIRE(scoped.rows().size() == 1 && scoped.lots().size() == 2); + exact(scoped.rows()[0].qty, request); + CHECK(scoped.lots()[0].qty > 0 && scoped.lots()[0].qty < 1e-10); + same_lot(scoped.lots()[1], sibling); +} + +void identity_fallback(double sign, const std::vector& identities) { + scenario = "unowned or split physical identities fall back without scope expansion"; + Book source, native; + seed(source, sign, identities); + seed(native, sign, identities); + source.source(.8); + native.native(.8); + same_native_effects(source, native); + REQUIRE(source.rows().size() == 3 && source.lots().size() == 1); + exact(source.rows()[2].qty, 1.1102230246251565e-16); + CHECK(source.lots()[0].entry_incarnation == identities[2]); +} + +void complete_fragments(double sign) { + scenario = "complete repeated identity inside prefix is selected once and closes all fragments"; + Book book; + seed(book, sign, {11, 11, 13}); + const auto sibling = book.lots()[2]; + book.source(.8); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 1); + CHECK(book.rows()[0].entry_incarnation == 11 && book.rows()[1].entry_incarnation == 11); + exact(book.rows()[0].qty, .7); + exact(book.rows()[1].qty, .1); + same_lot(book.lots()[0], sibling); +} + +void unavailable_selection_cycle(double sign) { + scenario = "unavailable selected opening cycle keeps scalar source behavior"; + Book source, native; + seed(source, sign); + seed(native, sign); + source.cycle_for_fixture(0); + native.cycle_for_fixture(0); + source.source(.8); + native.native(.8); + same_native_effects(source, native); + REQUIRE(source.rows().size() == 3 && source.lots().size() == 1); + exact(source.rows()[2].qty, 1.1102230246251565e-16); +} + +void stop_before_tiny_sibling(double sign) { + scenario = "source stops before next sibling without using its tiny size"; + Book book; + book.open(sign * .7, 100, 11, "prefix"); + book.open(sign * .1, 100, 12, "prefix"); + book.open(sign * 5e-11, 100, 13, "tiny-unselected", .25); + book.open(sign, 100, 14, "later", 17); + const auto tiny = book.lots()[2], later = book.lots()[3]; + book.source(.8); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 2); + same_lot(book.lots()[0], tiny); + same_lot(book.lots()[1], later); +} + +void logical_funding_physical_fifo(double sign) { + scenario = "logical close funding selects oldest physical FIFO identities"; + Book book; + seed(book, sign, {11, 12, 13}, "L5"); + // The logical credit has diverged from L5's live physical1 after prior + // default-FIFO attribution. It funds .8 while the old prefix bears other IDs. + book.funded("L5", .8); + const auto sibling = book.lots()[2]; + const auto fills = book.broker_fills(); + book.close_logical("L5"); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 1); + CHECK(book.rows()[0].entry_incarnation == 11 && book.rows()[1].entry_incarnation == 12); + CHECK(book.rows()[0].entry_id == "prefix" && book.rows()[1].entry_id == "prefix"); + CHECK(book.rows()[0].exit_id == "__close__L5" && book.rows()[1].exit_id == "__close__L5"); + CHECK(book.rows()[0].exit_comment == "funded close" && book.rows()[1].exit_comment == "funded close"); + CHECK(!book.funded("L5")); + CHECK(book.broker_fills() == fills + 1); + same_lot(book.lots()[0], sibling); +} + +void frozen_reservation(double sign) { + scenario = "priced frozen reservation closes old prefix and preserves newer sibling"; + Book book; + book.open(sign * .7, 100, 11, "old-first", 7); + book.bar(2, 100); + book.open(sign * .1, 100, 12, "old-second", 1); + PendingOrder order; + order.type = OrderType::EXIT; + order.id = "frozen-basket"; + order.from_entry = ""; + order.qty = .8; + order.qty_percent = 100; + order.incarnation = 90; + order.created_seq = 90; + order.created_bar = 3; + order.created_position_side = sign > 0 ? PositionSide::LONG : PositionSide::SHORT; + order.created_position_cycle_seq = book.cycle(); + order.quantity_request.request(QuantityIntent::units(.8)); + order.quantity_request.reserve(.8, .8); + order.legs.set_limit_price(100); + book.bar(4, 100); + book.open(sign, 100, 13, "newer", 17); + const auto newer = book.lots()[2]; + const auto cycle = book.cycle(), next_cycle = book.next_cycle(); + book.bar(7, 100); + book.schedule(CommissionType::CASH_PER_ORDER, 6); + book.slots(9); + book.exit(order, 100); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 1); + exact(book.rows()[0].qty, .7); + exact(book.rows()[1].qty, .1); + same_lot(book.lots()[0], newer); + exact(order.qty, .8); + CHECK(book.cycle() == cycle && book.next_cycle() == next_cycle && book.slots() == 9); + CHECK(book.rows()[0].exit_time == book.timestamp() && book.rows()[1].exit_time == book.timestamp()); + near(book.rows()[0].commission + book.rows()[1].commission, 14); +} + +void fee_schedule(double sign, CommissionType type, double fee, + double expected_first, double expected_second, double expected_balance) { + scenario = "source prefix uses one existing commission quote and full paid historical costs"; + Book book; + seed(book, sign); + const auto sibling = book.lots()[2]; + book.schedule(type, fee); + // Wipe-side endpoint must realize full .1 and all historical cost1. + book.source(.8 - 5e-11); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 1); + exact(book.rows()[0].qty, .7); + exact(book.rows()[1].qty, .1); + near(book.rows()[0].commission, expected_first); + near(book.rows()[1].commission, expected_second); + near(book.balance(), expected_balance); + same_lot(book.lots()[0], sibling); +} + +void historical_fx(double sign) { + scenario = "prefix Flatten realizes historical percentage costs at original FX"; + Book book; + book.schedule(CommissionType::PERCENT, 1, 2, 2); + book.open(sign * .7, 100, 11, "prefix", std::nullopt); + book.open(sign * .1, 100, 12, "prefix", std::nullopt); + book.open(sign, 100, 13, "sibling", std::nullopt); + near(book.lots()[0].entry_commission_account, 2.8); + near(book.lots()[1].entry_commission_account, .4); + const auto sibling = book.lots()[2]; + book.schedule(CommissionType::PERCENT, 1, 2, 3); + book.source(.8 - 5e-11); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 1); + near(book.rows()[0].commission, 7); + near(book.rows()[1].commission, 1); + near(book.balance(), 992); + same_lot(book.lots()[0], sibling); +} + +void source_slots_price_and_clock(double sign, Cause cause) { + scenario = "prefix settlement retains source slot policy and applies slippage once"; + Book book; + seed(book, sign); + const auto sibling = book.lots()[2]; + const auto cycle = book.cycle(), next_cycle = book.next_cycle(); + const auto timestamp = book.timestamp(); + book.slots(9); + book.slippage(2, .25); + book.source(.8, 110, cause); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 1); + CHECK(book.slots() == (cause == Bracket ? 9 : 1)); + CHECK(book.cycle() == cycle && book.next_cycle() == next_cycle); + CHECK(book.timestamp() == timestamp && book.index() == 7); + same_lot(book.lots()[0], sibling); + for (const auto& row : book.rows()) { + exact(row.exit_price, 110 - sign * .5); + CHECK(row.exit_time == timestamp && row.exit_bar_index == 7); + } +} + +void whole_book_and_noop(double sign) { + scenario = "existing whole-book source endpoint promotion remains unchanged"; + Book book; + seed(book, sign); + const auto next = book.next_cycle(); + book.source(book.quantity() - 5e-11); + REQUIRE(book.rows().size() == 3 && book.lots().empty()); + exact(book.rows()[0].qty, .7); + exact(book.rows()[1].qty, .1); + exact(book.rows()[2].qty, 1); + CHECK(book.cycle() == 0 && book.next_cycle() == next); + + scenario = "empty sub-epsilon source prefix remains a no-op"; + Book noop; + seed(noop, sign); + const auto broker = noop.broker_state_hash(), stream = noop.stream_state_hash(); + const auto actions = noop.actions(); + noop.source(5e-11); + CHECK(noop.broker_state_hash() == broker && noop.stream_state_hash() == stream); + CHECK(noop.rows().empty() && noop.lots().size() == 3 && noop.actions() == actions); +} + +template void run(F call) { + try { call(); } + catch (const Abort&) {} + catch (const std::exception& error) { + ++failures; + std::printf("FAIL %s exception: %s\n", scenario, error.what()); + } +} +} // namespace + +int main() { + static_assert(std::variant_size_v == 3); + for (double sign : {1.0, -1.0}) { + for (double quantity : {.8, .8 - 5e-11, .8 + 5e-11}) + run([&] { endpoint(sign, quantity); }); + for (double quantity : {.8 - 2e-10, .8 + 1e-10, .75, .85}) + run([&] { reduce_control(sign, quantity); }); + run([&] { native_spill_and_scope_walls(sign); }); + for (const auto& identities : {std::vector{0, 0, 13}, {0, 12, 13}, {11, 0, 13}, {11, 12, 11}}) + run([&] { identity_fallback(sign, identities); }); + run([&] { complete_fragments(sign); }); + run([&] { unavailable_selection_cycle(sign); }); + run([&] { stop_before_tiny_sibling(sign); }); + run([&] { logical_funding_physical_fifo(sign); }); + run([&] { frozen_reservation(sign); }); + run([&] { fee_schedule(sign, CommissionType::CASH_PER_ORDER, 6, 12.25, 1.75, 986); }); + run([&] { fee_schedule(sign, CommissionType::CASH_PER_ORDER, -6, 1.75, .25, 998); }); + run([&] { fee_schedule(sign, CommissionType::CASH_PER_CONTRACT, 2, 8.4, 1.2, 990.4); }); + run([&] { fee_schedule(sign, CommissionType::PERCENT, 1, 7.7, 1.1, 991.2); }); + run([&] { historical_fx(sign); }); + for (Cause cause : {Script, Bracket, Margin}) + run([&] { source_slots_price_and_clock(sign, cause); }); + run([&] { whole_book_and_noop(sign); }); + } + std::printf("%s source FIFO endpoints: %d checks, %d failures\n", + failures ? "FAIL" : "PASS", checks, failures); + return failures ? 1 : 0; +}