From 6ab8f2a0350a9926336ea91e4ec44278837a7dce Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 13 Sep 2026 03:36:19 +0800 Subject: [PATCH 1/4] Unify physical fills under native settlement and account projections --- .github/workflows/ci.yml | 14 + .github/workflows/native-live.yml | 5 + AGENTS.md | 2 + README.md | 6 +- docs/native-refactor-progress.md | 46 +- docs/native-settlement.md | 50 +- include/pineforge/engine.hpp | 143 ++-- include/pineforge/execution_close_scope.hpp | 3 + .../pineforge/execution_close_selection.hpp | 18 + include/pineforge/execution_projection.hpp | 45 + scripts/check_settlement_cpp_abi.py | 473 +++++++++++ scripts/prepare_settlement_cpp_abi_base.py | 206 +++++ scripts/test_settlement_cpp_abi.py | 65 ++ src/engine_execution.cpp | 788 ++++++++++++------ src/engine_fills.cpp | 124 ++- src/engine_orders.cpp | 461 +++++----- tests/CMakeLists.txt | 21 + tests/fixtures/settlement_cpp_abi/README.md | 92 ++ .../settlement_cpp_abi/e60e571/manifest.json | 267 ++++++ tests/test_native_selected_settlement.cpp | 171 ++++ tests/test_native_settlement_callers.cpp | 251 ++++++ tests/test_native_settlement_projection.cpp | 233 ++++++ 22 files changed, 2866 insertions(+), 618 deletions(-) create mode 100644 include/pineforge/execution_close_selection.hpp create mode 100644 include/pineforge/execution_projection.hpp create mode 100644 scripts/check_settlement_cpp_abi.py create mode 100644 scripts/prepare_settlement_cpp_abi_base.py create mode 100644 scripts/test_settlement_cpp_abi.py create mode 100644 tests/fixtures/settlement_cpp_abi/README.md create mode 100644 tests/fixtures/settlement_cpp_abi/e60e571/manifest.json create mode 100644 tests/test_native_selected_settlement.cpp create mode 100644 tests/test_native_settlement_callers.cpp create mode 100644 tests/test_native_settlement_projection.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5eb67c9a..b3d59a7d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Install Eigen3 (Linux) if: runner.os == 'Linux' @@ -37,6 +39,11 @@ jobs: - name: Build run: cmake --build build -j 4 + - name: Prepare real R2 settlement ABI provider + run: | + python3 scripts/prepare_settlement_cpp_abi_base.py --source-repo . \ + --current-build build --output build/settlement-abi-base --jobs 4 + - name: Test run: ctest --test-dir build --output-on-failure @@ -83,6 +90,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Install Eigen3 run: | @@ -95,6 +104,11 @@ jobs: - name: Build run: cmake --build build-asan -j 4 + - name: Prepare real R2 settlement ABI provider + run: | + python3 scripts/prepare_settlement_cpp_abi_base.py --source-repo . \ + --current-build build-asan --output build-asan/settlement-abi-base --jobs 4 + - name: Test under sanitizers env: ASAN_OPTIONS: detect_leaks=1:halt_on_error=1:abort_on_error=1 diff --git a/.github/workflows/native-live.yml b/.github/workflows/native-live.yml index 529daf4af..129040fe8 100644 --- a/.github/workflows/native-live.yml +++ b/.github/workflows/native-live.yml @@ -18,6 +18,7 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: persist-credentials: false + fetch-depth: 0 - name: Install native build dependencies run: | sudo apt-get update @@ -43,6 +44,10 @@ jobs: -DPINEFORGE_BUILD_LIVE_RUNNER=ON -DPINEFORGE_BUILD_TUTORIAL=OFF \ -DCURL_DIR="$PWD/build-native-deps/curl-install/lib/cmake/CURL" cmake --build build-live -j 4 + - name: Prepare real R2 settlement ABI provider + run: | + python3 scripts/prepare_settlement_cpp_abi_base.py --source-repo . \ + --current-build build-live --output build-live/settlement-abi-base --jobs 4 - name: Test native engine, parser, journal and transports run: | ctest --test-dir build-live --output-on-failure -j 4 diff --git a/AGENTS.md b/AGENTS.md index e8963b9a2..e8e949051 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,6 +13,8 @@ cmake -B build -S . \ -DPINEFORGE_BUILD_TESTS=ON \ -DPINEFORGE_BUILD_CORPUS_STRATEGIES=ON cmake --build build -j$(nproc 2>/dev/null || echo 4) +python3 scripts/prepare_settlement_cpp_abi_base.py --source-repo . \ + --current-build build --output build/settlement-abi-base ctest --test-dir build --output-on-failure # 2. Run full validation corpus sweep (against TV exported trades) diff --git a/README.md b/README.md index 03bdbf23e..1fd0f24b9 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # PineForge -**The open-source PineScript v6 backtest engine that reproduces TradingView trade-for-trade.** +**An open-source C++17 engine for backtesting and forward execution, with PineScript support through code generation.** [![CI](https://img.shields.io/github/actions/workflow/status/pineforge-4pass/pineforge-engine/ci.yml?branch=main&label=ci&logo=github)](https://github.com/pineforge-4pass/pineforge-engine/actions) [![Parity](https://img.shields.io/badge/TradingView%20parity-4%2C190%20%2F%204%2C190%20probes-brightgreen)](#validation-scoreboard) @@ -76,11 +76,15 @@ Prefer zero install? The hosted server at **[mcp.pineforge.dev/mcp](https://mcp. git clone https://github.com/pineforge-4pass/pineforge-engine.git && cd pineforge-engine cmake -B build -DCMAKE_BUILD_TYPE=Release cmake --build build -j +python3 scripts/prepare_settlement_cpp_abi_base.py --source-repo . \ + --current-build build --output build/settlement-abi-base ctest --test-dir build --output-on-failure # all enabled unit, replay and ABI checks bash tutorial/run.sh # MACD on BTC/USDT, end to end python3 tutorial/run_stream.py # OHLCV warm-up → realtime trades ``` +The ABI check links callers against both the current library and a separately built, pinned historical library. Preparation uses local Git history and the configured compiler; see the [ABI fixture guide](tests/fixtures/settlement_cpp_abi/README.md) for shallow clones and repeated checks. CTest itself stays offline. + Prerequisites: CMake ≥ 3.16, a C++17 compiler (GCC ≥ 9, Clang ≥ 10, Apple Clang ≥ 12), Eigen 3.3+ (fetched automatically if absent), Python 3 for the tests (`-DPINEFORGE_BUILD_TESTS=OFF` for a library-only build). `cmake --install build --prefix /usr/local` installs `lib/libpineforge.a`, `include/pineforge/`, and the `find_package(PineForge)` config. ### Embedded in your own harness diff --git a/docs/native-refactor-progress.md b/docs/native-refactor-progress.md index 20f6dae8a..7472d2937 100644 --- a/docs/native-refactor-progress.md +++ b/docs/native-refactor-progress.md @@ -1,6 +1,6 @@ # Standalone native engine refactor -Status at 2026-09-12: **3 of 9 roadmap phases complete (33%)**. This counts +Status at 2026-09-13: **4 of 9 roadmap phases complete (44%)**. This counts completed phases; it is not an estimate of elapsed work or remaining time. The acceptance boundary is a standalone C++ backtest and forward-execution @@ -14,10 +14,10 @@ improvement resumes after the final native audit. | --- | --- | --- | | R0 | Native settlement and CI convergence | Complete, [PR #241](https://github.com/pineforge-4pass/pineforge-engine/pull/241) | | R1 | Native market orders, host, drivers, calendar and forward runner | Complete, [PR #243](https://github.com/pineforge-4pass/pineforge-engine/pull/243) | -| R2 | Resting orders, partial execution, owner-bound children and group effects | R2a prerequisite complete ([PR #244](https://github.com/pineforge-4pass/pineforge-engine/pull/244)); resting lifecycle implemented, validation and acceptance pending | -| R3 | Accounting, admission, risk and observation ownership | Open | +| R2 | Resting orders, partial execution, owner-bound children and group effects | Complete, [PR #246](https://github.com/pineforge-4pass/pineforge-engine/pull/246) | +| R3 | Accounting, admission, risk and observation ownership | Accounting implementation in progress; admission/risk/observation remain open | | R3a | Reversal execution and lifecycle settlement | Complete, [PR #242](https://github.com/pineforge-4pass/pineforge-engine/pull/242) | -| R3b | Migrate remaining physical fill paths to shared settlement | Open | +| R3b | Migrate remaining physical fill paths to shared settlement | Implementation in progress | | R4 | Complete codegen/adapter policy ownership and native independence | Open | | R5 | Final requirement and compatibility audit | Open | | R6 | Resume the parity improvement campaign after the audit | Queued | @@ -55,10 +55,9 @@ These results apply to R2a. They do not establish acceptance of the later resting-order implementation, and R2a is not an additional completed roadmap phase. -## Current R2 implementation +## Accepted R2 resting-order state machine -This branch implements the pinned native contract. Validation and review -repairs are still in progress: +PR #246 completed the native resting-order contract: 1. Limit, stop, stop-limit and trailing requests retain trigger state and partial remainders. Matching distinguishes actual tick prints from @@ -84,10 +83,17 @@ partial exits, exact deferred-receipt arithmetic and same-cursor execution price The sanitizer ABI control also supplies the historical v12 destructor needed for its RTTI; this changes only the link-test stub, not runtime behavior. -Final review of the publication candidate, remaining R2 acceptance coverage, -a new fixed-population 4190 comparison, the actual gate and published-head CI -remain required. R2 is not complete. The earlier R2a and separate AUTO-isolation -compatibility results do not substitute for this implementation's proof. +The final candidate `c57e8687d91027df09474b414369162e47b94bbb` passed independent +review, 326 local CTests (one macOS libcurl WebSocket skip), all nine PR checks +and four post-merge workflows. It was squash merged as +`e60e57156a54bb1c74c0c0a4f5a338fd924c046d`, with identical tree +`e209c89035ec884774f8f3e68e1dc118f26aaf51`. + +The completed Cloud comparison measured 72/72 cases and 4190/4190 probes with +zero grade, raw-trade, substantive-verifier, metadata, configuration or coverage +differences. The actual gate returned **FAIL: `target.not-positive` only**. +That neutral result is retained under the authorized refactor exception; +the baseline stayed unchanged and promotion was skipped. R2 preserves all currently supported calendar, session, timezone, DST and timeframe behavior. Source labels remain inert in the native kernel; @@ -97,6 +103,24 @@ The native R2 boundary is pinned; complete Pine lowering remains mandatory R4 work. The parity-improvement campaign remains paused through the final native audit. +## Current R3 accounting and fill migration + +The next implementation brings the remaining partial, bound, percent, scratch, +reversal, RAW and open/add fill paths onto the shared settlement owner. A +transient set of opening identities selects physical exposure without putting +Pine `from_entry` strings into the kernel. A non-applying account projection +supplies post-close equity to source sizing before one resolved reversal. + +The native owner retains historical paid costs, one current execution ticket, +physical residual quantities and deterministic observations. Adapters retain +quantity provenance, source slot policy and the existing exit-ordering rules. +Existing native request/event layouts and C ABI 4 remain stable. This work +requires implementation, native and real-caller tests, independent review and +the full publication proof below before R3b can be marked complete. + +R3's remaining admission, quota/day clocks, source counters and physical versus +script-visible observation ownership stay open and must finish before R4. + ## Publication and acceptance Publish reviewed increments in an evolving draft PR so implementation and diff --git a/docs/native-settlement.md b/docs/native-settlement.md index b5248fe1b..772f33f42 100644 --- a/docs/native-settlement.md +++ b/docs/native-settlement.md @@ -70,13 +70,40 @@ PnL, minus remaining paid entry costs of every fee type. At an unchanged mark, a native execution decreases this value by its current charge, subject to ordinary floating-point rounding. A rebate increases it. +## Selected exposure and account projection + +`execution::SelectedOpeningSet` selects one or more opening identities in the +current position cycle. The separately named selected settlement methods accept +it by const reference; it is not another `CloseScope` alternative or a queued +request field. All fragments of each selected opening participate in roster FIFO +order. Empty sets, zero or duplicate identities, missing openings and stale cycles +are refused before effects. Source adapters construct distinct identities from +`from_entry` or other source predicates; the native owner does not interpret them. + +`project_native_settlement_v1`, `project_native_settlement_scoped_v1` and +`project_native_settlement_selected_v1` quote the resulting physical and account +facts without changing the engine. The projection includes realized balance, +remaining paid entry costs, marked equity, signed exposure and resulting cycle. +It includes unselected surviving lots and any prospective opening's paid cost. +A valid no-effect action quotes the unchanged book at the supplied mark; invalid +input supplies no usable quote. An unavailable fresh cycle throws without +consuming it. Same-side additions retain the existing cycle. + +Projection and settlement share allocation and fee logic. Realized balance adds +close-row PnL in commit order, and marked equity applies each resulting lot's mark +and paid cost in roster order. A projection is data, not commit authority: later +settlement revalidates against the current book. Source percent sizing can consume +a close-only projection's equity before resolving one reversal transaction. + ## Integration and limits -The kernel currently serves full market exits, the close-opposite-then-enter -reversal family, and selected frozen-transaction and same-side materialization -paths. Existing source scheduling and dust decisions remain at their call -sites. The reversal helper consumes one already-resolved `Fill` and one current -fee; it does not slip again. `compat::pine` suspension selection stays at the +The kernel serves native execution and the source adapters' full, partial, +bound and percent closes, scratch fills, reversals, RAW orders and opening/add +paths. Callers resolve source scheduling, quantity grids, price and slot policy; +the owner applies the physical effects and accounting once. A separately matched +scratch fill remains its own execution. Reversal closing and opening share one +already-resolved price and one current ticket. +`compat::pine` suspension selection stays at the replacement caller. `settle_resolved_execution` remains the original two-argument symbol and forwards empty effects to `settle_execution_with_lifecycle`, the protected seam that consumes transient @@ -94,9 +121,9 @@ reallocate `pending_orders_`. Native settlement does not recognize source cases, rewrite supplied window/barrier facts, or install a callback/plan. Migrated frozen transactions and the final short-seed crossing settle their close/open effects in one native call. -Other legacy close loops are not yet -grouped into one parent execution; their per-row current ticket behavior is -not a claim about the native contract. +Production fill paths no longer use the old per-row close loops as a separate +accounting owner. Historical private helpers remain for source compatibility and +tests; they are not an alternate production execution path. The shared close builder now consumes historical entry costs. This changes the former reconstruction that converted both commission legs at exit-time @@ -114,6 +141,7 @@ lifecycle exceptions abort the owning run; callers must discard that failed run rather than retry a partially committed execution in place. Strong rollback on allocation failure is not promised. -The work does not remove `ShortSeedCollisionRole` or complete migration to a -generic queued order machine. Existing executable state remains represented -in ABI projections and fingerprints. +The work retains `ShortSeedCollisionRole` while its source-policy consumers +remain. Admission, source day/quota counters, script-visible observations and +complete Pine lowering remain separate refactor work. Existing executable state +remains represented in ABI projections and fingerprints. diff --git a/include/pineforge/engine.hpp b/include/pineforge/engine.hpp index 3624e74dd..68cc25414 100644 --- a/include/pineforge/engine.hpp +++ b/include/pineforge/engine.hpp @@ -21,6 +21,8 @@ #include "quantity_intent.hpp" #include "execution.hpp" #include "execution_close_scope.hpp" +#include "execution_close_selection.hpp" +#include "execution_projection.hpp" #include "market_admission.hpp" #include "reservation_expansion.hpp" #include "order_cancellation.hpp" @@ -1840,6 +1842,35 @@ class BacktestEngine { const execution::Action& action, const execution::Fill& fill, const execution::PhysicalExecutionContext& context, execution::CloseScope scope); + // --- Additive selected close (not a CloseScope overload) --- + execution::SettlementInspection inspect_native_settlement_selected( + const execution::Action& action, + const execution::Fill& fill, + const execution::SelectedOpeningSet& selection) const; + execution::Result settle_native_execution_selected_at( + const execution::Action& action, + const execution::Fill& fill, + const execution::PhysicalExecutionContext& context, + const execution::SelectedOpeningSet& selection); + // Legacy lifecycle copy of current_bar_/fold flags, then selected commit. + // Empty lifecycle is valid. Do not fold selection into LifecycleEffects. + execution::Result settle_execution_selected_with_lifecycle( + const execution::Action& action, + const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::SelectedOpeningSet& selection); + // --- Additive non-applying account/effect projection --- + execution::AccountEffectProjection project_native_settlement_v1( + const execution::Action& action, + const execution::Fill& fill) const; + execution::AccountEffectProjection project_native_settlement_scoped_v1( + const execution::Action& action, + const execution::Fill& fill, + execution::CloseScope scope) const; + execution::AccountEffectProjection project_native_settlement_selected_v1( + const execution::Action& action, + const execution::Fill& fill, + const execution::SelectedOpeningSet& selection) const; // Native account value: realized balance plus marked physical lots minus // their remaining paid entry costs, for every fee type. No Pine sizing or // end-of-range reporting convention participates in this value. @@ -2641,68 +2672,9 @@ class BacktestEngine { // tape discriminating round(close) from close has been replayed. A // future CASH mismatch on a sub-tick feed is traced here first. double calc_qty(double fill_price) const { - const double basis = round_to_mintick(fill_price); - switch (default_qty_type_) { - case QtyType::FIXED: - return apply_qty_step(default_qty_value_); - case QtyType::PERCENT_OF_EQUITY: { - // KI-56's clean-room v6 flat/holding pair proves that an - // omitted percent-of-equity add sizes from mark-to-market - // equity AFTER the paid percent commission on surviving lots. - // This is a ledger rule, independent of pct=100, margin, or - // how the already-open position was sized. The open lot is - // marked at the ROUNDED close (see above): a sub-tick print - // never reaches the broker ledger. - // round 8 family R: the broker sizes from the equity at - // TEN SIGNIFICANT DIGITS (tv_money_round above) — the lot - // count flips one lot early/late whenever the exact equity - // sits within half a money unit of a lot boundary. - double equity = percent_commission_live_equity( - round_to_mintick(current_bar_.close)); - // Round 10 family AE (NASDAQ:AAPL@15, fast-scalper; campaign - // note log-20260905t223336z-67f0f181): the rounding is NOT - // confined to sub-unit lots. On a cent-priced integer-share - // book the exact equity is a cent sum, but the engine's - // ledger carries float-accumulation noise (1094521.68 held - // as 1094521.6800000002) and the quotient by the close can - // sit exactly on an integer in decimal (1094521.68 / 238.77 - // = 4584): TradingView floors the ten-digit equity's raw - // double quotient (4583.999999999999 -> 4583 shares); the - // noisy numerator (4584.000000000001) or the 1e-6 nudge of - // apply_qty_step both hand it 4584, one share the account - // cannot pay at the 238.78 fill, and the reversal is declined - // where TradingView admits it. Nine lab tv capital sweeps on - // AAPL (famae-sz-*): 897890.32 / 213.58 -> 4203 (the raw - // floor; 4204 in exact decimal), 897890.3200004 -> 4203 (the - // 4e-7 is rounded away before the division), 897890.321 -> - // 4204; 887295.33 / 211.11 -> 4202, .3300004 -> 4202, - // .331 -> 4203; 1094521.68 / 238.77 -> 4583 (the probe's - // row), 1094521.681 -> 4584 (dropped: 4584 x 238.78 > C). - // Every lot-stepped instrument therefore sizes from - // tv_money_round(equity) with the raw lot floor; only the - // corpus' continuous qty_step 0 keeps the exact arithmetic. - if (tv_money_lot_sizing()) equity = tv_money_round(equity); - if (!std::isfinite(equity)) return 0.0; - double cash = reserve_percent_commission(equity * (default_qty_value_ / 100.0)) / active_account_currency_fx(); - // Reject (qty 0) on a non-finite / non-positive fill price — a - // degenerate $0/NaN print must NOT size as the raw % number. - if (!(std::isfinite(basis) && basis > 0)) return 0.0; - // Rule 1 floors the rounded-equity quantity onto the lot - // grid via tv_money_floor_lot, including its guarded recovery - // of a representable cent-lot point. No representation nudge: - // apply_qty_step's epsilon would still over-size the pinned - // everybar 1.085 placement by one lot. - if (tv_money_lot_sizing()) { - return tv_money_floor_lot(cash / (basis * syminfo_.pointvalue), - qty_step_); - } - return apply_qty_step(cash / (basis * syminfo_.pointvalue)); - } - case QtyType::CASH: - return (std::isfinite(basis) && basis > 0) - ? apply_qty_step((default_qty_value_ / active_account_currency_fx()) / (basis * syminfo_.pointvalue)) : 0.0; - } - return apply_qty_step(default_qty_value_); + const double equity = default_qty_type_ == QtyType::PERCENT_OF_EQUITY + ? percent_commission_live_equity(round_to_mintick(current_bar_.close)) : 0.0; + return calc_default_qty_from_equity(fill_price, equity); } // TradingView freezes DEFAULT (qty=na) market-order sizing at the SIGNAL @@ -3897,6 +3869,35 @@ class BacktestEngine { const execution::LifecycleEffects& lifecycle, const execution::PhysicalExecutionContext& context, execution::CloseScope scope); + execution::Result settle_with_context_selected( + const execution::Action& action, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::PhysicalExecutionContext& context, + const execution::SelectedOpeningSet& selection); + execution::AccountEffectProjection project_with_membership( + const execution::Action& action, + const execution::Fill& fill, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected) const; + execution::SettlementInspection inspect_with_membership( + const execution::Action& action, + const execution::Fill& fill, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected) const; + execution::Result settle_with_membership( + const execution::Action& action, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::PhysicalExecutionContext& context, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected); + struct NativeSettlementStage; + void stage_native_settlement( + NativeSettlementStage& stage, + const execution::Action& action, + const execution::Fill& fill, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected, + const execution::LifecycleEffects* lifecycle) const; enum class PositionReductionCause { SCRIPT_ORDER, // strategy.close / close_all / market exit / reversal BRACKET_EXIT, // a strategy.exit bracket leg fill @@ -4339,6 +4340,10 @@ class BacktestEngine { double active_account_currency_fx() const; void settle_position_after_partial_exit( PositionReductionCause cause); + void restore_source_partial_exit_slots(int pre_count, PositionReductionCause cause); + execution::Result settle_source_opening( + PositionSide requested, double fill_price, double qty, + const std::string& id, const std::string& comment, uint64_t incarnation); void enter_market_from_flat(const std::string& id, bool is_long, double fill_price, double explicit_qty, int explicit_qty_type, @@ -4353,6 +4358,11 @@ class BacktestEngine { PositionSide created_position_side, bool is_priced_entry, uint64_t entry_incarnation); + void add_to_pyramid_market_with_qty_provenance( + const std::string& id, bool is_long, double fill_price, double explicit_qty, + int explicit_qty_type, PositionSide created_position_side, + bool is_priced_entry, bool explicit_qty_prequantized, + uint64_t entry_incarnation); // `fill_price` is already resolved. Source sizing, direction and dust // selection stay here; purge_pending_exits is translated into exact // pending removals for the settlement coordinator. False does not @@ -4378,6 +4388,15 @@ class BacktestEngine { double fill_price, double explicit_qty, int explicit_qty_type, uint64_t entry_incarnation); + void sequential_same_tick_reversal_fill_with_qty_provenance( + const std::string& id, bool is_long, double fill_price, double explicit_qty, + int explicit_qty_type, bool explicit_qty_prequantized, + uint64_t entry_incarnation); + double calc_default_qty_from_equity(double fill_price, double equity) const; + double calc_qty_for_type_from_equity( + double fill_price, double qty_value, int qty_type, double equity) const; + double source_reversal_qty(double fill_price, double explicit_qty, + int explicit_qty_type, bool prequantized) const; void bind_exit_activation(PendingOrder& order); void bind_retained_exit_activations(); void unbind_exit_activations(); diff --git a/include/pineforge/execution_close_scope.hpp b/include/pineforge/execution_close_scope.hpp index 65294d9f2..d46335244 100644 --- a/include/pineforge/execution_close_scope.hpp +++ b/include/pineforge/execution_close_scope.hpp @@ -18,5 +18,8 @@ struct OpeningExposure { using CloseScope = std::variant; +static_assert(std::variant_size_v == 2, + "CloseScope is Book|OpeningExposure; do not add alternatives"); + } // inline namespace close_scope_v1 } // namespace pineforge::execution diff --git a/include/pineforge/execution_close_selection.hpp b/include/pineforge/execution_close_selection.hpp new file mode 100644 index 000000000..1a6e9e894 --- /dev/null +++ b/include/pineforge/execution_close_selection.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace pineforge::execution { +inline namespace close_selection_v1 { + +// Transient membership for one inspect/project/settle call. +// Run-local opening-request provenances in the engine's current cycle. +// Not a CloseScope alternative, not a Request field, not hashed, not stored. +struct SelectedOpeningSet { + std::int64_t cycle = 0; + std::vector incarnations; +}; + +} // inline namespace close_selection_v1 +} // namespace pineforge::execution diff --git a/include/pineforge/execution_projection.hpp b/include/pineforge/execution_projection.hpp new file mode 100644 index 000000000..68abfa45d --- /dev/null +++ b/include/pineforge/execution_projection.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include "execution.hpp" +#include +#include + +namespace pineforge::execution { +inline namespace settlement_projection_v1 { + +// Non-applying physical+account quote of one Action+Fill against the live book. +// Destroyed when the caller is done sizing. Not commit authority. +struct AccountEffectProjection { + Status status = Status::NoEffect; + + // Same physical aggregates SettlementInspection reports for this Action. + double closed_units = 0.0; + double opened_units = 0.0; + double resulting_abs_units = 0.0; + std::size_t resulting_lot_count = 0; + double resulting_abs_notional = 0.0; + double current_ticket = 0.0; + bool would_open = false; + bool incoming_short = false; + + // Account facts SettlementInspection lacks. Units are account currency. + // Copy net_profit_sum_, add each projected close-row pnl in roster order, + // then add initial_capital_. Do not sum rows first or regroup. + double realized_balance = 0.0; + // All paid costs on the projected resulting open book, including a + // prospective opening lot's allocated current-ticket share. 0 when empty. + double remaining_entry_cost = 0.0; + // Start from realized_balance; add each resulting lot's marked pnl minus + // its paid remaining entry cost in roster order at fill.price. Do not + // subtract opening cost twice. + double marked_equity = 0.0; + // 0 if projected book empty. Else the live position_cycle_seq_ when + // survivors remain. If would_open from flat or after emptying the old + // side, the peeked next_position_cycle_seq_ (NOT consumed). + std::int64_t cycle_after = 0; + // Signed resulting units (short negative). 0 when empty. + double signed_units_after = 0.0; +}; + +} // inline namespace settlement_projection_v1 +} // namespace pineforge::execution diff --git a/scripts/check_settlement_cpp_abi.py b/scripts/check_settlement_cpp_abi.py new file mode 100644 index 000000000..dba7c3226 --- /dev/null +++ b/scripts/check_settlement_cpp_abi.py @@ -0,0 +1,473 @@ +#!/usr/bin/env python3 +"""Actual old/new settlement ABI pairings. Compile and link; NEVER run callers. + +Requires a separately prepared real e60 R2 archive. No Git/network/build fallback +is performed by this CTest-time checker. Existing native/script ABI guards stay +separate and mandatory, including their old epoch and sanitizer RTTI controls. +""" +from __future__ import annotations +import argparse +import hashlib +import json +from pathlib import Path +import re +import shlex +import shutil +import subprocess +import tempfile + +from check_aggregate_cpp_versions import clean, body +from check_native_cpp_abi import HOST_EVENTS_CALLER, assembly_layout_values +from prepare_settlement_cpp_abi_base import ( + BASE_COMMIT, BASE_TREE, COPY_CACHE, authenticate_headers, compiler_identity, + extract_tar, identity, read_cache, run, +) + +ROOT = Path(__file__).resolve().parents[1] +ENGINE = 'pineforge::engine_script_run_v13::BacktestEngine::' +OLD_METHODS = ('inspect_native_settlement', 'inspect_native_settlement_scoped', + 'settle_native_execution_at', 'settle_native_execution_scoped_at', + 'settle_resolved_execution','settle_execution_with_lifecycle','settle_with_context') +OLD_PRIVATE = ('add_to_pyramid_market', 'sequential_same_tick_reversal_fill') +NEW_METHODS = ('inspect_native_settlement_selected', 'settle_native_execution_selected_at', + 'settle_execution_selected_with_lifecycle', 'project_native_settlement_v1', + 'project_native_settlement_scoped_v1', 'project_native_settlement_selected_v1') +NEW_PRIVATE = ('add_to_pyramid_market_with_qty_provenance', + 'sequential_same_tick_reversal_fill_with_qty_provenance') +PRESERVED_ARCHIVE_SHA = 'e13d3d19ad4613c28beddfabb119f1dddadb2c39e474edd6304a2f75c7321f60' +PRESERVED_HEADERS_SHA = '1001102a496ae927ae98e111dd7dc68ab6c23ecc41a9eba00995144d9a532109' +FROZEN_NATIVE_HEADERS = ('native_order.hpp', 'native_order_identity.hpp', 'native_host.hpp', + 'native_run_spec.hpp', 'market_driver.hpp', 'native_calendar.hpp', + 'execution_consumer.hpp') + +COMMON = '''#include +#include +#include +#include +using E = pineforge::engine_script_run_v13::BacktestEngine; +namespace ex = pineforge::execution; +using A = ex::Action; +using F = ex::Fill; +using C = ex::PhysicalExecutionContext; +using S = ex::CloseScope; +using I = ex::SettlementInspection; +using R = ex::Result; +using L = ex::LifecycleEffects; +static_assert(std::is_same_v); +static_assert(std::variant_size_v == 2); +static_assert(std::variant_size_v == 16); +''' +OLD_CALLER = COMMON + ''' +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +auto old_inspect = &E::inspect_native_settlement; +auto old_scoped_inspect = &E::inspect_native_settlement_scoped; +auto old_settle = &E::settle_native_execution_at; +auto old_scoped_settle = &E::settle_native_execution_scoped_at; +int main(int argc, char** argv) { + auto* e = reinterpret_cast(argv); + A a = ex::Flatten{}; F f{100,"","",1}; C c{}; S s = ex::Book{}; + auto i0 = (e->*old_inspect)(a,f); auto i1 = (e->*old_scoped_inspect)(a,f,s); + auto r0 = (e->*old_settle)(a,f,c); auto r1 = (e->*old_scoped_settle)(a,f,c,s); + auto p0=e->settle_resolved_execution(a,f); auto p1=e->settle_execution_with_lifecycle(a,f,L{}); + auto p2=e->settle_with_context(a,f,L{},c); + return int(i0.closed_units+i1.closed_units+r0.closed_units+r1.closed_units+p0.closed_units+p1.closed_units+p2.closed_units); +} +''' +NEW_CALLER = COMMON + ''' +using Set = ex::SelectedOpeningSet; +using P = ex::AccountEffectProjection; +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +int main(int argc, char** argv) { + auto* e = reinterpret_cast(argv); + A a = ex::Flatten{}; F f{100,"","",1}; C c{}; L life{}; Set set{1,{1}}; + auto i = e->inspect_native_settlement_selected(a,f,set); + auto r = e->settle_native_execution_selected_at(a,f,c,set); + auto l = e->settle_execution_selected_with_lifecycle(a,f,life,set); + auto p0 = e->project_native_settlement_v1(a,f); + auto p1 = e->project_native_settlement_scoped_v1(a,f,ex::Book{}); + auto p2 = e->project_native_settlement_selected_v1(a,f,set); + return int(i.closed_units+r.closed_units+l.closed_units+p0.realized_balance+p1.remaining_entry_cost+p2.marked_equity); +} +''' +PRIVATE_OLD_CALLER = COMMON + ''' +using Add = void(E::*)(const std::string&,bool,double,double,int,pineforge::PositionSide,bool,uint64_t); +using Seq = void(E::*)(const std::string&,bool,double,double,int,uint64_t); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +auto old_add = &E::add_to_pyramid_market; +auto old_sequential = &E::sequential_same_tick_reversal_fill; +int main(int argc,char** argv) { + auto* e = reinterpret_cast(argv); + (e->*old_add)("old",true,100,1,-1,pineforge::PositionSide::LONG,false,1); + (e->*old_sequential)("old",true,100,1,-1,1); + return argc; +} +''' +PRIVATE_NEW_CALLER = COMMON + ''' +using Add = void(E::*)(const std::string&,bool,double,double,int,pineforge::PositionSide,bool,bool,uint64_t); +using Seq = void(E::*)(const std::string&,bool,double,double,int,bool,uint64_t); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +int main(int argc,char** argv) { + auto* e = reinterpret_cast(argv); + e->add_to_pyramid_market_with_qty_provenance("new",true,100,1,-1,pineforge::PositionSide::LONG,false,true,1); + e->sequential_same_tick_reversal_fill_with_qty_provenance("new",true,100,1,-1,true,1); + return argc; +} +''' + +FIELDS = { + 'Result': [('status','ex::Status'),('closed_units','double'),('opened_units','double'), + ('current_ticket','double'),('first_trade_index','std::size_t'), + ('closed_trade_count','std::size_t'),('opened_lot_incarnation','uint64_t')], + 'SettlementInspection': [('status','ex::Status'),('closed_units','double'),('opened_units','double'), + ('resulting_abs_units','double'),('resulting_lot_count','std::size_t'), + ('resulting_abs_notional','double'),('current_ticket','double'),('would_open','bool'),('incoming_short','bool')], +} + + +def normalized(text: str) -> str: + return re.sub(r'\s+', ' ', clean(text)).strip() + + +def storage_declarations(header: str) -> list[str]: + """Read top-level named data declarations; skip method bodies/declarations. + + Actual compiler offsets/types below supplement this source-order guard. + String contents are irrelevant to storage declarations and may contain braces. + """ + text = re.sub(r'"(?:\\.|[^"\\])*"', '""', clean(header)) + text = body(text, r'class\s+BacktestEngine\s*\{', 'BacktestEngine') + statements, start, depth = [], 0, 0 + for at, char in enumerate(text): + if char == '{': + depth += 1 + elif char == '}': + depth -= 1 + if depth == 0: + prefix = text[start:at + 1] + before = prefix.split('{', 1)[0] + # Function definition (including const/noexcept) is not storage. + if ')' in before and '=' not in before: + start = at + 1 + elif char == ';' and depth == 0: + segment = text[start:at] + start = at + 1 + segment = re.sub(r'\b(?:public|protected|private)\s*:', '', segment).strip() + prefix = segment.split('=', 1)[0].split('{', 1)[0].strip() + if '(' in prefix or re.match(r'^(?:using|typedef|friend|static_assert|class|struct|enum)\b', prefix): + continue + if re.search(r'\b[A-Za-z_]\w*\s*(?:\[[^]]*\])?\s*$', prefix): + statements.append(re.sub(r'\s+', ' ', prefix)) + if len(statements) < 100: + raise RuntimeError('engine storage inventory unexpectedly small; inspect source parser before accepting ABI') + return statements + + +def frozen_shape(old_include: Path, current_include: Path) -> tuple[list[str], dict]: + old_exec = (old_include/'pineforge/execution.hpp').read_text() + cur_exec = (current_include/'pineforge/execution.hpp').read_text() + for name in ['Result','SettlementInspection','Status','Fill','PhysicalExecutionContext','LifecycleEffects']: + pattern = r'(?:struct|enum\s+class)\s+' + name + r'\s*\{' + if normalized(body(old_exec, pattern, name)) != normalized(body(cur_exec, pattern, name)): + raise RuntimeError('frozen execution aggregate/enum changed: ' + name) + for name in FROZEN_NATIVE_HEADERS: + if normalized((old_include/'pineforge'/name).read_text()) != normalized((current_include/'pineforge'/name).read_text()): + raise RuntimeError('R3 must preserve native header layout/contracts: ' + name) + scopes = [(directory/'pineforge/execution_close_scope.hpp').read_text() + for directory in (old_include,current_include)] + for name in ['Book','OpeningExposure']: + pattern = r'struct\s+' + name + r'\s*\{' + if normalized(body(scopes[0],pattern,name)) != normalized(body(scopes[1],pattern,name)): + raise RuntimeError('frozen CloseScope alternative changed: ' + name) + aliases = [re.search(r'using\s+CloseScope\s*=\s*[^;]+;',clean(text)) for text in scopes] + if any(alias is None for alias in aliases) or normalized(aliases[0].group()) != normalized(aliases[1].group()): + raise RuntimeError('CloseScope alternative identities/order changed') + old_engine = (old_include/'pineforge/engine.hpp').read_text() + cur_engine = (current_include/'pineforge/engine.hpp').read_text() + old_storage, current_storage = storage_declarations(old_engine), storage_declarations(cur_engine) + if old_storage != current_storage: + raise RuntimeError('engine named data declarations/order changed') + virtuals = lambda text: re.findall(r'\bvirtual\b[^;{]*(?:;|\{)', clean(text)) + if [normalized(v) for v in virtuals(old_engine)] != [normalized(v) for v in virtuals(cur_engine)]: + raise RuntimeError('engine virtual method inventory changed') + members = [re.search(r'\b([A-Za-z_]\w*)\s*(?:\[[^]]*\])?\s*$', declaration).group(1) + for declaration in old_storage if not declaration.startswith('static ')] + return members, {'engineStorage': old_storage, 'virtuals': [normalized(v) for v in virtuals(old_engine)]} + + +def layout_source(members: list[str]) -> tuple[str, int]: + values = [] + assertions = [] + for name in ['Result','SettlementInspection']: + values += [f'sizeof(ex::{name})',f'alignof(ex::{name})'] + for field, expected_type in FIELDS[name]: + assertions.append(f'static_assert(std::is_same_v);') + values.append(f'offsetof(ex::{name},{field})') + for index, status in enumerate(['Applied','NoEffect','InvalidPrice','InvalidQuantity','InvalidBook', + 'UnrepresentableQuantity','InvalidAccounting','InvalidLifecycle','InvalidCloseTarget']): + assertions.append(f'static_assert(int(ex::Status::{status})=={index});') + values += ['sizeof(ex::Status)','sizeof(S)','alignof(S)','std::variant_size_v', + 'sizeof(E)','alignof(E)','sizeof(pineforge::PendingOrder)', + 'sizeof(pineforge::NativeStrategyHost)','sizeof(pineforge::NativeMarketEvent)', + 'sizeof(pineforge::NativeStateView)','sizeof(pineforge::native_order::Request)', + 'sizeof(pineforge::native_order::WorkingRequestCore)', + 'sizeof(pineforge::native_order::CommandEvent)'] + for member in members: + values += [f'offsetof(E,{member})', f'sizeof(decltype(E::{member}))', f'alignof(decltype(E::{member}))'] + return COMMON + '\n'.join(assertions) + '\nextern "C" const unsigned long long abi_layout[] = {\n' + ',\n'.join(values) + '\n};\n', len(values) + + +def defined_symbols(library: Path) -> str: + raw = run(['nm','-g','-C',str(library)]).decode('utf-8','replace') + return '\n'.join(line for line in raw.splitlines() if re.search(r'\b[TWtw]\s+', line)) + + +def validate_rejection(diagnostic: str, missing, domain=None) -> list[str]: + symbols = re.findall(r'^\s*"(.+)", referenced from:', diagnostic, re.M) + symbols += re.findall(r"undefined reference to [`'](.+)'", diagnostic) + symbols += re.findall(r'undefined symbol:\s*(.+)', diagnostic) + if not symbols: + raise RuntimeError('link failure has no recognized undefined-symbol diagnostics') + for method in missing: + matching = [symbol for symbol in symbols if symbol.startswith(ENGINE+method+'(')] + if not matching: + raise RuntimeError('link failure omits expected undefined method: '+method) + if domain and 'selected' in method and any(domain not in symbol for symbol in matching): + raise RuntimeError('selected method has wrong/missing parameter namespace: '+method) + unrelated = [symbol for symbol in symbols if not any(symbol.startswith(ENGINE+method+'(') for method in missing)] + if unrelated: + raise RuntimeError('link failure includes unrelated undefined symbols: '+', '.join(unrelated)) + return symbols + + +def load_base(args, destination: Path, current_cache: dict) -> tuple[Path, Path, Path, dict]: + if not args.base_receipt.is_file(): + raise RuntimeError('real R2 archive receipt missing; run scripts/prepare_settlement_cpp_abi_base.py ' + '--source-repo . --current-build BUILD --output BUILD/settlement-abi-base before CTest') + receipt = json.loads(args.base_receipt.read_text()) + if receipt.get('commit') != BASE_COMMIT or receipt.get('tree') != BASE_TREE: + raise RuntimeError('real base receipt does not pin e60 R2') + base_root = args.base_receipt.resolve().parent + resolve_artifact = lambda name: Path(name) if Path(name).is_absolute() else base_root/name + library = resolve_artifact(receipt['archive']) + headers = resolve_artifact(receipt.get('headers','r2-headers.tar')) + if identity(library)['sha256'] != receipt['archiveSha256'] or identity(headers)['sha256'] != receipt['headersSha256']: + raise RuntimeError('real R2 archive/header bytes do not match receipt') + if not library.read_bytes().startswith(b'!\n'): + raise RuntimeError('real base is not a static archive') + if len(run(['ar','-t',str(library)]).splitlines()) < 20: + raise RuntimeError('real base is not a full product archive') + extract_tar(headers.read_bytes(), destination) + authenticate_headers(destination) + if receipt.get('schemaVersion') == 'pineforge-settlement-abi-base/v1': + old_compiler = receipt['compiler'] + current_compiler = compiler_identity(args.compiler) + if any(old_compiler[key] != current_compiler[key] for key in ('target','sha256','version')): + raise RuntimeError('base compiler implementation/version/target differs; prepare base on this runner') + actual_settings = {key: current_cache[key] for key in COPY_CACHE if key in current_cache} + if receipt['copiedCurrentCache'] != actual_settings: + raise RuntimeError('base preparation compiler/configuration differs from current build; prepare a matching base') + generated = resolve_artifact(receipt['generatedInclude']) + if identity(generated/'pineforge/version.h')['sha256'] != receipt['generatedHeaderSha256']: + raise RuntimeError('base generated version header changed') + else: + # Explicit reuse of root's preserved Mac Release artifact, never a stub. + if receipt['archiveSha256'] != PRESERVED_ARCHIVE_SHA or receipt['headersSha256'] != PRESERVED_HEADERS_SHA: + raise RuntimeError('unrecognized legacy base receipt; use portable preparation') + if args.extra_flag or current_cache.get('PINEFORGE_ENABLE_SANITIZERS') == 'ON' or current_cache.get('PINEFORGE_ENABLE_COVERAGE') == 'ON': + raise RuntimeError('preserved uninstrumented R2 archive is not this lane; prepare a matching real base') + if not args.base_generated_include: + raise RuntimeError('preserved base requires --base-generated-include from its original build') + generated = args.base_generated_include.resolve() + identity(generated/'pineforge/version.h') + old_cache=read_cache(generated.parent/'CMakeCache.txt') + old_settings={key:old_cache[key] for key in COPY_CACHE if key in old_cache} + current_settings={key:current_cache[key] for key in COPY_CACHE if key in current_cache} + if old_settings != current_settings: + raise RuntimeError('preserved R2 build configuration differs; use matching portable preparation') + if identity(generated.parent/'lib/libpineforge.a')['sha256'] != receipt['archiveSha256']: + raise RuntimeError('preserved base cache/header directory no longer belongs to its archive') + symbols = defined_symbols(library) + for method in (*OLD_METHODS,*OLD_PRIVATE): + if ENGINE + method + '(' not in symbols: + raise RuntimeError('real old archive omits original symbol: ' + method) + for method in (*NEW_METHODS,*NEW_PRIVATE): + if ENGINE + method + '(' in symbols: + raise RuntimeError('supplied old archive already exports new method: ' + method) + return library, destination/'include', generated, receipt + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--compiler', required=True) + parser.add_argument('--library', type=Path, required=True) + parser.add_argument('--include', type=Path, required=True) + parser.add_argument('--generated-include', type=Path, required=True) + parser.add_argument('--base-receipt', type=Path, required=True) + parser.add_argument('--base-generated-include', type=Path) + parser.add_argument('--extra-flag', action='append', default=[]) + parser.add_argument('--receipt', type=Path, required=True) + parser.add_argument('--base-only', action='store_true', help='preparation control only; does not claim new API proof') + parser.add_argument('--old-rejections-only', action='store_true', + help='new public header callers versus actual old provider only; no current archive acceptance') + parser.add_argument('--public-only', action='store_true', + help='complete public API matrix before new private helpers are integrated; partial R3 proof') + args = parser.parse_args() + library, include = args.library.resolve(), args.include.resolve() + source = include.parent + cache = read_cache(library.parent.parent/'CMakeCache.txt') + if compiler_identity(cache['CMAKE_CXX_COMPILER'])['sha256'] != compiler_identity(args.compiler)['sha256']: + raise RuntimeError('--compiler differs from current archive build compiler') + if cache.get('PINEFORGE_ENABLE_SANITIZERS') == 'ON' and '-fsanitize=address,undefined' not in args.extra_flag: + raise RuntimeError('sanitizer archive requires --extra-flag=-fsanitize=address,undefined') + source_files = [source/'CMakeLists.txt', *[p for root in ('src','include','cmake') for p in (source/root).rglob('*') if p.is_file()]] + stale = [str(path.relative_to(source)) for path in source_files if path.stat().st_mtime > library.stat().st_mtime] + if sum((args.base_only,args.old_rejections_only,args.public_only)) > 1: + raise RuntimeError('select only one partial proof mode') + if stale and not args.old_rejections_only: + raise RuntimeError('current archive predates source; full rebuild required: '+', '.join(stale)) + initial_identity = identity(library) + flags = shlex.split(cache.get('CMAKE_CXX_FLAGS','')) + common = [args.compiler,'-std=c++17',*flags,'-O0','-UNDEBUG','-ffp-contract=off', + '-fno-access-control','-Wno-invalid-'+'offsetof',*args.extra_flag] + if cache.get('PINEFORGE_ENABLE_SANITIZERS') == 'ON': + common += ['-DEIGEN_MAX_ALIGN_BYTES=0'] + args.receipt.parent.mkdir(parents=True, exist_ok=True) + log_root = Path(tempfile.mkdtemp(prefix=args.receipt.stem+'.artifacts-',dir=args.receipt.parent)) + report = {'schemaVersion':'pineforge-settlement-abi/v1','status':'incomplete', + 'mode':'base-only' if args.base_only else 'old-rejections-only' if args.old_rejections_only else 'public-only' if args.public_only else 'full-new-old-matrix', + 'currentArchive':{'path':str(library),**initial_identity}, + 'compiler':compiler_identity(args.compiler),'callerFlags':common[1:], + 'privateAccessScaffolding':'-fno-access-control only for ABI TUs; product visibility unchanged', + 'artifactsDirectory':str(log_root),'newArchivePairingComplete':False, + 'sourceSha256':{str(path.relative_to(source)):identity(path)['sha256'] for path in source_files}, + 'compiles':[],'links':[],'executedBinaries':0,'networkFetches':0} + try: + with tempfile.TemporaryDirectory(prefix='pineforge-settlement-abi-') as temporary: + scratch = Path(temporary) + old_library,old_include,old_generated,old_receipt = load_base(args,scratch/'r2',cache) + report['base']={'receiptSha256':identity(args.base_receipt)['sha256'], + 'archiveSha256':identity(old_library)['sha256'], + 'commit':BASE_COMMIT,'tree':BASE_TREE} + members,shape = frozen_shape(old_include,include) + report['frozenShape']=shape + + def compile_tu(name,text,headers,generated): + path=log_root/(name+'.cpp');path.write_text(text) + obj=log_root/(name+'.o') + argv=[*common,'-I',str(headers),'-I',str(generated),'-c',str(path),'-o',str(obj)] + run(argv,timeout=120,log=log_root/(name+'.compile.log')) + report['compiles'].append({'name':name,'argv':argv,'sourceSha256':identity(path)['sha256'],'objectSha256':identity(obj)['sha256']}) + return obj + + def link(name,obj,runtime,missing=(),domain=None): + argv=[*common,str(obj),str(runtime),'-pthread','-o',str(log_root/name)] + result=subprocess.run(argv,capture_output=True,text=True,timeout=120) + diagnostic=result.stdout+result.stderr + (log_root/(name+'.link.log')).write_text(diagnostic) + if not missing: + if result.returncode: raise RuntimeError(name+' positive pair failed:\n'+diagnostic) + else: + if result.returncode == 0: raise RuntimeError(name+' unexpectedly linked') + validate_rejection(diagnostic,missing,domain) + report['links'].append({'name':name,'argv':argv,'exitCode':result.returncode, + 'outcome':'expected-rejection' if missing else 'linked','requiredMissing':list(missing), + 'selectionDomain':domain,'executed':False}) + + # Compile every actual caller before interpreting any link outcome. + old=compile_tu('old-book-singleton',OLD_CALLER,old_include,old_generated) + cur_old=compile_tu('current-old-member-types',OLD_CALLER,include,args.generated_include) + private_old=compile_tu('old-private-f8-f11',PRIVATE_OLD_CALLER,old_include,old_generated) + current_private_old=compile_tu('current-old-private-types',PRIVATE_OLD_CALLER,include,args.generated_include) + old_events=compile_tu('old-host-events-return',HOST_EVENTS_CALLER,old_include,old_generated) + current_events=compile_tu('current-host-events-return',HOST_EVENTS_CALLER,include,args.generated_include) + layout_text,word_count=layout_source(members) + layouts=[] + for label,headers,generated in [('old',old_include,old_generated),('current',include,args.generated_include)]: + compile_tu(label+'-layout',layout_text,headers,generated) + src=log_root/(label+'-layout.cpp');asm=log_root/(label+'-layout.s') + run([*common,'-I',str(headers),'-I',str(generated),'-S',str(src),'-o',str(asm)],timeout=120, + log=log_root/(label+'-layout.assembly.log')) + layouts.append(assembly_layout_values(asm.read_text(),word_count)) + if layouts[0]!=layouts[1]: raise RuntimeError('actual compiler old/current layout/offset/type-size arrays differ') + report['layout']={'wordCount':word_count,'values':layouts[0],'members':members} + if not args.base_only: + current_header=clean((include/'pineforge/engine.hpp').read_text()) + for method in NEW_METHODS: + match=re.search(r'[^;{}]*\b'+method+r'\s*\([^;{}]*;',current_header) + if not match or re.search(r'\bvirtual\b',match.group()): + raise RuntimeError('new method must have one nonvirtual declaration: '+method) + preceding=current_header[:match.start()] + access=re.findall(r'\b(public|protected|private)\s*:',preceding) + if not access or access[-1]!='protected': + raise RuntimeError('new method must stay protected: '+method) + new=compile_tu('new-six-methods',NEW_CALLER,include,args.generated_include) + if not args.old_rejections_only and not args.public_only: + private_new=compile_tu('new-private-provenance',PRIVATE_NEW_CALLER,include,args.generated_include) + wrong_project=scratch/'wrong-project';shutil.copytree(include,wrong_project) + header=wrong_project/'pineforge/engine.hpp';changed=header.read_text();wrong_source=NEW_CALLER + wrong_names=[] + for method in NEW_METHODS[3:]: + future=method[:-2]+'v2';wrong_names.append(future) + changed=changed.replace(method,future);wrong_source=wrong_source.replace(method,future) + header.write_text(changed) + projection_header=wrong_project/'pineforge/execution_projection.hpp' + projection_header.write_text(projection_header.read_text().replace('settlement_projection_v1','settlement_projection_v2')) + wrong_source=wrong_source.replace('settlement_projection_v1','settlement_projection_v2') + wrong_projection=compile_tu('synthetic-future-project-v2',wrong_source,wrong_project,args.generated_include) + wrong_selection=scratch/'wrong-selection';shutil.copytree(include,wrong_selection) + header=wrong_selection/'pineforge/execution_close_selection.hpp' + changed=header.read_text();require_name='close_selection_v1' + if require_name not in changed: raise RuntimeError('selection namespace missing from real header') + header.write_text(changed.replace(require_name,'close_selection_v2')) + wrong_set=compile_tu('synthetic-selection-v2',NEW_CALLER.replace(require_name,'close_selection_v2'),wrong_selection,args.generated_include) + link('old-api-old-real',old,old_library) + link('old-private-old-real',private_old,old_library) + link('old-events-old-real',old_events,old_library) + if not args.old_rejections_only: + link('old-api-new-real',old,library) + link('current-old-api-new-real',cur_old,library) + link('old-private-new-real',private_old,library) + link('current-old-private-new-real',current_private_old,library) + link('old-events-new-real',old_events,library) + link('current-events-new-real',current_events,library) + if not args.base_only: + link('new-api-old-real-rejected',new,old_library,NEW_METHODS,'close_selection_v1::SelectedOpeningSet') + if not args.old_rejections_only: + link('new-api-new-real',new,library) + if not args.public_only: + link('new-private-new-real',private_new,library) + link('new-private-old-real-rejected',private_new,old_library,NEW_PRIVATE) + link('synthetic-project-v2-rejected',wrong_projection,library,wrong_names) + link('synthetic-selection-v2-rejected',wrong_set,library, + (NEW_METHODS[0],NEW_METHODS[1],NEW_METHODS[2],NEW_METHODS[5]),'close_selection_v2::SelectedOpeningSet') + if identity(library)!=initial_identity: raise RuntimeError('current archive changed during ABI proof') + if any(identity(source/name)['sha256']!=sha for name,sha in report['sourceSha256'].items()): + raise RuntimeError('source changed during ABI proof') + report['status']='base-control-only' if args.base_only else 'old-rejections-only' if args.old_rejections_only else 'public-matrix-only' if args.public_only else 'passed' + report['newArchivePairingComplete']=report['status']=='passed' + except Exception as error: + report['status']='failed';report['error']=str(error) + raise + finally: + args.receipt.write_text(json.dumps(report,indent=2,sort_keys=True)+'\n') + print(f"settlement ABI: {report['status']}; {len(report['compiles'])} compiles, {len(report['links'])} links, 0 executed binaries") + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/prepare_settlement_cpp_abi_base.py b/scripts/prepare_settlement_cpp_abi_base.py new file mode 100644 index 000000000..2bdd47252 --- /dev/null +++ b/scripts/prepare_settlement_cpp_abi_base.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Build the exact R2 settlement ABI provider from local Git objects, without tests. + +Run explicitly before CTest. This script never fetches Git history or dependencies. +The exact Git object and Eigen must already be available to the current build. +""" +from __future__ import annotations +import argparse +import hashlib +import io +import json +import os +from pathlib import Path +import platform +import shlex +import shutil +import subprocess +import tarfile + +BASE_COMMIT = 'e60e57156a54bb1c74c0c0a4f5a338fd924c046d' +BASE_TREE = 'e209c89035ec884774f8f3e68e1dc118f26aaf51' +ROOT = Path(__file__).resolve().parents[1] +HEADER_MANIFEST = ROOT / 'tests/fixtures/settlement_cpp_abi/e60e571/manifest.json' +COPY_CACHE = ( + 'CMAKE_BUILD_TYPE', 'CMAKE_C_COMPILER', 'CMAKE_CXX_COMPILER', + 'CMAKE_C_COMPILER_TARGET', 'CMAKE_CXX_COMPILER_TARGET', + 'CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN', 'CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN', + 'CMAKE_C_FLAGS', 'CMAKE_CXX_FLAGS', + 'CMAKE_C_FLAGS_DEBUG', 'CMAKE_CXX_FLAGS_DEBUG', + 'CMAKE_C_FLAGS_RELEASE', 'CMAKE_CXX_FLAGS_RELEASE', + 'CMAKE_C_FLAGS_RELWITHDEBINFO', 'CMAKE_CXX_FLAGS_RELWITHDEBINFO', + 'CMAKE_C_FLAGS_MINSIZEREL', 'CMAKE_CXX_FLAGS_MINSIZEREL', + 'CMAKE_EXE_LINKER_FLAGS', 'CMAKE_STATIC_LINKER_FLAGS', + 'CMAKE_POSITION_INDEPENDENT_CODE', 'CMAKE_INTERPROCEDURAL_OPTIMIZATION', + 'CMAKE_OSX_ARCHITECTURES', 'CMAKE_OSX_SYSROOT', 'CMAKE_OSX_DEPLOYMENT_TARGET', + 'CMAKE_SYSROOT', 'CMAKE_TOOLCHAIN_FILE', 'CMAKE_PREFIX_PATH', 'Eigen3_DIR', + 'CMAKE_DISABLE_FIND_PACKAGE_Eigen3', 'FETCHCONTENT_SOURCE_DIR_EIGEN', + 'PINEFORGE_ENABLE_SANITIZERS', 'PINEFORGE_ENABLE_COVERAGE', +) + + +def identity(path: Path) -> dict: + raw = path.read_bytes() + return {'sha256': hashlib.sha256(raw).hexdigest(), 'bytes': len(raw)} + + +def read_cache(path: Path) -> dict[str, str]: + found = {} + for line in path.read_text().splitlines(): + if line and not line.startswith(('#', '//')) and ':' in line and '=' in line: + name_type, value = line.split('=', 1) + found[name_type.split(':', 1)[0]] = value + return found + + +def current_eigen_source(cache: dict[str, str], current: Path) -> Path | None: + """Reuse the current build's populated dependency without a network fetch.""" + candidates = [cache.get(key) for key in + ('FETCHCONTENT_SOURCE_DIR_EIGEN', 'Eigen3_SOURCE_DIR', 'eigen_SOURCE_DIR')] + candidates.append(str(Path(cache.get('FETCHCONTENT_BASE_DIR') or current / '_deps') / 'eigen-src')) + for value in candidates: + if not value: + continue + path = Path(value).resolve() + if (path / 'CMakeLists.txt').is_file() and (path / 'Eigen/Core').is_file(): + return path + return None + + +def run(argv, *, cwd=None, timeout=60, log: Path | None = None) -> bytes: + result = subprocess.run(argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) + if log: + log.write_bytes(result.stdout + result.stderr) + if result.returncode: + raise RuntimeError(f'command failed ({result.returncode}): {shlex.join(map(str, argv))}\n' + + result.stderr.decode('utf-8', 'replace')[-6000:]) + return result.stdout + + +def extract_tar(raw: bytes, destination: Path) -> None: + destination.mkdir(parents=True, exist_ok=False) + with tarfile.open(fileobj=io.BytesIO(raw)) as archive: + members = archive.getmembers() + seen = set() + for member in members: + path = Path(member.name) + if path.is_absolute() or '..' in path.parts or member.name in seen: + raise RuntimeError('unsafe or duplicate archive member: ' + member.name) + if not member.isdir() and not member.isfile(): + raise RuntimeError('ABI preparation refuses archive links/special files: ' + member.name) + seen.add(member.name) + for member in members: + target = destination / member.name + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + else: + target.parent.mkdir(parents=True, exist_ok=True) + stream = archive.extractfile(member) + assert stream is not None + target.write_bytes(stream.read()) + target.chmod(member.mode & 0o777) + + +def authenticate_headers(directory: Path) -> dict: + manifest = json.loads(HEADER_MANIFEST.read_text()) + if manifest['commit'] != BASE_COMMIT or manifest['tree'] != BASE_TREE: + raise RuntimeError('tracked R2 header pin changed') + actual = {str(path.relative_to(directory)) for path in (directory / 'include').rglob('*') if path.is_file()} + if actual != set(manifest['files']): + raise RuntimeError('R2 header closure differs from the tracked exact file inventory') + for name, expected in manifest['files'].items(): + raw = (directory / name).read_bytes() + blob = hashlib.sha1(b'blob ' + str(len(raw)).encode() + b'\0' + raw).hexdigest() + if identity(directory / name) != {'sha256': expected['sha256'], 'bytes': expected['bytes']} or blob != expected['git_blob']: + raise RuntimeError('R2 header bytes differ: ' + name) + return manifest + + +def compiler_identity(command: str) -> dict: + resolved = shutil.which(command) if not Path(command).is_absolute() else command + if not resolved: + raise RuntimeError('compiler not found: ' + command) + implementation = Path(resolved).resolve() + if platform.system() == 'Darwin' and implementation.parent == Path('/usr/bin'): + implementation = Path(run(['xcrun', '--find', Path(resolved).name]).decode().strip()).resolve() + return {'command': str(resolved), 'implementation': str(implementation), + **identity(implementation), 'version': run([command, '--version']).decode().strip(), + 'target': run([command, '-dumpmachine']).decode().strip()} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source-repo', type=Path, required=True) + parser.add_argument('--current-build', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--jobs', type=int, default=4) + args = parser.parse_args() + repo, current, output = args.source_repo.resolve(), args.current_build.resolve(), args.output.resolve() + if output.exists(): + raise RuntimeError('output must be new; reuse its existing receipt explicitly instead of rebuilding over it') + if not 1 <= args.jobs <= 64: + raise RuntimeError('--jobs must be 1..64') + cache = read_cache(current / 'CMakeCache.txt') + if Path(cache['CMAKE_HOME_DIRECTORY']).resolve() != repo: + raise RuntimeError('current CMake cache belongs to a different source repository') + try: + commit = run(['git', '-C', str(repo), 'rev-parse', BASE_COMMIT + '^{commit}']).decode().strip() + tree = run(['git', '-C', str(repo), 'rev-parse', BASE_COMMIT + '^{tree}']).decode().strip() + except RuntimeError as error: + raise RuntimeError(f'exact R2 source is unavailable; use a full-history checkout in CI, ' + f'or fetch the object in a local shallow clone: ' + f'git fetch --no-tags --depth=1 origin {BASE_COMMIT}') from error + if commit != BASE_COMMIT or tree != BASE_TREE: + raise RuntimeError('R2 commit/tree mismatch') + source_raw = run(['git', '-C', str(repo), 'archive', '--format=tar', BASE_COMMIT, + 'CMakeLists.txt', 'VERSION', 'cmake', 'include', 'src']) + output.mkdir(parents=True) + (output / 'source.tar').write_bytes(source_raw) + source, build = output / 'source', output / 'build' + extract_tar(source_raw, source) + authenticate_headers(source) + settings = {key: cache[key] for key in COPY_CACHE if key in cache} + eigen_source = current_eigen_source(cache, current) + if eigen_source is not None: + settings['FETCHCONTENT_SOURCE_DIR_EIGEN'] = str(eigen_source) + settings.update({'PINEFORGE_BUILD_TESTS': 'OFF', 'PINEFORGE_BUILD_TUTORIAL': 'OFF', + 'PINEFORGE_BUILD_CORPUS_STRATEGIES': 'OFF', 'PINEFORGE_BUILD_LIVE_RUNNER': 'OFF', + 'PINEFORGE_BUILD_BENCH_STRATEGIES': 'OFF', 'PINEFORGE_BUILD_SPEED_BENCH': 'OFF', + 'CMAKE_EXPORT_COMPILE_COMMANDS': 'ON', 'FETCHCONTENT_FULLY_DISCONNECTED': 'ON'}) + configure = ['cmake', '-S', str(source), '-B', str(build), '-G', cache['CMAKE_GENERATOR'], + *[f'-D{key}={value}' for key, value in settings.items()]] + run(configure, timeout=120, log=output / 'configure.log') + build_argv = ['cmake', '--build', str(build), '--target', 'pineforge', '--parallel', str(args.jobs)] + run(build_argv, timeout=1800, log=output / 'build.log') + library = build / 'lib/libpineforge.a' + if not library.read_bytes().startswith(b'!\n'): + raise RuntimeError('base library is not a full static archive') + archive_members = run(['ar', '-t', str(library)]).decode().splitlines() + if len(archive_members) < 20: + raise RuntimeError('base archive is not the full R2 library') + headers = output / 'r2-headers.tar' + with tarfile.open(headers, 'w') as archive: + archive.add(source / 'include', arcname='include', recursive=True) + source_identity = {str(path.relative_to(source)): identity(path)['sha256'] + for path in sorted(source.rglob('*')) if path.is_file()} + receipt = {'schemaVersion': 'pineforge-settlement-abi-base/v1', 'commit': BASE_COMMIT, 'tree': BASE_TREE, + 'archive': str(library.relative_to(output)), 'archiveSha256': identity(library)['sha256'], + 'headers': 'r2-headers.tar', 'headersSha256': identity(headers)['sha256'], + 'generatedInclude': 'build/include', + 'generatedHeaderSha256': identity(build / 'include/pineforge/version.h')['sha256'], + 'sourceSha256': source_identity, 'sourceArchive': identity(output / 'source.tar'), + 'compiler': compiler_identity(cache['CMAKE_CXX_COMPILER']), + 'settings': settings, 'copiedCurrentCache': {key: cache[key] for key in COPY_CACHE if key in cache}, + 'currentCacheSha256': identity(current / 'CMakeCache.txt')['sha256'], + 'baseCacheSha256': identity(build / 'CMakeCache.txt')['sha256'], + 'compileCommandsSha256': identity(build / 'compile_commands.json')['sha256'], + 'configureArgv': configure, 'buildArgv': build_argv, 'archiveMembers': archive_members, + 'executedBinaries': 0, 'networkFetches': 0, + 'sourceMode': 'immutable Git archive; generated version metadata uses VERSION fallback'} + (output / 'receipt.json').write_text(json.dumps(receipt, indent=2, sort_keys=True) + '\n') + print(str(output / 'receipt.json')) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/test_settlement_cpp_abi.py b/scripts/test_settlement_cpp_abi.py new file mode 100644 index 000000000..a3ae51514 --- /dev/null +++ b/scripts/test_settlement_cpp_abi.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Narrow offline refusal tests for the settlement ABI tooling; no C++ execution.""" +import io +from pathlib import Path +import tempfile +import tarfile +import unittest + +from check_settlement_cpp_abi import ENGINE, ROOT, storage_declarations, validate_rejection +from prepare_settlement_cpp_abi_base import extract_tar, read_cache + + +class AbiToolingTests(unittest.TestCase): + def test_mac_link_diagnostic_requires_only_named_method(self): + method='project_native_settlement_v1' + valid=f'Undefined symbols for architecture arm64:\n "{ENGINE}{method}(int) const", referenced from:\n _main\n' + self.assertEqual(len(validate_rejection(valid,[method])),1) + with self.assertRaisesRegex(RuntimeError,'unrelated'): + validate_rejection(valid+' "other_dependency()", referenced from:\n _main\n',[method]) + + def test_gnu_and_lld_diagnostics_keep_selection_domain(self): + method='inspect_native_settlement_selected' + symbol=ENGINE+method+'(pineforge::execution::close_selection_v1::SelectedOpeningSet const&) const' + for text in ["a.cpp: undefined reference to `"+symbol+"'",'ld.lld: error: undefined symbol: '+symbol]: + self.assertEqual(len(validate_rejection(text,[method],'close_selection_v1::SelectedOpeningSet')),1) + with self.assertRaisesRegex(RuntimeError,'namespace'): + validate_rejection(text,[method],'close_selection_v2::SelectedOpeningSet') + + def test_generic_failure_or_missing_one_method_cannot_pass(self): + with self.assertRaisesRegex(RuntimeError,'no recognized'): + validate_rejection('linker error: file not found',['project_native_settlement_v1']) + valid=f'undefined reference to `{ENGINE}project_native_settlement_v1(int)\'' + with self.assertRaisesRegex(RuntimeError,'omits expected'): + validate_rejection(valid,['project_native_settlement_v1','project_native_settlement_scoped_v1']) + + def test_same_size_member_change_or_added_padding_member_is_visible(self): + source=(ROOT/'include/pineforge/engine.hpp').read_text() + original=storage_declarations(source) + changed=source.replace('double position_entry_price_', 'uint64_t position_entry_price_',1) + self.assertNotEqual(original,storage_declarations(changed)) + changed=source.replace('double position_entry_price_', 'int abi_padding_member;\n double position_entry_price_',1) + self.assertNotEqual(original,storage_declarations(changed)) + self.assertTrue(any('abi_padding_member' in item for item in storage_declarations(changed))) + + def test_tar_links_and_traversal_are_refused_before_extraction(self): + for name,kind in [('../escape',tarfile.REGTYPE),('link',tarfile.SYMTYPE)]: + data=io.BytesIO() + with tarfile.open(fileobj=data,mode='w') as archive: + info=tarfile.TarInfo(name);info.type=kind;info.linkname='outside' + archive.addfile(info) + with tempfile.TemporaryDirectory() as temporary: + with self.assertRaisesRegex(RuntimeError,'archive'): + extract_tar(data.getvalue(),Path(temporary)/'source') + + def test_cache_preserves_spaces_and_semicolon_paths(self): + with tempfile.TemporaryDirectory() as temporary: + path=Path(temporary)/'CMakeCache.txt' + path.write_text('// comment\nCMAKE_CXX_FLAGS:STRING=-fsanitize=address,undefined -fno-omit-frame-pointer\nCMAKE_PREFIX_PATH:PATH=/one;/two\n') + found=read_cache(path) + self.assertEqual(found['CMAKE_CXX_FLAGS'],'-fsanitize=address,undefined -fno-omit-frame-pointer') + self.assertEqual(found['CMAKE_PREFIX_PATH'],'/one;/two') + + +if __name__=='__main__': + unittest.main() diff --git a/src/engine_execution.cpp b/src/engine_execution.cpp index 1a344d6c1..e3e1cfa8a 100644 --- a/src/engine_execution.cpp +++ b/src/engine_execution.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace pineforge { @@ -48,6 +49,43 @@ CloseScopeInspection inspect_close_scope( return {execution::Status::Applied, held}; } +CloseScopeInspection inspect_selected_opening_set( + const execution::SelectedOpeningSet& selection, + const execution::Action& action, + int64_t current_cycle, const std::vector& lots, + std::unordered_set& ids_out) { + ids_out.clear(); + if (std::holds_alternative(action) + || selection.cycle <= 0 || selection.cycle != current_cycle + || selection.incarnations.empty()) + return {execution::Status::InvalidCloseTarget}; + std::unordered_set live; + live.reserve(lots.size()); + for (const auto& lot : lots) live.insert(lot.entry_incarnation); + ids_out.reserve(selection.incarnations.size()); + for (const auto id : selection.incarnations) { + if (id == 0 || !ids_out.insert(id).second + || live.find(id) == live.end()) { + ids_out.clear(); + return {execution::Status::InvalidCloseTarget}; + } + } + double held = 0.0; + for (const auto& lot : lots) { + if (ids_out.count(lot.entry_incarnation) == 0) continue; + held += lot.qty; + if (!std::isfinite(held)) { + ids_out.clear(); + return {execution::Status::InvalidBook}; + } + } + if (held == 0.0) { + ids_out.clear(); + return {execution::Status::InvalidCloseTarget}; + } + return {execution::Status::Applied, held}; +} + bool selected_for_close(const execution::CloseScope& scope, const PyramidEntry& lot) { if (std::holds_alternative(scope)) return true; @@ -84,8 +122,228 @@ CloseSplit next_close_split(const PyramidEntry& lot, bool closes, bool flatten, closed = next_closed; return {execution::Status::Applied, amount, kept}; } + +execution::AccountEffectProjection invalid_projection(execution::Status status) { + execution::AccountEffectProjection out; + out.status = status; + return out; +} } // namespace +struct BacktestEngine::NativeSettlementStage { + enum class Phase { Invalid, NoEffect, Ready }; + Phase phase = Phase::Invalid; + execution::Status status = execution::Status::NoEffect; + bool flatten = false; + bool scoped = false; + bool opposite = false; + bool closes = false; + bool use_selected = false; + PositionSide incoming = PositionSide::LONG; + bool was_long = false; + double requested = 0.0; + double selected_held = 0.0; + double allocation_requested = 0.0; + double closed = 0.0; + double opening = 0.0; + double ticket = 0.0; + double after_qty = 0.0; + double after_price = 0.0; + std::vector closing_indices; + std::vector closing_quantities; + std::vector current_costs; + std::vector survivors; + std::unordered_set selected_ids; +}; + +void BacktestEngine::stage_native_settlement( + NativeSettlementStage& stage, + const execution::Action& action, + const execution::Fill& fill, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected, + const execution::LifecycleEffects* lifecycle) const { + using execution::Status; + stage = NativeSettlementStage{}; + auto fail = [&](Status status) { + stage.phase = NativeSettlementStage::Phase::Invalid; + stage.status = status; + }; + auto no_effect = [&]() { + stage.phase = NativeSettlementStage::Phase::NoEffect; + stage.status = fill.commission_account && *fill.commission_account != 0.0 + ? Status::InvalidAccounting : Status::NoEffect; + }; + + if (!std::isfinite(fill.price)) { fail(Status::InvalidPrice); return; } + if (fill.commission_account && !std::isfinite(*fill.commission_account)) { + fail(Status::InvalidAccounting); + return; + } + + stage.flatten = std::holds_alternative(action); + const auto* reduce = std::get_if(&action); + const auto* transact = std::get_if(&action); + stage.requested = reduce ? reduce->units + : transact ? std::abs(transact->signed_units) : 0.0; + if (!std::isfinite(stage.requested) || (reduce && stage.requested < 0.0)) { + fail(Status::InvalidQuantity); + return; + } + if (position_side_ != PositionSide::FLAT + && position_side_ != PositionSide::LONG + && position_side_ != PositionSide::SHORT) { + fail(Status::InvalidBook); + return; + } + if ((position_side_ == PositionSide::FLAT) != pyramid_entries_.empty()) { + fail(Status::InvalidBook); + return; + } + + double held = 0.0; + for (const auto& lot : pyramid_entries_) { + if (!std::isfinite(lot.qty) || lot.qty <= 0.0 || !std::isfinite(lot.price)) { + fail(Status::InvalidBook); + return; + } + held += lot.qty; + if (!std::isfinite(held)) { + fail(Status::InvalidBook); + return; + } + } + if (lifecycle) { + if (auto invalid = validate_lifecycle_effects(*lifecycle)) { + fail(*invalid); + return; + } + } + + CloseScopeInspection selection; + if (selected) { + selection = inspect_selected_opening_set( + *selected, action, position_cycle_seq_, pyramid_entries_, + stage.selected_ids); + stage.use_selected = true; + stage.scoped = true; + } else { + selection = inspect_close_scope( + book_or_opening, action, position_cycle_seq_, pyramid_entries_, held); + stage.scoped = std::holds_alternative( + book_or_opening); + } + if (selection.status != Status::Applied) { + fail(selection.status); + return; + } + stage.selected_held = selection.held; + stage.allocation_requested = stage.scoped && reduce + ? std::min(stage.requested, selection.held) : stage.requested; + const double signed_held = position_side_ == PositionSide::SHORT + ? -selection.held : selection.held; + if (!stage.flatten && stage.requested == 0.0) { no_effect(); return; } + if ((stage.flatten || reduce) && pyramid_entries_.empty()) { + no_effect(); + return; + } + if (reduce && !order_action::plan(signed_held, + order_action::Reduce{stage.allocation_requested})) { + fail(Status::UnrepresentableQuantity); + return; + } + if (transact && !order_action::plan(signed_held, *transact)) { + fail(Status::UnrepresentableQuantity); + return; + } + + stage.incoming = transact && transact->signed_units < 0.0 + ? PositionSide::SHORT : PositionSide::LONG; + stage.opposite = transact && position_side_ != PositionSide::FLAT + && position_side_ != stage.incoming; + stage.closes = stage.flatten || reduce || stage.opposite; + stage.was_long = position_side_ == PositionSide::LONG; + + if (!stage.flatten || stage.scoped) + stage.survivors.reserve(pyramid_entries_.size()); + stage.closing_indices.reserve(stage.closes ? pyramid_entries_.size() : 0); + stage.closing_quantities.reserve(stage.closes ? pyramid_entries_.size() : 0); + double closed = 0.0; + double remaining = stage.allocation_requested; + for (size_t index = 0; index < pyramid_entries_.size(); ++index) { + const auto& lot = pyramid_entries_[index]; + const bool member = stage.use_selected + ? stage.selected_ids.count(lot.entry_incarnation) != 0 + : selected_for_close(book_or_opening, lot); + const auto split = next_close_split( + lot, stage.closes && member, stage.flatten, + stage.allocation_requested, closed, remaining); + if (split.status != Status::Applied) { + fail(split.status); + return; + } + if (split.amount == 0.0) { + stage.survivors.push_back(lot); + continue; + } + stage.closing_indices.push_back(index); + stage.closing_quantities.push_back(split.amount); + if (split.kept > 0.0) { + auto survivor = lot; + const double scale = split.kept / lot.qty; + survivor.qty = split.kept; + survivor.max_runup *= scale; + survivor.max_drawdown *= scale; + survivor.entry_commission_account = open_entry_commission(lot) + - allocated_entry_commission(lot, split.amount); + stage.survivors.push_back(std::move(survivor)); + } + } + stage.closed = closed; + stage.opening = !transact ? 0.0 : stage.opposite ? remaining : stage.requested; + if (stage.opening > 0.0 && stage.opposite && !stage.survivors.empty()) { + fail(Status::UnrepresentableQuantity); + return; + } + stage.current_costs = quote_execution_commissions( + stage.closing_quantities, stage.opening, fill); + stage.ticket = 0.0; + for (double cost : stage.current_costs) { + if (!std::isfinite(cost)) { + fail(Status::InvalidAccounting); + return; + } + stage.ticket += cost; + if (!std::isfinite(stage.ticket)) { + fail(Status::InvalidAccounting); + return; + } + } + double after_qty = 0.0; + double weighted = 0.0; + for (const auto& lot : stage.survivors) { + after_qty += lot.qty; + weighted += lot.price * lot.qty; + } + if (stage.opening > 0.0) { + const double next = after_qty + stage.opening; + if (!std::isfinite(next) || (after_qty > 0.0 + && (next == after_qty || next == stage.opening))) { + fail(Status::UnrepresentableQuantity); + return; + } + after_qty = next; + weighted += fill.price * stage.opening; + } + const size_t after_lots = stage.survivors.size() + (stage.opening > 0.0 ? 1 : 0); + stage.after_qty = after_qty; + stage.after_price = after_lots == 1 + ? (stage.opening > 0.0 ? fill.price : stage.survivors.front().price) + : after_qty > 0.0 ? weighted / after_qty : 0.0; + stage.phase = NativeSettlementStage::Phase::Ready; + stage.status = Status::Applied; +} + execution::Result BacktestEngine::settle_resolved_execution( const execution::Action& action, const execution::Fill& fill) { return settle_execution_with_lifecycle(action, fill, {}); @@ -124,151 +382,82 @@ execution::Result BacktestEngine::settle_with_context( return settle_with_context_scoped(action, fill, lifecycle, context, execution::Book{}); } +execution::Result BacktestEngine::settle_native_execution_selected_at( + const execution::Action& action, const execution::Fill& fill, + const execution::PhysicalExecutionContext& context, + const execution::SelectedOpeningSet& selection) { + return settle_with_context_selected(action, fill, {}, context, selection); +} + +execution::Result BacktestEngine::settle_execution_selected_with_lifecycle( + const execution::Action& action, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::SelectedOpeningSet& selection) { + execution::PhysicalExecutionContext context; + context.effective_time_ms = current_bar_.timestamp; + context.interval_index = bar_index_; + context.preceding_exit_path_prefix = fold_exit_path_extremes_; + if (!std::isnan(fold_exit_trail_peak_)) { + context.preceding_exit_trail_peak = fold_exit_trail_peak_; + } + return settle_with_context_selected(action, fill, lifecycle, context, selection); +} + execution::Result BacktestEngine::settle_with_context_scoped( const execution::Action& action, const execution::Fill& fill, const execution::LifecycleEffects& lifecycle, const execution::PhysicalExecutionContext& context, execution::CloseScope scope) { + return settle_with_membership(action, fill, lifecycle, context, scope, nullptr); +} + +execution::Result BacktestEngine::settle_with_context_selected( + const execution::Action& action, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::PhysicalExecutionContext& context, + const execution::SelectedOpeningSet& selection) { + return settle_with_membership( + action, fill, lifecycle, context, execution::Book{}, &selection); +} + +execution::Result BacktestEngine::settle_with_membership( + const execution::Action& action, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::PhysicalExecutionContext& context, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected) { using execution::Status; - if (!std::isfinite(fill.price)) return {Status::InvalidPrice}; - if (fill.commission_account && !std::isfinite(*fill.commission_account)) - return {Status::InvalidAccounting}; + NativeSettlementStage stage; + stage_native_settlement( + stage, action, fill, book_or_opening, selected, &lifecycle); + if (stage.phase != NativeSettlementStage::Phase::Ready) + return {stage.status}; const auto no_effect = [&]() -> execution::Result { return {fill.commission_account && *fill.commission_account != 0.0 ? Status::InvalidAccounting : Status::NoEffect}; }; + if (stage.scoped && stage.closed == 0.0) return no_effect(); - const bool flatten = std::holds_alternative(action); - const auto* reduce = std::get_if(&action); - const auto* transact = std::get_if(&action); - const double requested = reduce ? reduce->units - : transact ? std::abs(transact->signed_units) : 0.0; - if (!std::isfinite(requested) || (reduce && requested < 0.0)) - return {Status::InvalidQuantity}; - - if (position_side_ != PositionSide::FLAT - && position_side_ != PositionSide::LONG - && position_side_ != PositionSide::SHORT) - return {Status::InvalidBook}; - if ((position_side_ == PositionSide::FLAT) != pyramid_entries_.empty()) - return {Status::InvalidBook}; - - double held = 0.0; - for (const auto& lot : pyramid_entries_) { - if (!std::isfinite(lot.qty) || lot.qty <= 0.0 || !std::isfinite(lot.price)) - return {Status::InvalidBook}; - held += lot.qty; - if (!std::isfinite(held)) return {Status::InvalidBook}; - } - if (auto invalid = validate_lifecycle_effects(lifecycle)) - return {*invalid}; - const auto selection = inspect_close_scope( - scope, action, position_cycle_seq_, pyramid_entries_, held); - if (selection.status != Status::Applied) return {selection.status}; - const bool scoped = std::holds_alternative(scope); - const double allocation_requested = scoped && reduce - ? std::min(requested, selection.held) : requested; - const double signed_held = position_side_ == PositionSide::SHORT - ? -selection.held : selection.held; - if (!flatten && requested == 0.0) return no_effect(); - if ((flatten || reduce) && pyramid_entries_.empty()) return no_effect(); - - // The scalar planner validates the request's representability. Physical - // FIFO allocation below is authoritative for the surviving position: it - // must not reject a valid decimal lot merely because a differently grouped - // scalar addition has a one-ulp difference. - if (reduce && !order_action::plan(signed_held, order_action::Reduce{allocation_requested})) - return {Status::UnrepresentableQuantity}; - if (transact && !order_action::plan(signed_held, *transact)) - return {Status::UnrepresentableQuantity}; - - const PositionSide incoming = transact && transact->signed_units < 0.0 - ? PositionSide::SHORT : PositionSide::LONG; - const bool opposite = transact && position_side_ != PositionSide::FLAT - && position_side_ != incoming; - const bool closes = flatten || reduce || opposite; - const bool was_long = position_side_ == PositionSide::LONG; - - // Stage all physical splits and financial rows before changing the book. - std::vector survivors; - std::vector closing_indices; - std::vector closing_quantities; - if (!flatten || scoped) survivors.reserve(pyramid_entries_.size()); - closing_indices.reserve(closes ? pyramid_entries_.size() : 0); - closing_quantities.reserve(closes ? pyramid_entries_.size() : 0); - double closed = 0.0; - double remaining = allocation_requested; - for (size_t index = 0; index < pyramid_entries_.size(); ++index) { - const auto& lot = pyramid_entries_[index]; - const auto split = next_close_split(lot, closes && selected_for_close(scope, lot), - flatten, allocation_requested, closed, remaining); - if (split.status != Status::Applied) return {split.status}; - const double amount = split.amount; - if (amount == 0.0) { - survivors.push_back(lot); - continue; - } - const double kept = split.kept; - closing_indices.push_back(index); - closing_quantities.push_back(amount); - if (kept > 0.0) { - auto survivor = lot; - const double scale = kept / lot.qty; - survivor.qty = kept; - survivor.max_runup *= scale; - survivor.max_drawdown *= scale; - survivor.entry_commission_account = open_entry_commission(lot) - - allocated_entry_commission(lot, amount); - survivors.push_back(std::move(survivor)); - } - } - - if (scoped && closed == 0.0) return no_effect(); - const double opening = !transact ? 0.0 : opposite ? remaining : requested; - if (opening > 0.0 && opposite && !survivors.empty()) - return {Status::UnrepresentableQuantity}; - const auto current_costs = quote_execution_commissions(closing_quantities, opening, fill); - for (double cost : current_costs) - if (!std::isfinite(cost)) return {Status::InvalidAccounting}; - const double opening_commission = current_costs.back(); std::vector closed_trades; - closed_trades.reserve(closing_indices.size()); - for (size_t i = 0; i < closing_indices.size(); ++i) { - const auto& lot = pyramid_entries_[closing_indices[i]]; - auto trade = build_close_trade_with_costs(lot, closing_quantities[i], - fill.price, was_long, allocated_entry_commission(lot, closing_quantities[i]), - current_costs[i], context); + closed_trades.reserve(stage.closing_indices.size()); + for (size_t i = 0; i < stage.closing_indices.size(); ++i) { + const auto& lot = pyramid_entries_[stage.closing_indices[i]]; + auto trade = build_close_trade_with_costs(lot, stage.closing_quantities[i], + fill.price, stage.was_long, + allocated_entry_commission(lot, stage.closing_quantities[i]), + stage.current_costs[i], context); trade.exit_id = fill.id; trade.exit_comment = fill.comment; closed_trades.push_back(std::move(trade)); } - if (opening > 0.0 && (position_side_ == PositionSide::FLAT || survivors.empty()) + if (stage.opening > 0.0 + && (position_side_ == PositionSide::FLAT || stage.survivors.empty()) && (next_position_cycle_seq_ <= 0 || next_position_cycle_seq_ == std::numeric_limits::max())) throw std::overflow_error("position cycle sequence exhausted"); - double after_qty = 0.0; - double weighted = 0.0; - for (const auto& lot : survivors) { - after_qty += lot.qty; - weighted += lot.price * lot.qty; - } - if (opening > 0.0) { - const double next = after_qty + opening; - if (!std::isfinite(next) || (after_qty > 0.0 - && (next == after_qty || next == opening))) - return {Status::UnrepresentableQuantity}; - after_qty = next; - weighted += fill.price * opening; - } - const size_t after_lots = survivors.size() + (opening > 0.0 ? 1 : 0); - const double after_price = after_lots == 1 - ? (opening > 0.0 ? fill.price : survivors.front().price) - : after_qty > 0.0 ? weighted / after_qty : 0.0; - if (!std::isfinite(after_qty) || !std::isfinite(after_price)) + if (!std::isfinite(stage.after_qty) || !std::isfinite(stage.after_price)) return {Status::InvalidAccounting}; - // A financial sum must remain representable; reporting-only percentages - // or excursion projections do not decide whether an action may settle. double next_profit = net_profit_sum_; double next_gross_profit = gross_profit_sum_; double next_gross_loss = gross_loss_sum_; @@ -285,27 +474,27 @@ execution::Result BacktestEngine::settle_with_context_scoped( || !std::isfinite(next_gross_loss) || !std::isfinite(next_intraday)) return {Status::InvalidAccounting}; validate_close_trade_counters(closed_trades.data(), closed_trades.size()); - if (opening > 0.0 && !survivors.empty() + if (stage.opening > 0.0 && !stage.survivors.empty() && position_entry_count_ == std::numeric_limits::max()) throw std::overflow_error("position entry counter exhausted"); - const bool will_reset = closed > 0.0 && survivors.empty(); - const bool will_open_quoted = opening > 0.0 - && (position_side_ == PositionSide::FLAT || survivors.empty()); + const bool will_reset = stage.closed > 0.0 && stage.survivors.empty(); + const bool will_open_quoted = stage.opening > 0.0 + && (position_side_ == PositionSide::FLAT || stage.survivors.empty()); if (auto invalid = preflight_settlement_lifecycle( lifecycle, will_reset, will_open_quoted)) return {*invalid}; const size_t first_trade = trades_.size(); const size_t first_action = stream_order_actions_.size(); - const size_t events = closed_trades.size() + (opening > 0.0 ? 1 : 0); + const size_t events = closed_trades.size() + (stage.opening > 0.0 ? 1 : 0); if (stream_observe_actions_) { if (events > std::numeric_limits::max() - stream_action_sequence_) throw std::overflow_error("stream action sequence overflow"); reserve_effects(stream_order_actions_, events); } reserve_effects(trades_, closed_trades.size()); - if (opening > 0.0) reserve_effects(pyramid_entries_, 1); + if (stage.opening > 0.0) reserve_effects(pyramid_entries_, 1); // Commit through the existing accounting/observation sinks. Allocation or // lifecycle exceptions still abort the owning engine run; this internal @@ -314,40 +503,40 @@ execution::Result BacktestEngine::settle_with_context_scoped( // unbind, authorized pending removals, then quoted opening bind. if (lifecycle.pre_close) apply_pre_close_lifecycle_batch(*lifecycle.pre_close); for (auto& trade : closed_trades) record_close_trade(std::move(trade)); - if (closed > 0.0) { - if (survivors.empty()) { + if (stage.closed > 0.0) { + if (stage.survivors.empty()) { reset_position_state_to_flat(); } else { - pyramid_entries_ = std::move(survivors); - position_qty_ = after_qty; - position_entry_price_ = after_price; + pyramid_entries_ = std::move(stage.survivors); + position_qty_ = stage.after_qty; + position_entry_price_ = stage.after_price; position_entry_count_ = static_cast(pyramid_entries_.size()); } } apply_authorized_pending_removals(lifecycle.removals); - if (opening > 0.0) { - PyramidEntry lot{fill.price, context.effective_time_ms, opening, fill.id, + if (stage.opening > 0.0) { + const double opening_commission = stage.current_costs.back(); + PyramidEntry lot{fill.price, context.effective_time_ms, stage.opening, fill.id, context.interval_index}; lot.entry_incarnation = fill.incarnation; lot.entry_comment = fill.comment; lot.entry_commission_account = opening_commission; if (position_side_ == PositionSide::FLAT) { - open_quoted_position(incoming, std::move(lot)); + open_quoted_position(stage.incoming, std::move(lot)); } else { - append_quoted_lot(std::move(lot), after_qty, after_price); + append_quoted_lot(std::move(lot), stage.after_qty, stage.after_price); } } if (stream_observe_actions_) stream_refresh_action_metadata(first_action, first_trade); - double ticket = 0.0; - for (double cost : current_costs) ticket += cost; execution::Result applied; applied.status = Status::Applied; - applied.closed_units = closed; - applied.opened_units = incoming == PositionSide::SHORT ? -opening : opening; - applied.current_ticket = ticket; + applied.closed_units = stage.closed; + applied.opened_units = stage.incoming == PositionSide::SHORT + ? -stage.opening : stage.opening; + applied.current_ticket = stage.ticket; applied.first_trade_index = first_trade; applied.closed_trade_count = closed_trades.size(); - applied.opened_lot_incarnation = opening > 0.0 ? fill.incarnation : 0; + applied.opened_lot_incarnation = stage.opening > 0.0 ? fill.incarnation : 0; return applied; } @@ -359,147 +548,202 @@ execution::SettlementInspection BacktestEngine::inspect_native_settlement( execution::SettlementInspection BacktestEngine::inspect_native_settlement_scoped( const execution::Action& action, const execution::Fill& fill, execution::CloseScope scope) const { + return inspect_with_membership(action, fill, scope, nullptr); +} + +execution::SettlementInspection BacktestEngine::inspect_native_settlement_selected( + const execution::Action& action, const execution::Fill& fill, + const execution::SelectedOpeningSet& selection) const { + return inspect_with_membership(action, fill, execution::Book{}, &selection); +} + +execution::SettlementInspection BacktestEngine::inspect_with_membership( + const execution::Action& action, const execution::Fill& fill, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected) const { using execution::Status; + NativeSettlementStage stage; + stage_native_settlement( + stage, action, fill, book_or_opening, selected, nullptr); execution::SettlementInspection out; - if (!std::isfinite(fill.price)) { - out.status = Status::InvalidPrice; + if (stage.phase != NativeSettlementStage::Phase::Ready) { + out.status = stage.status; return out; } - if (fill.commission_account && !std::isfinite(*fill.commission_account)) { - out.status = Status::InvalidAccounting; - return out; - } - const bool flatten = std::holds_alternative(action); - const auto* reduce = std::get_if(&action); - const auto* transact = std::get_if(&action); - const double requested = reduce ? reduce->units - : transact ? std::abs(transact->signed_units) : 0.0; - if (!std::isfinite(requested) || (reduce && requested < 0.0)) { - out.status = Status::InvalidQuantity; - return out; - } - if (position_side_ != PositionSide::FLAT - && position_side_ != PositionSide::LONG - && position_side_ != PositionSide::SHORT) { - out.status = Status::InvalidBook; - return out; - } - if ((position_side_ == PositionSide::FLAT) != pyramid_entries_.empty()) { - out.status = Status::InvalidBook; - return out; - } - double held = 0.0; - for (const auto& lot : pyramid_entries_) { - if (!std::isfinite(lot.qty) || lot.qty <= 0.0 || !std::isfinite(lot.price)) { - out.status = Status::InvalidBook; - return out; - } - held += lot.qty; - if (!std::isfinite(held)) { - out.status = Status::InvalidBook; - return out; - } - } - const auto selection = inspect_close_scope( - scope, action, position_cycle_seq_, pyramid_entries_, held); - if (selection.status != Status::Applied) { - out.status = selection.status; - return out; - } - const bool scoped = std::holds_alternative(scope); - const double allocation_requested = scoped && reduce - ? std::min(requested, selection.held) : requested; - const double signed_held = position_side_ == PositionSide::SHORT - ? -selection.held : selection.held; - if (!flatten && requested == 0.0) { - out.status = fill.commission_account && *fill.commission_account != 0.0 - ? Status::InvalidAccounting : Status::NoEffect; - return out; - } - if ((flatten || reduce) && pyramid_entries_.empty()) { + const double fx = active_account_currency_fx(); + out.status = Status::Applied; + out.closed_units = stage.closed; + out.opened_units = stage.incoming == PositionSide::SHORT + ? -stage.opening : stage.opening; + out.resulting_abs_units = stage.after_qty; + out.resulting_lot_count = stage.survivors.size() + (stage.opening > 0.0 ? 1 : 0); + out.resulting_abs_notional = stage.after_qty * std::abs(fill.price) + * syminfo_.pointvalue * fx; + out.current_ticket = stage.ticket; + out.would_open = stage.opening > 0.0; + out.incoming_short = stage.incoming == PositionSide::SHORT; + if (stage.closed == 0.0 && stage.opening == 0.0) { out.status = fill.commission_account && *fill.commission_account != 0.0 ? Status::InvalidAccounting : Status::NoEffect; - return out; - } - if (reduce && !order_action::plan(signed_held, order_action::Reduce{allocation_requested})) { - out.status = Status::UnrepresentableQuantity; - return out; + out.would_open = false; } - if (transact && !order_action::plan(signed_held, *transact)) { - out.status = Status::UnrepresentableQuantity; + return out; +} + +execution::AccountEffectProjection BacktestEngine::project_native_settlement_v1( + const execution::Action& action, const execution::Fill& fill) const { + return project_native_settlement_scoped_v1(action, fill, execution::Book{}); +} + +execution::AccountEffectProjection BacktestEngine::project_native_settlement_scoped_v1( + const execution::Action& action, const execution::Fill& fill, + execution::CloseScope scope) const { + return project_with_membership(action, fill, scope, nullptr); +} + +execution::AccountEffectProjection BacktestEngine::project_native_settlement_selected_v1( + const execution::Action& action, const execution::Fill& fill, + const execution::SelectedOpeningSet& selection) const { + return project_with_membership(action, fill, execution::Book{}, &selection); +} + +execution::AccountEffectProjection BacktestEngine::project_with_membership( + const execution::Action& action, const execution::Fill& fill, + execution::CloseScope book_or_opening, + const execution::SelectedOpeningSet* selected) const { + using execution::Status; + NativeSettlementStage stage; + stage_native_settlement( + stage, action, fill, book_or_opening, selected, nullptr); + if (stage.phase == NativeSettlementStage::Phase::Invalid) + return invalid_projection(stage.status); + + const auto unchanged_account = [&]() -> execution::AccountEffectProjection { + execution::AccountEffectProjection out; + out.status = Status::NoEffect; + const double fx = active_account_currency_fx(); + const double pv = syminfo_.pointvalue; + double abs_units = 0.0; + for (const auto& lot : pyramid_entries_) abs_units += lot.qty; + out.resulting_abs_units = abs_units; + out.resulting_lot_count = pyramid_entries_.size(); + out.resulting_abs_notional = + abs_units * std::abs(fill.price) * pv * fx; + double remaining = 0.0; + for (const auto& lot : pyramid_entries_) + remaining += open_entry_commission(lot); + out.realized_balance = initial_capital_ + net_profit_sum_; + out.remaining_entry_cost = remaining; + out.marked_equity = marked_equity(fill.price); + out.cycle_after = pyramid_entries_.empty() ? 0 : position_cycle_seq_; + out.signed_units_after = position_side_ == PositionSide::SHORT + ? -abs_units + : position_side_ == PositionSide::LONG ? abs_units : 0.0; + if (!std::isfinite(out.resulting_abs_units) + || !std::isfinite(out.resulting_abs_notional) + || !std::isfinite(out.realized_balance) + || !std::isfinite(out.remaining_entry_cost) + || !std::isfinite(out.marked_equity) + || !std::isfinite(out.signed_units_after)) + return invalid_projection(Status::InvalidAccounting); return out; + }; + + if (stage.phase == NativeSettlementStage::Phase::NoEffect) { + if (stage.status != Status::NoEffect) + return invalid_projection(stage.status); + return unchanged_account(); } - const PositionSide incoming = transact && transact->signed_units < 0.0 - ? PositionSide::SHORT : PositionSide::LONG; - const bool opposite = transact && position_side_ != PositionSide::FLAT - && position_side_ != incoming; - const bool closes = flatten || reduce || opposite; - std::vector survivors; - std::vector closing_quantities; - if (!flatten || scoped) survivors.reserve(pyramid_entries_.size()); - closing_quantities.reserve(closes ? pyramid_entries_.size() : 0); - double closed = 0.0; - double remaining = allocation_requested; - for (const auto& lot : pyramid_entries_) { - const auto split = next_close_split(lot, closes && selected_for_close(scope, lot), - flatten, allocation_requested, closed, remaining); - if (split.status != Status::Applied) { - out.status = split.status; - return out; - } - const double amount = split.amount; - if (amount == 0.0) { - survivors.push_back(lot); - continue; - } - const double kept = split.kept; - closing_quantities.push_back(amount); - if (kept > 0.0) { - auto survivor = lot; - survivor.qty = kept; - survivors.push_back(std::move(survivor)); - } - } - const double opening = !transact ? 0.0 : opposite ? remaining : requested; - if (opening > 0.0 && opposite && !survivors.empty()) { - out.status = Status::UnrepresentableQuantity; - return out; + if (stage.closed == 0.0 && stage.opening == 0.0) { + if (fill.commission_account && *fill.commission_account != 0.0) + return invalid_projection(Status::InvalidAccounting); + return unchanged_account(); } - const auto current_costs = quote_execution_commissions(closing_quantities, opening, fill); - double ticket = 0.0; - for (double cost : current_costs) { - if (!std::isfinite(cost)) { - out.status = Status::InvalidAccounting; - return out; - } - ticket += cost; + + if (stage.opening > 0.0 + && (position_side_ == PositionSide::FLAT || stage.survivors.empty()) + && (next_position_cycle_seq_ <= 0 + || next_position_cycle_seq_ == std::numeric_limits::max())) + throw std::overflow_error("position cycle sequence exhausted"); + + execution::PhysicalExecutionContext context; + std::vector closed_trades; + closed_trades.reserve(stage.closing_indices.size()); + for (size_t i = 0; i < stage.closing_indices.size(); ++i) { + const auto& lot = pyramid_entries_[stage.closing_indices[i]]; + auto trade = build_close_trade_with_costs(lot, stage.closing_quantities[i], + fill.price, stage.was_long, + allocated_entry_commission(lot, stage.closing_quantities[i]), + stage.current_costs[i], context); + closed_trades.push_back(std::move(trade)); } - double after_qty = 0.0; - for (const auto& lot : survivors) after_qty += lot.qty; - if (opening > 0.0) { - const double next = after_qty + opening; - if (!std::isfinite(next) || (after_qty > 0.0 - && (next == after_qty || next == opening))) { - out.status = Status::UnrepresentableQuantity; - return out; - } - after_qty = next; + + double realized = net_profit_sum_; + for (const auto& trade : closed_trades) { + if (!std::isfinite(trade.pnl) || !std::isfinite(trade.commission)) + return invalid_projection(Status::InvalidAccounting); + realized += trade.pnl; } + realized += initial_capital_; + if (!std::isfinite(realized)) + return invalid_projection(Status::InvalidAccounting); + const double fx = active_account_currency_fx(); + const double pv = syminfo_.pointvalue; + const PositionSide resulting_side = !stage.survivors.empty() + ? position_side_ + : stage.opening > 0.0 ? stage.incoming : PositionSide::FLAT; + const double direction = resulting_side == PositionSide::SHORT ? -1.0 : 1.0; + double remaining = 0.0; + double equity = realized; + for (const auto& lot : stage.survivors) { + if (!std::isfinite(lot.qty) || lot.qty <= 0.0 || !std::isfinite(lot.price)) + return invalid_projection(Status::InvalidAccounting); + const double paid = open_entry_commission(lot); + remaining += paid; + equity += direction * (fill.price - lot.price) * lot.qty * pv * fx - paid; + } + if (stage.opening > 0.0) { + const double paid = stage.current_costs.back(); + remaining += paid; + equity += direction * (fill.price - fill.price) * stage.opening * pv * fx + - paid; + } + if (!std::isfinite(remaining) || !std::isfinite(equity) + || !std::isfinite(stage.after_qty) + || !std::isfinite(stage.ticket)) + return invalid_projection(Status::InvalidAccounting); + + execution::AccountEffectProjection out; out.status = Status::Applied; - out.closed_units = closed; - out.opened_units = incoming == PositionSide::SHORT ? -opening : opening; - out.resulting_abs_units = after_qty; - out.resulting_lot_count = survivors.size() + (opening > 0.0 ? 1 : 0); - out.resulting_abs_notional = after_qty * std::abs(fill.price) * syminfo_.pointvalue * fx; - out.current_ticket = ticket; - out.would_open = opening > 0.0; - out.incoming_short = incoming == PositionSide::SHORT; - if (closed == 0.0 && opening == 0.0) { - out.status = fill.commission_account && *fill.commission_account != 0.0 - ? Status::InvalidAccounting : Status::NoEffect; - out.would_open = false; - } + out.closed_units = stage.closed; + out.opened_units = stage.incoming == PositionSide::SHORT + ? -stage.opening : stage.opening; + out.resulting_abs_units = stage.after_qty; + out.resulting_lot_count = stage.survivors.size() + (stage.opening > 0.0 ? 1 : 0); + out.resulting_abs_notional = + stage.after_qty * std::abs(fill.price) * pv * fx; + out.current_ticket = stage.ticket; + out.would_open = stage.opening > 0.0; + out.incoming_short = stage.incoming == PositionSide::SHORT; + out.realized_balance = realized; + out.remaining_entry_cost = remaining; + out.marked_equity = equity; + if (out.resulting_lot_count == 0) { + out.cycle_after = 0; + out.signed_units_after = 0.0; + } else if (!stage.survivors.empty()) { + out.cycle_after = position_cycle_seq_; + out.signed_units_after = position_side_ == PositionSide::SHORT + ? -stage.after_qty : stage.after_qty; + } else { + out.cycle_after = next_position_cycle_seq_; + out.signed_units_after = stage.incoming == PositionSide::SHORT + ? -stage.after_qty : stage.after_qty; + } + if (!std::isfinite(out.resulting_abs_notional) + || !std::isfinite(out.signed_units_after)) + return invalid_projection(Status::InvalidAccounting); return out; } diff --git a/src/engine_fills.cpp b/src/engine_fills.cpp index 2ee91d805..ef671c616 100644 --- a/src/engine_fills.cpp +++ b/src/engine_fills.cpp @@ -29,6 +29,20 @@ using namespace internal; namespace { +std::size_t source_opening_fragment_count(const std::vector& lots, + uint64_t incarnation) { + return static_cast(std::count_if(lots.begin(), lots.end(), + [&](const PyramidEntry& lot) { return lot.entry_incarnation == incarnation; })); +} + +bool source_opening_was_created(const std::vector& lots, + uint64_t incarnation, int64_t cycle_before, + int64_t cycle_after, std::size_t fragments_before) { + return !lots.empty() && lots.back().entry_incarnation == incarnation + && (cycle_after != cycle_before + || source_opening_fragment_count(lots, incarnation) == fragments_before + 1); +} + // A pass keeps identities and ordering hints, never borrowed vector elements. // The hint makes the unchanged-book path constant time; OCA erasure requires // re-resolution by incarnation. A reused label/priority cannot match this key. @@ -6429,6 +6443,10 @@ void BacktestEngine::apply_filled_order_to_state( || std::abs(position_qty_ - position_qty_before_fill) > kQtyEpsilon || pyramid_entries_.size() != pyramid_lots_before_fill || trades_.size() != trades_before; + const bool opening_fill_applied = !pyramid_entries_.empty() + && pyramid_entries_.back().entry_incarnation == order.incarnation + && (position_cycle_seq_ != position_cycle_before_fill + || pyramid_entries_.size() > pyramid_lots_before_fill); max_intraday_filled_orders_.outcome( primary_fill_applied ? compat::pine::FillOutcome::Committed @@ -6456,7 +6474,7 @@ void BacktestEngine::apply_filled_order_to_state( } if (primary_fill_applied) { - if (order.type == OrderType::MARKET && process_orders_on_close_ + if (opening_fill_applied && order.type == OrderType::MARKET && process_orders_on_close_ && order.created_bar == bar_index_ && order.created_position_side == PositionSide::FLAT && !placement_has_prior_close(order) @@ -6471,7 +6489,7 @@ void BacktestEngine::apply_filled_order_to_state( || (coof_scheduler_active_ && coof_cursor_is_bar_close_))) { pyramid_entries_.front().pooc_terminal_market_entry = true; } - if (order.type == OrderType::MARKET + if (opening_fill_applied && order.type == OrderType::MARKET && !process_orders_on_close_ && !calc_on_order_fills_ && !bar_magnifier_enabled_ && !coof_scheduler_active_ && !stream_warmup_mode_ && stream_phase_ == StreamPhase::IDLE @@ -6482,7 +6500,7 @@ void BacktestEngine::apply_filled_order_to_state( && position_entry_price_ == round_to_mintick(bar.open)) { pyramid_entries_.front().ordinary_market_open = true; } - if (order.type == OrderType::ENTRY + if (opening_fill_applied && order.type == OrderType::ENTRY && std::isfinite(order.legs.prices().stop_price) && std::isnan(order.legs.prices().limit_price) && !order.stop_limit_activated @@ -6964,6 +6982,13 @@ void BacktestEngine::apply_market_order_fill(PendingOrder& order, double fill_pr const Bar& bar, double& trail_best_path_state, bool later_same_tick_entry) { + const int64_t source_cycle_before = position_cycle_seq_; + const std::size_t source_fragments_before = + source_opening_fragment_count(pyramid_entries_, order.incarnation); + const auto new_source_opening = [&]() { + return source_opening_was_created(pyramid_entries_, order.incarnation, + source_cycle_before, position_cycle_seq_, source_fragments_before); + }; // design-market-entry-affordability: the entry leg was declined (at // placement or at fill) while an OPPOSITE position was live — execute the // reversal's closing leg only (rampatel BTC 2025-05-12 07:15Z: TV closed @@ -7009,7 +7034,7 @@ void BacktestEngine::apply_market_order_fill(PendingOrder& order, double fill_pr keep_mc_close_surplus ? -1 : order.qty_type, /*explicit_qty_prequantized=*/keep_mc_close_surplus, /*close_only=*/!keep_mc_close_surplus, order.incarnation); - if (keep_mc_close_surplus && !pyramid_entries_.empty()) + if (keep_mc_close_surplus && new_source_opening()) pyramid_entries_.back().entry_comment = order.comment; } trail_best_path_state = trail_best_price_; @@ -7086,27 +7111,10 @@ void BacktestEngine::apply_market_order_fill(PendingOrder& order, double fill_pr const double entry_fill = apply_fill_slippage(fill_price, order.is_long); if (std::isfinite(entry_fill) && add_qty > kQtyEpsilon) { - const double total_qty = position_qty_ + add_qty; - position_entry_price_ = - (position_entry_price_ * position_qty_ - + entry_fill * add_qty) / total_qty; - position_qty_ = total_qty; - ++position_entry_count_; - trail_best_price_ = entry_fill; - PyramidEntry lot{}; - lot.price = entry_fill; - lot.time = current_bar_.timestamp; - lot.qty = add_qty; - lot.entry_id = order.id; - lot.entry_bar_index = bar_index_; - lot.entry_comment = order.comment; - lot.entry_incarnation = order.incarnation; - lot.market_pyramid_add = true; - snapshot_entry_commission(lot); - pyramid_entries_.push_back(std::move(lot)); - if (stream_observe_actions_) stream_observe_entry(pyramid_entries_.back()); - id_unclosed_qty_[order.id] += add_qty; - cycle_filled_entry_ids_.insert(order.id); + const auto result = settle_source_opening( + requested, entry_fill, add_qty, order.id, order.comment, order.incarnation); + if (result.status == execution::Status::Applied && result.opened_units != 0.0) + pyramid_entries_.back().market_pyramid_add = true; } const double trail_best_after_fill = trail_best_price_; if (position_side_ == PositionSide::LONG) { @@ -7160,7 +7168,7 @@ void BacktestEngine::apply_market_order_fill(PendingOrder& order, double fill_pr throw std::logic_error("replacement cancellation receipt rejected"); } } - if (position_side_ == PositionSide::SHORT && !pyramid_entries_.empty()) + if (position_side_ == PositionSide::SHORT && new_source_opening()) pyramid_entries_.back().entry_comment = order.comment; const double trail_best_after_fill = trail_best_price_; if (position_side_ == PositionSide::LONG) @@ -7194,9 +7202,7 @@ void BacktestEngine::apply_market_order_fill(PendingOrder& order, double fill_pr order.incarnation); double trail_best_after_fill = trail_best_price_; // Set entry comment on the just-created pyramid entry - if (!pyramid_entries_.empty() - && (!paired_flat_market - || pyramid_entries_.back().entry_id == order.id)) { + if (new_source_opening()) { pyramid_entries_.back().entry_comment = order.comment; } // Update trail_best_price_ with intra-bar extremes for same-bar exit eval @@ -7226,6 +7232,9 @@ void BacktestEngine::apply_entry_order_fill(PendingOrder& order, double fill_pri double qty_before = position_qty_; int count_before = position_entry_count_; size_t trades_before_entry = trades_.size(); + const int64_t source_cycle_before = position_cycle_seq_; + const std::size_t source_fragments_before = + source_opening_fragment_count(pyramid_entries_, order.incarnation); // A pending priced (stop/limit) ENTRY that reaches its trigger while an // OPPOSITE position it did NOT open is live closes that position at the @@ -7354,7 +7363,10 @@ void BacktestEngine::apply_entry_order_fill(PendingOrder& order, double fill_pri bool was_priced_entry = priced_entry; if (did_execute) { double trail_best_after_fill = trail_best_price_; - if (!pyramid_entries_.empty()) pyramid_entries_.back().entry_comment = order.comment; + const bool new_source_opening = source_opening_was_created( + pyramid_entries_, order.incarnation, source_cycle_before, + position_cycle_seq_, source_fragments_before); + if (new_source_opening) pyramid_entries_.back().entry_comment = order.comment; // See apply_market_order_fill's matching guard: skip folding this // bar's pre-fill high/low into the trail when the fill happened AT // the bar's close (a POOC entry created and filled this same bar). @@ -7372,7 +7384,7 @@ void BacktestEngine::apply_entry_order_fill(PendingOrder& order, double fill_pri // Mask pre-fill bar extremes for the entry this fill created // (guard: back() really is this order's same-bar entry — a // close-only-opposite fill creates no new entry). - if (!pyramid_entries_.empty() + if (new_source_opening && pyramid_entries_.back().entry_bar_index == bar_index_ && pyramid_entries_.back().entry_id == order.id) { set_entry_fill_excursion_masks(pyramid_entries_.back(), bar, @@ -7729,29 +7741,12 @@ void BacktestEngine::apply_raw_order_fill(PendingOrder& order, double fill_price // Prefer the signal-time frozen quantity when the order carries one. double qty = !std::isnan(order.frozen_default_qty) ? order.frozen_default_qty : (std::isnan(order.qty) ? calc_qty(fill_price) : order.qty); - position_side_ = order.is_long ? PositionSide::LONG : PositionSide::SHORT; - position_cycle_seq_ = next_position_cycle_seq_++; - position_entry_price_ = fill_price; - // The shared post-dispatch hook queues the new fill's event. Clear any - // prior-cycle provenance first; RAW_ORDER opens do not route through - // open_fresh_position. - opening_obligations_.invalidate(); - position_entry_time_ = current_bar_.timestamp; - position_qty_ = qty; - position_entry_count_ = 1; - position_open_bar_ = bar_index_; - trail_best_price_ = fill_price; - pyramid_entries_.clear(); - id_unclosed_qty_.clear(); - cycle_filled_entry_ids_.clear(); - pyramid_entries_.push_back({fill_price, current_bar_.timestamp, qty, order.id, bar_index_}); - pyramid_entries_.back().entry_incarnation = order.incarnation; - snapshot_entry_commission(pyramid_entries_.back()); - if (stream_observe_actions_) stream_observe_entry(pyramid_entries_.back()); - id_unclosed_qty_[order.id] += qty; - cycle_filled_entry_ids_.insert(order.id); - bind_retained_exit_activations(); - if (!std::isnan(order.legs.prices().stop_price) || !std::isnan(order.legs.prices().limit_price)) { + const PositionSide requested = order.is_long ? PositionSide::LONG : PositionSide::SHORT; + const auto result = settle_source_opening( + requested, fill_price, qty, order.id, order.comment, order.incarnation); + if (result.status == execution::Status::Applied && result.opened_units != 0.0 + && (!std::isnan(order.legs.prices().stop_price) + || !std::isnan(order.legs.prices().limit_price))) { set_entry_fill_excursion_masks(pyramid_entries_.back(), current_bar_, fill_price); } trail_best_path_state = trail_best_price_; @@ -7794,23 +7789,14 @@ void BacktestEngine::apply_raw_order_fill(PendingOrder& order, double fill_price // Prefer the signal-time frozen quantity when the order carries one. double new_qty = !std::isnan(order.frozen_default_qty) ? order.frozen_default_qty : (std::isnan(order.qty) ? calc_qty(fill_price) : order.qty); - double total_qty = position_qty_ + new_qty; - position_entry_price_ = - (position_entry_price_ * position_qty_ + fill_price * new_qty) / total_qty; - position_qty_ = total_qty; - position_entry_count_++; - trail_best_price_ = fill_price; - pyramid_entries_.push_back({fill_price, current_bar_.timestamp, new_qty, order.id, bar_index_}); - pyramid_entries_.back().entry_incarnation = order.incarnation; - snapshot_entry_commission(pyramid_entries_.back()); - if (stream_observe_actions_) stream_observe_entry(pyramid_entries_.back()); + const auto result = settle_source_opening( + requested, fill_price, new_qty, order.id, order.comment, order.incarnation); // KI-62: flag same-direction MARKET adds (strategy.order path) so a // same-bar from_entry bracket exit can scratch them dur-0. - pyramid_entries_.back().market_pyramid_add = !is_priced_entry; - id_unclosed_qty_[order.id] += new_qty; - cycle_filled_entry_ids_.insert(order.id); - if (is_priced_entry) { - set_entry_fill_excursion_masks(pyramid_entries_.back(), current_bar_, fill_price); + if (result.status == execution::Status::Applied && result.opened_units != 0.0) { + pyramid_entries_.back().market_pyramid_add = !is_priced_entry; + if (is_priced_entry) + set_entry_fill_excursion_masks(pyramid_entries_.back(), current_bar_, fill_price); } } else { execute_market_exit(fill_price); diff --git a/src/engine_orders.cpp b/src/engine_orders.cpp index a43b680a6..13df2f1e9 100644 --- a/src/engine_orders.cpp +++ b/src/engine_orders.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,30 @@ namespace pineforge { using namespace internal; +namespace { +// Source predicates are resolved here, never retained by native settlement. +// Every fragment of an opening must agree with the selected source predicate. +template +std::vector source_opening_membership( + const std::vector& lots, Predicate selected) { + std::unordered_map membership; + std::vector incarnations; + for (const auto& lot : lots) { + const bool matches = selected(lot); + if (lot.entry_incarnation == 0) { + if (matches) + throw std::runtime_error("invalid resolved bound-close settlement: unowned opening"); + continue; + } + const auto [it, inserted] = membership.emplace(lot.entry_incarnation, matches); + if (!inserted && it->second != matches) + throw std::runtime_error("invalid resolved bound-close settlement: heterogeneous opening"); + if (inserted && matches) incarnations.push_back(lot.entry_incarnation); + } + return incarnations; +} +} // namespace + // Risk management + per-trade extreme tracking moved to engine_risk.cpp. @@ -25,6 +50,39 @@ double BacktestEngine::calc_qty_for_type(double fill_price, double qty_value, in if (std::isnan(qty_value)) { return calc_qty(fill_price); } + const double equity = qty_type == static_cast(QtyType::PERCENT_OF_EQUITY) + ? percent_commission_live_equity(round_to_mintick(current_bar_.close)) : 0.0; + return calc_qty_for_type_from_equity(fill_price, qty_value, qty_type, equity); +} + +double BacktestEngine::calc_default_qty_from_equity(double fill_price, double equity) const { + const double basis = round_to_mintick(fill_price); + switch (default_qty_type_) { + case QtyType::FIXED: + return apply_qty_step(default_qty_value_); + case QtyType::PERCENT_OF_EQUITY: { + // Source money precision and commission reservation stay exactly + // here for both live-equity and post-close projection callers. + if (tv_money_lot_sizing()) equity = tv_money_round(equity); + if (!std::isfinite(equity)) return 0.0; + const double cash = reserve_percent_commission( + equity * (default_qty_value_ / 100.0)) / active_account_currency_fx(); + if (!(std::isfinite(basis) && basis > 0.0)) return 0.0; + if (tv_money_lot_sizing()) + return tv_money_floor_lot(cash / (basis * syminfo_.pointvalue), qty_step_); + return apply_qty_step(cash / (basis * syminfo_.pointvalue)); + } + case QtyType::CASH: + return (std::isfinite(basis) && basis > 0.0) + ? apply_qty_step((default_qty_value_ / active_account_currency_fx()) + / (basis * syminfo_.pointvalue)) : 0.0; + } + return apply_qty_step(default_qty_value_); +} + +double BacktestEngine::calc_qty_for_type_from_equity( + double fill_price, double qty_value, int qty_type, double equity) const { + if (std::isnan(qty_value)) return calc_default_qty_from_equity(fill_price, equity); // qty_step_ lot-size flooring applies uniformly regardless of how the // caller's qty was derived — including this FIXED branch, which is the // common ``strategy.entry(qty=someComputedExpr)`` shape (e.g. a DCA base/ @@ -45,8 +103,6 @@ double BacktestEngine::calc_qty_for_type(double fill_price, double qty_value, in // tape pinned it (calc_qty carries the F / AAPL census). const double basis = round_to_mintick(fill_price); if (qty_type == static_cast(QtyType::PERCENT_OF_EQUITY)) { - const double equity = percent_commission_live_equity( - round_to_mintick(current_bar_.close)); if (!std::isfinite(equity)) return 0.0; double cash = reserve_percent_commission(equity * (qty_value / 100.0)); // Reject (qty 0) on a non-finite / non-positive fill price — a degenerate @@ -69,6 +125,24 @@ double BacktestEngine::calc_qty_for_type(double fill_price, double qty_value, in return apply_qty_step(qty_value); } +double BacktestEngine::source_reversal_qty( + double fill_price, double explicit_qty, int explicit_qty_type, + bool prequantized) const { + if (prequantized) return explicit_qty; + const bool needs_equity = std::isnan(explicit_qty) + ? default_qty_type_ == QtyType::PERCENT_OF_EQUITY + : explicit_qty_type == static_cast(QtyType::PERCENT_OF_EQUITY); + if (!needs_equity) + return calc_qty_for_type_from_equity(fill_price, explicit_qty, explicit_qty_type, 0.0); + const auto projection = project_native_settlement_v1( + execution::Flatten{}, execution::Fill{fill_price, {}, {}, 0}); + if (projection.status != execution::Status::Applied + && projection.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved reversal sizing projection"); + return calc_qty_for_type_from_equity( + fill_price, explicit_qty, explicit_qty_type, projection.realized_balance); +} + // Internal helper: execute a market entry (handles reverse-and-open). // @@ -102,8 +176,7 @@ void BacktestEngine::execute_market_entry(const std::string& id, bool is_long, d || (risk_direction_ == RiskDirection::SHORT_ONLY && is_long); if (is_opposite_entry && direction_blocked) { - double exit_fill = apply_slippage(fill_price, position_side_ == PositionSide::SHORT); - execute_market_exit(exit_fill); + execute_market_exit(fill_price); if (!paired_flat_market_transaction) purge_exit_orders(); return; } @@ -128,9 +201,10 @@ void BacktestEngine::execute_market_entry(const std::string& id, bool is_long, d } if (position_side_ == requested) { - add_to_pyramid_market(id, is_long, fill_price, explicit_qty, explicit_qty_type, - created_position_side, is_priced_entry, - entry_incarnation); + add_to_pyramid_market_with_qty_provenance( + id, is_long, fill_price, explicit_qty, explicit_qty_type, + created_position_side, is_priced_entry, explicit_qty_prequantized, + entry_incarnation); return; } @@ -147,9 +221,9 @@ void BacktestEngine::execute_market_entry(const std::string& id, bool is_long, d } if (later_same_tick_entry) { - sequential_same_tick_reversal_fill(id, is_long, fill_price, explicit_qty, - explicit_qty_type, - entry_incarnation); + sequential_same_tick_reversal_fill_with_qty_provenance( + id, is_long, fill_price, explicit_qty, explicit_qty_type, + explicit_qty_prequantized, entry_incarnation); return; } @@ -294,45 +368,34 @@ void BacktestEngine::record_range_end_close_trades() { // single from_entry id. See engine.hpp for the contract. Mirrors TradingView's // per-pyramid trade reporting: one Trade per drained slice. Returns total qty // drained so callers can assert / log if needed. +// Retained private ABI helper. Production close paths below use explicit +// source actions; this compatibility entry point also consumes the sole book. double BacktestEngine::fifo_drain(const std::string* from_entry, double qty_limit, double fill_price, bool was_long) { - double qty_closed = 0.0; - std::vector remaining; - for (auto& pe : pyramid_entries_) { - bool eligible = (from_entry == nullptr) || (pe.entry_id == *from_entry); - if (!eligible || qty_closed >= qty_limit - kQtyEpsilon) { - remaining.push_back(pe); - continue; - } - double close_qty = std::min(pe.qty, qty_limit - qty_closed); - double keep_qty = pe.qty - close_qty; - qty_closed += close_qty; - - emit_close_trade(pe, close_qty, fill_price, was_long); - - if (keep_qty > kQtyEpsilon) { - // Scale the accumulated USD excursion to the kept slice so the - // remaining entry's extremes stay consistent with its reduced - // qty (update_per_trade_extremes accumulates (diff) * pe.qty). - double keep_scale = keep_qty / pe.qty; - PyramidEntry kept = pe; - kept.qty = keep_qty; - kept.max_runup *= keep_scale; - kept.max_drawdown *= keep_scale; - // Realizing part of the lot consumes that part of its paid entry - // cost. A later rate/type/FX change cannot rewrite the payment. - kept.entry_commission_account = open_entry_commission(pe) - - allocated_entry_commission(pe, close_qty); - remaining.push_back(std::move(kept)); - } + (void)was_long; // physical orientation belongs to the authoritative book + const int pre_count = position_entry_count_; + execution::Result result; + if (from_entry) { + const auto incarnations = source_opening_membership(pyramid_entries_, + [&](const PyramidEntry& lot) { return lot.entry_id == *from_entry; }); + if (incarnations.empty()) return 0.0; + const execution::SelectedOpeningSet selection{position_cycle_seq_, incarnations}; + result = settle_execution_selected_with_lifecycle( + order_action::Reduce{qty_limit}, execution::Fill{fill_price, {}, {}, 0}, {}, selection); + } else { + result = settle_resolved_execution( + order_action::Reduce{qty_limit}, execution::Fill{fill_price, {}, {}, 0}); } - - pyramid_entries_ = std::move(remaining); - position_qty_ -= qty_closed; - return qty_closed; + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved compatibility drain settlement"); + // Old callers chose the later source slot policy themselves. Preserve + // that interface without retaining its former physical FIFO/fee loop. + if (result.status == execution::Status::Applied && position_side_ != PositionSide::FLAT) + position_entry_count_ = pre_count; + return result.closed_units; } - // Internal helper: execute a partial exit (reduce position by qty, create trade records) // TradingView creates individual trade records for each partial exit. void BacktestEngine::execute_partial_exit_qty( @@ -347,11 +410,16 @@ void BacktestEngine::execute_partial_exit_qty( bool is_buy = (position_side_ == PositionSide::SHORT); fill_price = apply_fill_slippage(fill_price, is_buy); - bool was_long = (position_side_ == PositionSide::LONG); - - // Close FIFO across all pyramid entries, creating trade records. - fifo_drain(/*from_entry=*/nullptr, qty_to_close, fill_price, was_long); - settle_position_after_partial_exit(cause); + const int pre_count = position_entry_count_; + execution::Action action = order_action::Reduce{qty_to_close}; + if (std::abs(held) - qty_to_close <= kQtyEpsilon) action = execution::Flatten{}; + const auto result = settle_resolved_execution( + action, execution::Fill{fill_price, {}, {}, 0}); + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved partial-close settlement"); + if (result.status == execution::Status::Applied) + restore_source_partial_exit_slots(pre_count, cause); } @@ -360,6 +428,8 @@ void BacktestEngine::execute_partial_exit_qty( // seam. It neither consults a pyramiding cap nor invents a source-policy bit. // The caller supplies the lot's immutable label, identity and fill metadata; // accounting and stream observations still use the engine's sole lot ledger. +// Test-only compatibility construction seam; no production caller. Live +// source entries use settle_source_opening and the shared settlement owner. void BacktestEngine::append_same_side_fill(PyramidEntry lot) { snapshot_entry_commission(lot); const double total_qty = position_qty_ + lot.qty; @@ -408,22 +478,19 @@ void BacktestEngine::execute_partial_exit_by_entry(double fill_price, PositionReductionCause cause) { if (position_side_ == PositionSide::FLAT || pyramid_entries_.empty()) return; - bool is_buy = (position_side_ == PositionSide::SHORT); - fill_price = apply_fill_slippage(fill_price, is_buy); - bool was_long = (position_side_ == PositionSide::LONG); - - std::vector remaining; - for (auto& pe : pyramid_entries_) { - if (pe.entry_id == from_entry) { - emit_close_trade(pe, pe.qty, fill_price, was_long); - position_qty_ -= pe.qty; - } else { - remaining.push_back(pe); - } - } - - pyramid_entries_ = std::move(remaining); - settle_position_after_partial_exit(cause); + const auto incarnations = source_opening_membership(pyramid_entries_, + [&](const PyramidEntry& lot) { return lot.entry_id == from_entry; }); + if (incarnations.empty()) return; + const execution::SelectedOpeningSet selection{position_cycle_seq_, incarnations}; + fill_price = apply_fill_slippage(fill_price, position_side_ == PositionSide::SHORT); + const int pre_count = position_entry_count_; + const auto result = settle_execution_selected_with_lifecycle( + execution::Flatten{}, execution::Fill{fill_price, {}, {}, 0}, {}, selection); + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved bound-close settlement"); + if (result.status == execution::Status::Applied) + restore_source_partial_exit_slots(pre_count, cause); } @@ -437,12 +504,28 @@ void BacktestEngine::execute_partial_exit_by_entry_qty( if (position_side_ == PositionSide::FLAT || pyramid_entries_.empty()) return; if (!std::isfinite(qty_to_close) || qty_to_close <= kQtyEpsilon) return; - bool is_buy = (position_side_ == PositionSide::SHORT); - fill_price = apply_fill_slippage(fill_price, is_buy); - bool was_long = (position_side_ == PositionSide::LONG); - - fifo_drain(&from_entry, qty_to_close, fill_price, was_long); - settle_position_after_partial_exit(cause); + const auto incarnations = source_opening_membership(pyramid_entries_, + [&](const PyramidEntry& lot) { return lot.entry_id == from_entry; }); + if (incarnations.empty()) return; + double selected_qty = 0.0; + bool has_unselected_lots = false; + for (const auto& lot : pyramid_entries_) { + if (lot.entry_id == from_entry) selected_qty += lot.qty; + else has_unselected_lots = true; + } + const execution::SelectedOpeningSet selection{position_cycle_seq_, incarnations}; + execution::Action action = order_action::Reduce{qty_to_close}; + if (!has_unselected_lots && selected_qty - qty_to_close <= kQtyEpsilon) + action = execution::Flatten{}; + fill_price = apply_fill_slippage(fill_price, position_side_ == PositionSide::SHORT); + const int pre_count = position_entry_count_; + const auto result = settle_execution_selected_with_lifecycle( + action, execution::Fill{fill_price, {}, {}, 0}, {}, selection); + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved bound-close settlement"); + if (result.status == execution::Status::Applied) + restore_source_partial_exit_slots(pre_count, cause); } @@ -492,28 +575,26 @@ double BacktestEngine::cover_samebar_market_adds_on_exit(const PendingOrder& ord || !std::isnan(order.legs.prices().trail_price); if (!priced_bracket) return 0.0; - bool is_buy = (position_side_ == PositionSide::SHORT); - double slipped = apply_fill_slippage(fill_price, is_buy); - bool was_long = (position_side_ == PositionSide::LONG); - - std::vector remaining; - remaining.reserve(pyramid_entries_.size()); - double closed = 0.0; - for (auto& pe : pyramid_entries_) { - if (pe.market_pyramid_add - && pe.entry_bar_index == bar_index_ - && pe.entry_id == order.from_entry) { - emit_close_trade(pe, pe.qty, slipped, was_long); - closed += pe.qty; - } else { - remaining.push_back(pe); - } - } - if (closed <= kQtyEpsilon) return 0.0; // nothing covered - pyramid_entries_ = std::move(remaining); - position_qty_ -= closed; - settle_position_after_partial_exit(cause); - return closed; + const auto incarnations = source_opening_membership(pyramid_entries_, + [&](const PyramidEntry& lot) { + return lot.market_pyramid_add && lot.entry_bar_index == bar_index_ + && lot.entry_id == order.from_entry; + }); + if (incarnations.empty()) return 0.0; + const execution::SelectedOpeningSet selection{position_cycle_seq_, incarnations}; + const double slipped = apply_fill_slippage(fill_price, position_side_ == PositionSide::SHORT); + // This is a second Fill after primary restoration and R20, so capture its + // own current source slot count rather than reusing the primary snapshot. + const int pre_count = position_entry_count_; + const auto result = settle_execution_selected_with_lifecycle( + execution::Flatten{}, execution::Fill{slipped, order.id, order.comment, order.incarnation}, + {}, selection); + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved bound-close settlement"); + if (result.status == execution::Status::Applied) + restore_source_partial_exit_slots(pre_count, cause); + return result.closed_units; } @@ -851,6 +932,15 @@ void BacktestEngine::reset_position_state_to_flat() { consumed_partial_exit_ids_.clear(); } +void BacktestEngine::restore_source_partial_exit_slots( + int pre_count, PositionReductionCause cause) { + // Settlement owns quantities, average price, cycles, and physical dust. + // This adapter step restores only the source's occupied-slot policy. + if (position_side_ == PositionSide::FLAT || pyramid_entries_.empty()) return; + if (cause == PositionReductionCause::BRACKET_EXIT) { + position_entry_count_ = std::max(pre_count, static_cast(pyramid_entries_.size())); + } +} // After a partial exit potentially empties pyramid_entries_, either reset // position state to FLAT (no entries left or qty effectively zero) or @@ -959,10 +1049,32 @@ void BacktestEngine::unbind_exit_activations() { void BacktestEngine::open_fresh_position(PositionSide requested, double fill_price, double qty, const std::string& id, uint64_t entry_incarnation) { - PyramidEntry lot{fill_price, current_bar_.timestamp, qty, id, bar_index_}; - lot.entry_incarnation = entry_incarnation; - snapshot_entry_commission(lot); - open_quoted_position(requested, std::move(lot)); + if (position_side_ != PositionSide::FLAT) + throw std::runtime_error("invalid resolved fresh opening: position not flat"); + settle_source_opening(requested, fill_price, qty, id, {}, entry_incarnation); +} + +execution::Result BacktestEngine::settle_source_opening( + PositionSide requested, double fill_price, double qty, + const std::string& id, const std::string& comment, uint64_t incarnation) { + if ((requested != PositionSide::LONG && requested != PositionSide::SHORT) + || (position_side_ != PositionSide::FLAT && position_side_ != requested) + || !std::isfinite(qty) || qty < 0.0) + throw std::runtime_error("invalid resolved source opening"); + const std::size_t before_lots = pyramid_entries_.size(); + const auto result = settle_resolved_execution( + order_action::Transact{requested == PositionSide::LONG ? qty : -qty}, + execution::Fill{fill_price, id, comment, incarnation}); + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved source opening settlement"); + if (result.status == execution::Status::Applied + && (result.closed_units != 0.0 || result.opened_units == 0.0 + || pyramid_entries_.size() != before_lots + 1 + || pyramid_entries_.back().entry_incarnation != incarnation + || pyramid_entries_.back().qty != std::abs(result.opened_units))) + throw std::runtime_error("invalid resolved source opening lot provenance"); + return result; } void BacktestEngine::open_quoted_position(PositionSide requested, PyramidEntry lot) { @@ -1139,6 +1251,16 @@ void BacktestEngine::add_to_pyramid_market(const std::string& id, bool is_long, PositionSide created_position_side, bool is_priced_entry, uint64_t entry_incarnation) { + add_to_pyramid_market_with_qty_provenance( + id, is_long, fill_price, explicit_qty, explicit_qty_type, + created_position_side, is_priced_entry, false, entry_incarnation); +} + +void BacktestEngine::add_to_pyramid_market_with_qty_provenance( + const std::string& id, bool is_long, double fill_price, double explicit_qty, + int explicit_qty_type, PositionSide created_position_side, + bool is_priced_entry, bool explicit_qty_prequantized, + uint64_t entry_incarnation) { PositionSide requested = is_long ? PositionSide::LONG : PositionSide::SHORT; bool flat_armed_priced = is_priced_entry && created_position_side == PositionSide::FLAT; @@ -1150,26 +1272,19 @@ void BacktestEngine::add_to_pyramid_market(const std::string& id, bool is_long, && position_entry_count_ >= pyramiding_) { return; } - double new_qty = calc_qty_for_type(fill_price, explicit_qty, explicit_qty_type); + const double new_qty = explicit_qty_prequantized + ? explicit_qty : calc_qty_for_type(fill_price, explicit_qty, explicit_qty_type); // Zero-lot add safety net. The fill kernel (apply_filled_order_to_state's // zero-lot decline) consumes such an order before it reaches here; should // any path bypass that gate, never materialize a qty-0 pyramid lot nor // spend a pyramiding slot on it — TV does not place the order at all. if (!(new_qty > kQtyEpsilon)) return; - double total_qty = position_qty_ + new_qty; - position_entry_price_ = (position_entry_price_ * position_qty_ + fill_price * new_qty) / total_qty; - position_qty_ = total_qty; - position_entry_count_++; - trail_best_price_ = fill_price; - pyramid_entries_.push_back({fill_price, current_bar_.timestamp, new_qty, id, bar_index_}); - pyramid_entries_.back().entry_incarnation = entry_incarnation; - snapshot_entry_commission(pyramid_entries_.back()); - if (stream_observe_actions_) stream_observe_entry(pyramid_entries_.back()); + const auto result = settle_source_opening( + requested, fill_price, new_qty, id, {}, entry_incarnation); // KI-62: only a same-direction MARKET add is scratched by a same-bar // from_entry bracket exit; a priced pyramid add is not this collision. - pyramid_entries_.back().market_pyramid_add = !is_priced_entry; - id_unclosed_qty_[id] += new_qty; - cycle_filled_entry_ids_.insert(id); + if (result.status == execution::Status::Applied && result.opened_units != 0.0) + pyramid_entries_.back().market_pyramid_add = !is_priced_entry; } @@ -1268,56 +1383,28 @@ void BacktestEngine::flip_market_position_to(const std::string& id, bool is_long bool explicit_qty_prequantized, bool close_only, uint64_t entry_incarnation) { - // For the close we need exit slippage based on closing direction. - // Closing a long = sell (price - slip); closing a short = buy (price + slip). - // fill_price already has entry slippage applied; un-slip it before - // re-applying with the exit direction. - double raw_price = fill_price; - if (slippage_ != 0 && !current_fill_is_limit_) { - // LIMIT-triggered entry fills were never slipped (limit-or-better - // path), so there is no entry slip to back out for the close leg. - double slip = slippage_ * syminfo_mintick_; - raw_price = is_long ? (fill_price - slip) : (fill_price + slip); - } - // Flag-aware close leg: a limit-triggered flip's close leg follows the - // entry leg's limit semantics (unslipped, limit-or-better) for internal - // consistency with the sibling close_opposite_then_enter path — both - // legs land at the identical unslipped snapped price. No direct TV - // evidence yet (needs a limit-flip slippage>0 export) — corpus provably - // indifferent at slippage=0. - double exit_fill = apply_fill_slippage(raw_price, position_side_ == PositionSide::SHORT); - bool was_long = (position_side_ == PositionSide::LONG); - - // Emit one Trade per pyramid entry (matches TradingView reporting) - for (auto& pe : pyramid_entries_) { - emit_close_trade(pe, pe.qty, exit_fill, was_long); - } - - // The old lots are fully realized above. Clear their live position/PnL - // and entry-fee snapshots before sizing the incoming leg; otherwise a - // percent-typed reversal adds stale open PnL and debits the already- - // realized entry commission a second time. - reset_position_state_to_flat(); - - if (close_only) { - // Priced-entry reduce-only cases: either this order was armed during a - // prior cycle and flips a later opposite position, or its same-cycle - // frozen transaction exactly equals the grown live opposite position. - // In both cases close the whole position and stay flat; do NOT open. - // See apply_entry_order_fill's close_only_opposite predicates. - return; + // The incoming direction is also the closing direction. The caller has + // already resolved this one price; no un-slip/re-slip or second ticket. + execution::Action action = execution::Flatten{}; + if (!close_only) { + double held = 0.0; + for (const auto& lot : pyramid_entries_) held += lot.qty; + const double incoming = source_reversal_qty( + fill_price, explicit_qty, explicit_qty_type, explicit_qty_prequantized); + if (!std::isfinite(incoming) || incoming < 0.0) + throw std::runtime_error("invalid resolved flip quantity"); + if (incoming > 0.0) { + const double total = held + incoming; + if (!std::isfinite(total) || total == held || (held > 0.0 && total == incoming)) + throw std::runtime_error("unrepresentable resolved flip quantity"); + action = order_action::Transact{is_long ? total : -total}; + } } - - // Default-sized MARKET quantities are frozen and exchange-quantized at - // signal time. Every sibling dispatch path preserves that provenance; - // ordinary flips must not feed the frozen contracts through qty_step a - // second time (binary64 can turn 1.3410 into 1.3409 on the second floor). - // Explicit/FIXED quantities keep their normal single fill-side floor. - double new_qty = explicit_qty_prequantized - ? explicit_qty - : calc_qty_for_type(fill_price, explicit_qty, explicit_qty_type); - PositionSide requested = is_long ? PositionSide::LONG : PositionSide::SHORT; - open_fresh_position(requested, fill_price, new_qty, id, entry_incarnation); + const auto result = settle_resolved_execution( + action, execution::Fill{fill_price, id, {}, entry_incarnation}); + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved flip settlement"); } @@ -1369,39 +1456,29 @@ void BacktestEngine::sequential_same_tick_reversal_fill(const std::string& id, double explicit_qty, int explicit_qty_type, uint64_t entry_incarnation) { - // fill_price arrives entry-slipped (execute_market_entry applied entry - // slippage). The close leg needs EXIT slippage for the closing - // direction — un-slip, then re-apply, mirroring flip_market_position_to. - double raw_price = fill_price; - if (slippage_ != 0 && !current_fill_is_limit_) { - double slip = slippage_ * syminfo_mintick_; - raw_price = is_long ? (fill_price - slip) : (fill_price + slip); - } - double exit_fill = apply_fill_slippage(raw_price, position_side_ == PositionSide::SHORT); - bool was_long = (position_side_ == PositionSide::LONG); - double old_qty = position_qty_; - - // Close the ENTIRE opposite position — one row per pyramid lot, all - // tagged with THIS order's id by apply_filled_order_to_state (matches - // TV's single-row exit attribution to the first closing signal). - for (auto& pe : pyramid_entries_) { - emit_close_trade(pe, pe.qty, exit_fill, was_long); - } - reset_position_state_to_flat(); - - // Plain (non-augmented) sizing, computed after the close so - // percent-of-equity sees the realized PnL — the same equity basis - // flip_market_position_to uses. Only the portion that crosses zero - // opens a position; if the plain qty doesn't reach past the old - // position, the LAST same-tick entry (filling next from flat) owns - // the new position instead. - double tx_qty = calc_qty_for_type(fill_price, explicit_qty, explicit_qty_type); - double remainder = tx_qty - old_qty; - if (remainder > kQtyEpsilon) { - PositionSide requested = is_long ? PositionSide::LONG : PositionSide::SHORT; - open_fresh_position( - requested, fill_price, remainder, id, entry_incarnation); - } + sequential_same_tick_reversal_fill_with_qty_provenance( + id, is_long, fill_price, explicit_qty, explicit_qty_type, false, entry_incarnation); +} + +void BacktestEngine::sequential_same_tick_reversal_fill_with_qty_provenance( + const std::string& id, bool is_long, double fill_price, double explicit_qty, + int explicit_qty_type, bool explicit_qty_prequantized, uint64_t entry_incarnation) { + double held = 0.0; + for (const auto& lot : pyramid_entries_) held += lot.qty; + const double transaction = source_reversal_qty( + fill_price, explicit_qty, explicit_qty_type, explicit_qty_prequantized); + if (!std::isfinite(transaction) || transaction < 0.0) + throw std::runtime_error("invalid resolved sequential quantity"); + execution::Action action = execution::Flatten{}; + if (transaction - held > kQtyEpsilon) + action = order_action::Transact{is_long ? transaction : -transaction}; + // Class B closes the entire old book here; its later sibling remains a + // separate Fill. Empty lifecycle preserves the active pending iteration. + const auto result = settle_resolved_execution( + action, execution::Fill{fill_price, id, {}, entry_incarnation}); + if (result.status != execution::Status::Applied + && result.status != execution::Status::NoEffect) + throw std::runtime_error("invalid resolved sequential settlement"); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 632e43be3..e89815535 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,6 +2,9 @@ set(TEST_SOURCES test_unbounded_margin_admission test_resolved_execution test_native_scoped_close + test_native_selected_settlement + test_native_settlement_projection + test_native_settlement_callers test_native_reversal_contract test_order_action test_native_order_core @@ -339,6 +342,24 @@ add_test( ) set_tests_properties(test_native_cpp_abi PROPERTIES TIMEOUT 240) +# The actual historical archive is prepared explicitly before CTest so the +# compatibility check itself stays offline and cannot silently use a stub. +set(PINEFORGE_SETTLEMENT_ABI_BASE_RECEIPT + "${PROJECT_BINARY_DIR}/settlement-abi-base/receipt.json" + CACHE FILEPATH "Prepared actual R2 ABI provider receipt") +add_test(NAME test_settlement_cpp_abi + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/check_settlement_cpp_abi.py + --compiler ${CMAKE_CXX_COMPILER} + --library $ + --include ${PROJECT_SOURCE_DIR}/include + --generated-include ${PROJECT_BINARY_DIR}/include + --base-receipt ${PINEFORGE_SETTLEMENT_ABI_BASE_RECEIPT} + --receipt ${PROJECT_BINARY_DIR}/settlement-abi-receipt.json + ${_pf_script_cpp_abi_flags}) +set_tests_properties(test_settlement_cpp_abi PROPERTIES TIMEOUT 600) +add_test(NAME test_settlement_cpp_abi_tooling + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/test_settlement_cpp_abi.py) + add_test( NAME test_derive_corpus_feeds COMMAND ${Python3_EXECUTABLE} diff --git a/tests/fixtures/settlement_cpp_abi/README.md b/tests/fixtures/settlement_cpp_abi/README.md new file mode 100644 index 000000000..02ef743a0 --- /dev/null +++ b/tests/fixtures/settlement_cpp_abi/README.md @@ -0,0 +1,92 @@ +# Settlement C++ ABI proof + +`e60e571/manifest.json` pins the complete 52-file public header closure of +R2 commit `e60e57156a54bb1c74c0c0a4f5a338fd924c046d`, tree +`e209c89035ec884774f8f3e68e1dc118f26aaf51`. Each entry records bytes, SHA-256 +and Git blob identity. Binary libraries are **not** checked into this fixture. + +The new settlement checker supplements `check_native_cpp_abi.py` and +`check_script_cpp_abi.py`; retain those existing checks and their old epoch +and UBSan RTTI controls. No generated pairing executable is run. + +## Prepare the real old provider explicitly + +After configuring the current candidate build, ensure the exact R2 Git object +is available. CI uses full-history checkouts and needs no separate fetch. For a local shallow +clone, fetch the pinned object explicitly before preparation, outside CTest: + +```sh +git fetch --no-tags --depth=1 origin e60e57156a54bb1c74c0c0a4f5a338fd924c046d +python3 scripts/prepare_settlement_cpp_abi_base.py \ + --source-repo . --current-build build \ + --output build/settlement-abi-base --jobs 4 +``` + +For the sanitizer lane, replace both build paths with `build-asan`; for the +native-runner lane use `build-live`. The preparation script copies the actual +current CMake compiler/build-type/flags/sanitizer settings, Eigen package or +already populated FetchContent source, and platform options. It builds the full pinned R2 static library +with tests, tutorial, runner and corpus disabled. Source comes only from +immutable local Git objects. Dependencies must already be available in the current build; its downloaded +Eigen source is reused when needed. FetchContent stays disconnected. The script itself performs no Git/dependency +network fetch and runs no test or strategy binary. + +Every preparation output must be new. Keep an existing receipt only for the +same compiler/configuration; the checker rejects mismatches. The receipt +binds archive/header bytes, actual compiler implementation/version/target, +configuration, compilation database and source identity. The version header +is generated from the archived base's VERSION fallback; it is retained and +hashed, not mislabeled as a live Git checkout descriptor. + +## Check the integrated current archive + +```sh +python3 scripts/check_settlement_cpp_abi.py \ + --compiler /path/to/current/c++ --library build/lib/libpineforge.a \ + --include include --generated-include build/include \ + --base-receipt build/settlement-abi-base/receipt.json \ + --receipt build/settlement-abi-receipt.json +``` + +Pass `--extra-flag=-fsanitize=address,undefined` for an instrumented archive, +as the existing CMake ABI guards do. The compiler must match the current +build and the prepared base. Missing artifacts fail with the preparation +command; no skip, stub or implicit network fallback is available. + +The checker: + +* authenticates old headers and both archives, checks old defined symbols, + and requires current archive freshness against all `src`, `include`, + `cmake` files and `CMakeLists.txt`; +* freezes execution aggregate field/order/type/status encodings, CloseScope + alternatives, native header contracts, engine storage declarations and + virtual methods; compares actual compiler-emitted offsets/alignment/type + sizes for 251 engine data members plus returned structures; +* compiles each old/new caller before any link result is interpreted; +* links old Book/singleton/lifecycle, old private F8/F11 wrappers and old + return-only host event callers against both full providers; +* links the six new selected/projection methods and new private provenance + helpers against the current provider, then rejects those same objects + against the real R2 provider for every required method; +* compiles wrong projection `_v2`/return-namespace and wrong selection-domain + callers, then requires rejection at those exact symbols. Return namespace + is not demanded in ordinary projection method mangling; +* refuses unrelated undefined symbols, retains full compiler/link logs and + records `executedBinaries: 0`. + +Private/protected member access uses explicit test-only `-fno-access-control` +on the pairing TUs. Product visibility is unchanged. Callers reinterpret an +unused command-line pointer and are link-only; **never execute them**. +The compiler's non-standard-layout `offsetof` extension is used only for +same-toolchain ABI observation, with its specific diagnostic suppressed. + +Three development modes return labeled partial receipts: +`--base-only` checks old-provider controls; `--old-rejections-only` checks +new public declarations against the real old provider before the new archive +exists; `--public-only` runs all public pairings before new private provenance +helpers are integrated. None returns `status: passed` or +`newArchivePairingComplete: true`. Do not use either mode in CI acceptance. + +The six offline Python refusal tests are run by +`python3 scripts/test_settlement_cpp_abi.py -v`; they are tooling checks, not +substitutes for the actual compiler/archive matrix. diff --git a/tests/fixtures/settlement_cpp_abi/e60e571/manifest.json b/tests/fixtures/settlement_cpp_abi/e60e571/manifest.json new file mode 100644 index 000000000..92cf74ab8 --- /dev/null +++ b/tests/fixtures/settlement_cpp_abi/e60e571/manifest.json @@ -0,0 +1,267 @@ +{ + "commit": "e60e57156a54bb1c74c0c0a4f5a338fd924c046d", + "files": { + "include/pineforge/bar.hpp": { + "bytes": 655, + "git_blob": "836f73daab382e9c8d5a6d03a942a921e39b756f", + "sha256": "665c2528dca64ac9474de622dfd7ffdb540ff91316f70317e39aada158d1e0de" + }, + "include/pineforge/broker_events.hpp": { + "bytes": 4343, + "git_blob": "2972d53c7b039af266f4da161301c01eec54c16d", + "sha256": "4f3d9948a4d1604750901fa3982d6ea49bd50dd15a86c4e5d690b83d1f5f23c0" + }, + "include/pineforge/color.hpp": { + "bytes": 1253, + "git_blob": "a4d54f37a1c9d8d10ff5423aca4d42814a666446", + "sha256": "29eb9d4473e4dd488f959fb9f53999656e3feb6aa25255287e80855ecabe96d6" + }, + "include/pineforge/compat/pine/exit_activation.hpp": { + "bytes": 3017, + "git_blob": "a62927dd936e839160671cee665de908cb7f5dc3", + "sha256": "9190382359099021870d260718ad593c567024603b016294d1b113c278ddd73d" + }, + "include/pineforge/compat/pine/exit_lifecycle.hpp": { + "bytes": 882, + "git_blob": "c78f64b82adaf18e013e017a633b9b9b6b48f22d", + "sha256": "2fcfce8c914424404767bdd7b2fc6ea30decbc3f5f5a6123cac9080e2b9bd8d2" + }, + "include/pineforge/compat/pine/frozen_market_instruction.hpp": { + "bytes": 3201, + "git_blob": "8890ad48c5e4485b9d5a65a5bdd7f7f876e39c24", + "sha256": "e67a4b470e82086a089d3bf0b8996f88ab0fd5d86046682d6cb19b2ed64922c8" + }, + "include/pineforge/compat/pine/intraday_cap.hpp": { + "bytes": 10469, + "git_blob": "0aba76b6a78251df33e92f19290473adecf5a61e", + "sha256": "0ac0eab34625ab9a974037ff8c0217603a01deb81599350a4b24f7bf48ac0cbd" + }, + "include/pineforge/compat/pine/intraday_order_budget.hpp": { + "bytes": 3878, + "git_blob": "730d941fffa3939e375489f40a50901ccea9ec87", + "sha256": "43f7700f9c66c630ab8a9d8fd899eecb03c2fd9aabff11d2dc2b7fa85f92981b" + }, + "include/pineforge/compat/pine/market_admission.hpp": { + "bytes": 1057, + "git_blob": "33cf82e5e53ae870c3665ede69d7cde2e9695752", + "sha256": "83ce322f8864a569e197493cd2bcca23aa226997d3508caaa6a783ff1bb97b03" + }, + "include/pineforge/compat/pine/order_birth.hpp": { + "bytes": 841, + "git_blob": "aa48c054f613b65ee5ded14fde41643f807347bd", + "sha256": "e819f5c4202911e5e592f4472925a15c07f7300fabcfecf4f214348e5ee605f8" + }, + "include/pineforge/compat/pine/order_priority.hpp": { + "bytes": 1577, + "git_blob": "86135165fd3d83fc4638a2295dda769024836517", + "sha256": "e88ffcf28bf3fac95bc439b3d99a3944fc9c36ce0d3892fd365d25f9f3f920e4" + }, + "include/pineforge/compat/pine/reservation_expansion.hpp": { + "bytes": 759, + "git_blob": "1115a6593f5fabadf722b14305a213b07afb6c85", + "sha256": "7ad4195af574df40a600b748399f00c41cafa9441bfdcb5b90898ec8a4b577dd" + }, + "include/pineforge/drawing.hpp": { + "bytes": 17150, + "git_blob": "d04f12bb214b82c5d0b174824c25ff9f148ab090", + "sha256": "1635cfc9e65a91369cc1491a64dc564944b6828333af62e3ecd80edb5ba74183" + }, + "include/pineforge/engine.hpp": { + "bytes": 312105, + "git_blob": "3624e74ddb4785db78dbabb216664aa938452f0f", + "sha256": "7cbc440c20f1b4c7b24c95ea30463eef509a11dc2131a484c5e8a752db48e31d" + }, + "include/pineforge/execution.hpp": { + "bytes": 4294, + "git_blob": "f270b7c64544ee9188c22752d5e41eaa78bf01d1", + "sha256": "e6e2440118f38a0eba5b75a0fb0cf03675f92939c154aaf8dc0837653a113895" + }, + "include/pineforge/execution_close_scope.hpp": { + "bytes": 583, + "git_blob": "65294d9f25468ccf3e4c79861beb96b8037b56d5", + "sha256": "daba6e221a19d3d3cd1db248a2861414ef91648c8330709917067ab32b2ebdb4" + }, + "include/pineforge/execution_consumer.hpp": { + "bytes": 3010, + "git_blob": "19e99314462c9c3729953b0f343abd4aa0bf1496", + "sha256": "ffd09417d1f1e8aab32794673d6802616c5055b7976101e49416c283b64856e0" + }, + "include/pineforge/exit_leg_lifecycle.hpp": { + "bytes": 22622, + "git_blob": "91492ec22e05d7b6d9327f60ff5e239ae6906fcd", + "sha256": "9d785271511a350114adc7cf5bcb1600cf569f0a10559a8671b7e4b9cb9dc89a" + }, + "include/pineforge/generic_matrix.hpp": { + "bytes": 25438, + "git_blob": "8694e5577d56221498cca844aa14cddb9a15e84f", + "sha256": "9c94a877b6f9e5965604b03f572ceea2ea3680de586d69ef0627c78208624150" + }, + "include/pineforge/leg_activation.hpp": { + "bytes": 1353, + "git_blob": "4160c841782800853bb574219b3dad2ca8aa0cc6", + "sha256": "859d75d0ae90ec3cc109765da066b37c3bbbff9c7c840368fbdd05dc7e178f3f" + }, + "include/pineforge/live_parser.h": { + "bytes": 3870, + "git_blob": "0d00afe0586477c69dea35a1bfef72f3acb491a3", + "sha256": "f1308086a66fe6a0636b8b2fc44c3fcb6b0c21ed0311654297ecc8ea5e959e1e" + }, + "include/pineforge/log.hpp": { + "bytes": 496, + "git_blob": "6cc4c35ad1a902f05022fb11357dff86281b5301", + "sha256": "df5db36e01ddaa66c44c1b378e6b0c3b05e0c732c59fbcb5401e3606585d25a3" + }, + "include/pineforge/magnifier.hpp": { + "bytes": 3241, + "git_blob": "1bbc910a869d162900d7610fa0bd9a88776b7b0f", + "sha256": "c0ed220b6df054b44596a434d4f48b1c3e960ca329bb8644a405a608175e06ff" + }, + "include/pineforge/map.hpp": { + "bytes": 14476, + "git_blob": "5dd59cbaad9b93138738d3f92b8c457c0ac92a7d", + "sha256": "772a4f6c79ea01af6ad057e361e821d79a7d9884e4aff8bac5a2f77565e55b9f" + }, + "include/pineforge/market_admission.hpp": { + "bytes": 10221, + "git_blob": "70be6af6b556bf77cab6c9f823bb3492cf7c8294", + "sha256": "22914434a2c82f761b98c1b5a2ff9365c89bb802b584cdda2a3c30c5986ec6fe" + }, + "include/pineforge/market_driver.hpp": { + "bytes": 5374, + "git_blob": "3e6c68fb491156520640b1095dfce057395375a3", + "sha256": "f89b8b49127dff19f88199da50db3defbcd891c2a25cc777f595d7f277a7cd3c" + }, + "include/pineforge/math.hpp": { + "bytes": 1316, + "git_blob": "9ec09d539dd0ba732449424e6a8c80c3f5454888", + "sha256": "be95fa75d1e8d288ed49d62fa56257efe99435be7ba357fbb93cc36c0dc9eda3" + }, + "include/pineforge/matrix.hpp": { + "bytes": 4868, + "git_blob": "22b29448fb60dded446543b1940c1a1466986133", + "sha256": "f463ab76db92436fd818839c390e69462a794e07413d99fb49af4ed1b67bca6f" + }, + "include/pineforge/metrics.hpp": { + "bytes": 1353, + "git_blob": "f468f485bbdd0512536e96390d2b411228c2f517", + "sha256": "9e35101f16c2717dbfde428ecc515802dd758056e080fd45ee6411e9d0ead5be" + }, + "include/pineforge/na.hpp": { + "bytes": 1064, + "git_blob": "e98505aadc68a09cbe5a14aacf591a799a15ecd2", + "sha256": "0e8c256fcdeda94a0855458fa3b8c4ad4f9864dc2aecce969934f6bea82fd0a5" + }, + "include/pineforge/native_calendar.hpp": { + "bytes": 19589, + "git_blob": "742ecfabc3764d626617c2762d4b574d46d8ff46", + "sha256": "5495cb007509937616255a9615dcc92c49e5115692c8375ef93b594a36e55860" + }, + "include/pineforge/native_host.hpp": { + "bytes": 10304, + "git_blob": "58573006ad056ecaa773dc79d22ca1375d51fac8", + "sha256": "9b7236e5946d1e5d2355cd6a72a02d987ab6b215acf2155ebce89771c8015808" + }, + "include/pineforge/native_order.hpp": { + "bytes": 36944, + "git_blob": "238475d372cc87b91d163c6c1a378f10751d444e", + "sha256": "3881f6b1ebecb95801e5ea6f4acb5a5a8203d94322ffdc36663d69b4966b7c08" + }, + "include/pineforge/native_order_identity.hpp": { + "bytes": 1948, + "git_blob": "c0a6835c8fdb5c3b331ad519439af12946e34cf2", + "sha256": "6f6d63c1a1f54499f04a137d0b13ea343aaf94ffe090058a62c854311345ac69" + }, + "include/pineforge/native_run_spec.hpp": { + "bytes": 5518, + "git_blob": "1b5ed883a71ed772e9f115415ab5c7acb1f920f0", + "sha256": "e7be6d930c4eed13d9fa2358f529c98e5ad37b75ad72013f1ba65513cabb8da1" + }, + "include/pineforge/order_action.hpp": { + "bytes": 3470, + "git_blob": "d992472cbefefdf8e29704a21f04614d68d39a58", + "sha256": "7d76ddc3f6ba76216d5feb34cdf28908e33a92a56757a707ac6bd94dca315001" + }, + "include/pineforge/order_birth.hpp": { + "bytes": 4833, + "git_blob": "ed4a4038e9d079c92b4ebd777a5f22f1f1256b0c", + "sha256": "0eadb3e2a92248fbc8e6ba8f94285b15b0e83d7a0396f5904b40c216619604e4" + }, + "include/pineforge/order_cancellation.hpp": { + "bytes": 7172, + "git_blob": "a4f11409f19c7a85347879ecfd942bd7bae5c2eb", + "sha256": "389eb3659f121351ad4b53b936d3a2c79f343ecdb6c0c4bb0bc13777596b90ee" + }, + "include/pineforge/order_priority.hpp": { + "bytes": 694, + "git_blob": "3e9fa5ee4121971a7962db8a60859aba6fb9ec95", + "sha256": "25bb7898972153b044675245db230c1171644856ef904f4dd02353b8968d433b" + }, + "include/pineforge/pending_order_mirror.hpp": { + "bytes": 21978, + "git_blob": "fc08bfd7328718de1c658bdcea1826c2fbc9eebe", + "sha256": "7921179f690dbb6f00ee8ab6ca71b3174fcc42a61ebc6786c3727c7f85a576a4" + }, + "include/pineforge/pine_float_compare.hpp": { + "bytes": 2893, + "git_blob": "a3de0e9a2ef3517098be500880fda71bdcfe2260", + "sha256": "3d389142b8e4fbd899c6923c11a27447920a1c9e9db0338ff9abc8a9f4a06d1c" + }, + "include/pineforge/pineforge.h": { + "bytes": 74362, + "git_blob": "31db9f0b9f8ac5f33041917f7d599daff91013c8", + "sha256": "e4c735a1e39411db4156ac13a50cee10c554039b20f33e3c2299cd73be6a6c3e" + }, + "include/pineforge/position_close_obligation.hpp": { + "bytes": 1060, + "git_blob": "09576ee0f2a1d49becc46f5701f4369cf8f87bf6", + "sha256": "6f6ad0aaca952dda7a9386cb78e57705b3494364dbd8970b0a051fe99d05b777" + }, + "include/pineforge/quantity_intent.hpp": { + "bytes": 3054, + "git_blob": "41262fee5aa2db4032fc008542c573032563629b", + "sha256": "c4cc0f1cd5d6ed1f248825712a5ba4b4f5a808b787c05c58971934b89d770b90" + }, + "include/pineforge/reservation_expansion.hpp": { + "bytes": 2434, + "git_blob": "b4302c9eed97c3409d9911fd6cecdc64fa0aad86", + "sha256": "27e514249f62aa7353ac084a6e59e25e338a867fe4d67ce4646446a1cf9f9dab" + }, + "include/pineforge/series.hpp": { + "bytes": 3439, + "git_blob": "61baaef2c351d37b0d6a5eb179e91ce670eb83c3", + "sha256": "e7709f67df9046c9eba163c2dbe91f8161084a955265ccf6b163355856c0afde" + }, + "include/pineforge/session_time.hpp": { + "bytes": 10337, + "git_blob": "af3a4472d852e0965947c006d0b6ad85304539e7", + "sha256": "f58c94bfae948e08fa21e52f20eae3940b31e398d83b800baf3a31f3a5fde669" + }, + "include/pineforge/str_utils.hpp": { + "bytes": 820, + "git_blob": "d44059f10b41872ea57a1abd9c246fadb3b7b3eb", + "sha256": "58e052a040da94f3824ec36db05b37dfe9e2219a32e843b56998fb1f0efe1393" + }, + "include/pineforge/ta.hpp": { + "bytes": 35301, + "git_blob": "3baeef801f44b938c16137fca5881784646760f9", + "sha256": "7221324be971ef6de425253656327395d8a1ffd081b3c1eef0c8c5b96f51e6b6" + }, + "include/pineforge/timeframe.hpp": { + "bytes": 29534, + "git_blob": "3d1bc9d495ac7e5b39ea5bae57c9d9afe580192b", + "sha256": "5e4c312da463fdcff55e0887188523c8bbf85a2ae62f3317a2b23f22fe0161e2" + }, + "include/pineforge/version.h.in": { + "bytes": 893, + "git_blob": "27118bd81741597e187e3a49d04ac201f9508011", + "sha256": "9b9e81d5e0998441a82dbe15a4495182fdfe126baea1cf26bdaaf51e491ad99f" + }, + "include/pineforge/window_sum.hpp": { + "bytes": 2828, + "git_blob": "1ea86860f6cb901fe79ff89ef46f1c77ce7e2b8c", + "sha256": "b5519f95d57d46e2a8045d380c1e1e1427ce744f4d53ee3a04ebfdaf942d3f68" + } + }, + "schemaVersion": "pineforge-settlement-abi-base-headers/v1", + "tree": "e209c89035ec884774f8f3e68e1dc118f26aaf51" +} diff --git a/tests/test_native_selected_settlement.cpp b/tests/test_native_selected_settlement.cpp new file mode 100644 index 000000000..4c45c6265 --- /dev/null +++ b/tests/test_native_selected_settlement.cpp @@ -0,0 +1,171 @@ +// R3 selected-set owner: literal economics on the actual settlement seam. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge; +namespace x = pineforge::execution; +namespace { +int checks=0, failures=0; +const char* scenario="setup"; +struct Abort {}; +#define CHECK(v) do { ++checks; if(!(v)){++failures;std::printf("FAIL %s:%d %s\n",scenario,__LINE__,#v);} } while(0) +#define REQUIRE(v) do { bool ok_=bool(v);CHECK(ok_);if(!ok_)throw Abort{}; } while(0) +void near(double a,double b) { + const bool ok=std::isfinite(a)&&std::isfinite(b)&&std::abs(a-b)<=1e-12*std::max(1.0,std::abs(b)); + if(!ok)std::printf(" actual=%.17g expected=%.17g\n",a,b);CHECK(ok); +} +struct Book final : BacktestEngine { + Book(){initial_capital_=1000;commission_type_=CommissionType::CASH_PER_ORDER;commission_value_=6; + syminfo_.pointvalue=1;account_currency_fx_=1;stream_observe_actions_=true; + current_bar_={100,100,100,100,1,1736121600000LL};bar_index_=7;} + void on_bar(const Bar&) override {} + x::PhysicalExecutionContext context() const {return {current_bar_.timestamp,bar_index_,{}, {}};} + void open(double q,double price,uint64_t id,const char* label,double paid=0){ + const auto r=settle_native_execution_at(order_action::Transact{q},x::Fill{price,label,"",id,paid},context()); + REQUIRE(r.status==x::Status::Applied); + } + void schedule(CommissionType kind,double value,double pv=1,double fx=1){ + commission_type_=kind;commission_value_=value;syminfo_.pointvalue=pv;account_currency_fx_=fx; + } + x::SettlementInspection inspect(const x::Action& a,const x::Fill& f,const x::SelectedOpeningSet& s)const{ + return inspect_native_settlement_selected(a,f,s); + } + x::AccountEffectProjection project(const x::Action& a,const x::Fill& f,const x::SelectedOpeningSet& s)const{ + return project_native_settlement_selected_v1(a,f,s); + } + x::Result settle(const x::Action& a,const x::Fill& f,const x::SelectedOpeningSet& s){ + return settle_native_execution_selected_at(a,f,context(),s); + } + int64_t cycle()const{return position_cycle_seq_;} + int64_t next_cycle()const{return next_position_cycle_seq_;} + uint64_t next_order()const{return next_order_incarnation_;} + const auto& lots()const{return pyramid_entries_;} + const auto& rows()const{return trades_;} + size_t actions()const{return stream_order_actions_.size();} + double qty()const{return position_qty_;} + double net()const{return net_profit_sum_;} +}; +x::Fill fill(double price,std::optional ticket={}){return {price,"selected-close","",900,ticket};} + +void invalid_sets(double sign){ + scenario="selected invalid atomicity";Book b;b.open(sign,100,11,"A",2);b.open(sign*3,100,22,"B",6); + const uint64_t hash=b.broker_state_hash(),stream=b.stream_state_hash(); + const auto actions=b.actions();const auto next=b.next_cycle();const auto order=b.next_order(); + const std::vector bad={{b.cycle(),{}},{b.cycle(),{0}}, + {b.cycle(),{11,11}},{b.cycle(),{11,999}},{0,{11}},{-1,{11}},{b.cycle()+1,{11}}}; + for(const auto& selector:bad){ + const auto original=selector.incarnations; + CHECK(b.inspect(x::Flatten{},fill(100),selector).status==x::Status::InvalidCloseTarget); + CHECK(b.project(x::Flatten{},fill(100),selector).status==x::Status::InvalidCloseTarget); + CHECK(b.settle(x::Flatten{},fill(100),selector).status==x::Status::InvalidCloseTarget); + CHECK(selector.incarnations==original); + CHECK(b.broker_state_hash()==hash&&b.stream_state_hash()==stream); + CHECK(b.next_cycle()==next&&b.next_order()==order&&b.actions()==actions&&b.rows().empty()); + } + const x::SelectedOpeningSet valid{b.cycle(),{22,11}}; + CHECK(b.inspect(order_action::Transact{sign},fill(100),valid).status==x::Status::InvalidCloseTarget); + CHECK(b.project(order_action::Transact{sign},fill(100),valid).status==x::Status::InvalidCloseTarget); + CHECK(b.settle(order_action::Transact{sign},fill(100),valid).status==x::Status::InvalidCloseTarget); + CHECK(b.settle(order_action::Reduce{0},fill(100),valid).status==x::Status::NoEffect); + for(double fee:{1.0,-1.0}) + CHECK(b.settle(order_action::Reduce{0},fill(100,fee),valid).status==x::Status::InvalidAccounting); + CHECK(b.broker_state_hash()==hash&&b.stream_state_hash()==stream); +} + +void unequal_fee_attribution(double sign,std::optional total,double first_fee,double last_fee){ + scenario="selected FIFO and unequal one-ticket attribution";Book b; + b.open(sign,100,11,"A");b.open(sign*2,105,22,"untouched",2);b.open(sign*3,100,33,"C"); + const PyramidEntry untouched=b.lots()[1]; + x::SelectedOpeningSet selected{b.cycle(),{33,11}};const auto ids=selected.incarnations; + const auto before=b.broker_state_hash(); + const auto q=b.inspect(x::Flatten{},fill(100+sign*10,total),selected); + REQUIRE(q.status==x::Status::Applied);CHECK(q.closed_units==4&&q.resulting_lot_count==1); + near(q.current_ticket,first_fee+last_fee);CHECK(b.broker_state_hash()==before); + const auto r=b.settle(x::Flatten{},fill(100+sign*10,total),selected); + REQUIRE(r.status==x::Status::Applied);CHECK(r.closed_units==4&&r.closed_trade_count==2); + CHECK(selected.incarnations==ids);REQUIRE(b.rows().size()==2&&b.lots().size()==1); + CHECK(b.rows()[0].entry_incarnation==11&&b.rows()[1].entry_incarnation==33); + near(b.rows()[0].commission,first_fee);near(b.rows()[1].commission,last_fee); + near(b.rows()[0].pnl,10-first_fee);near(b.rows()[1].pnl,30-last_fee); + near(b.net(),40-first_fee-last_fee); + CHECK(b.lots()[0].entry_incarnation==untouched.entry_incarnation); + CHECK(b.lots()[0].price==untouched.price&&b.lots()[0].qty==untouched.qty); + CHECK(b.lots()[0].time==untouched.time&&b.lots()[0].entry_commission_account==untouched.entry_commission_account); + CHECK(b.cycle()==selected.cycle); +} + +void fragments_and_schedule_change(double sign){ + scenario="selected legal fragments and historical fee residue";Book b; + b.schedule(CommissionType::PERCENT,10); + b.open(sign,100,11,"A-fragment-1",2);b.open(sign*3,100,22,"other",3); + b.open(sign*2,100,11,"A-fragment-2",4); + b.schedule(CommissionType::CASH_PER_ORDER,6); + const x::SelectedOpeningSet selection{b.cycle(),{11}}; + const auto r=b.settle(order_action::Reduce{1.5},fill(100+sign*10),selection); + REQUIRE(r.status==x::Status::Applied);CHECK(r.closed_units==1.5&&r.closed_trade_count==2); + REQUIRE(b.rows().size()==2&&b.lots().size()==2); + CHECK(b.rows()[0].entry_incarnation==11&&b.rows()[1].entry_incarnation==11); + near(b.rows()[0].qty,1);near(b.rows()[1].qty,.5); + // Current shares4/2 plus historical2/1. A schedule change cannot reprice the past. + near(b.rows()[0].commission,6);near(b.rows()[1].commission,3); + near(b.rows()[0].pnl,4);near(b.rows()[1].pnl,2); + CHECK(b.lots()[0].entry_incarnation==22);near(b.lots()[0].qty,3);near(b.lots()[0].entry_commission_account,3); + CHECK(b.lots()[1].entry_incarnation==11);near(b.lots()[1].qty,1.5);near(b.lots()[1].entry_commission_account,3); +} + +void per_unit_and_negative_price_percent(double sign){ + scenario="selected cash-per-unit with scalar FX";Book units; + units.schedule(CommissionType::CASH_PER_CONTRACT,2,3,2); + units.open(sign,100,11,"A");units.open(sign*3,100,22,"B"); + const auto u=units.settle(x::Flatten{},fill(100+sign*10),{units.cycle(),{22,11}}); + REQUIRE(u.status==x::Status::Applied);near(u.current_ticket,8); + near(units.rows()[0].commission,2);near(units.rows()[1].commission,6);near(units.net(),232); + scenario="selected absolute-notional percent at negative price";Book percent; + percent.schedule(CommissionType::PERCENT,1,2,3); + percent.open(sign,-100,11,"A");percent.open(sign*3,-100,22,"B"); + const auto p=percent.settle(x::Flatten{},fill(-100+sign*10),{percent.cycle(),{22,11}}); + REQUIRE(p.status==x::Status::Applied); + const double one=sign>0?5.4:6.6; + near(p.current_ticket,one*4);near(percent.rows()[0].commission,one);near(percent.rows()[1].commission,one*3); + near(percent.net(),240-one*4); +} + +void interior_dust(double sign){ + scenario="selected interior positive residue";Book b;b.schedule(CommissionType::CASH_PER_ORDER,0); + b.open(sign,100,11,"A",6);b.open(sign*3,100,22,"B",0); + const double q=1.0-5e-12;const double residue=1.0-q; + const x::SelectedOpeningSet selection{b.cycle(),{11}}; + const auto r=b.settle(order_action::Reduce{q},fill(100+sign*10),selection); + REQUIRE(r.status==x::Status::Applied);CHECK(r.closed_units==q&&r.closed_trade_count==1); + REQUIRE(b.lots().size()==2);CHECK(b.lots()[0].entry_incarnation==11&&b.lots()[0].qty==residue); + CHECK(b.lots()[0].qty>0&&b.lots()[0].entry_commission_account>0); + CHECK(b.lots()[0].entry_commission_account==6.0-6.0*q); + CHECK(b.lots()[1].entry_incarnation==22&&b.lots()[1].qty==3); + CHECK(b.qty()==b.lots()[0].qty+b.lots()[1].qty);CHECK(b.cycle()==selection.cycle); + CHECK(b.rows()[0].qty==q);near(b.rows()[0].commission,6*q); +} + +templatevoid run(F fn){try{fn();}catch(const Abort&){}catch(const std::exception& e){++failures;std::printf("FAIL %s exception %s\n",scenario,e.what());}} +} +int main(){ + for(double sign:{1.0,-1.0}){ + run([&]{invalid_sets(sign);}); + run([&]{unequal_fee_attribution(sign,std::nullopt,1.5,4.5);}); + run([&]{unequal_fee_attribution(sign,12,3,9);}); + run([&]{unequal_fee_attribution(sign,0,0,0);}); + run([&]{unequal_fee_attribution(sign,-6,-1.5,-4.5);}); + run([&]{fragments_and_schedule_change(sign);}); + run([&]{per_unit_and_negative_price_percent(sign);}); + run([&]{interior_dust(sign);}); + } + std::printf("%s native selected settlement: %d checks %d failures\n",failures?"FAIL":"PASS",checks,failures); + return failures?1:0; +} diff --git a/tests/test_native_settlement_callers.cpp b/tests/test_native_settlement_callers.cpp new file mode 100644 index 000000000..923bab0a0 --- /dev/null +++ b/tests/test_native_settlement_callers.cpp @@ -0,0 +1,251 @@ +// R3 live adapter helpers: actual source selection/sizing followed by the one book owner. +// Private member access uses the same explicit-instantiation pattern as R3a tests. +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge; +namespace x=pineforge::execution; +namespace { +int checks=0,failures=0;const char* scenario="setup"; +struct Abort{}; +#define CHECK(v) do{++checks;if(!(v)){++failures;std::printf("FAIL %s:%d %s\n",scenario,__LINE__,#v);}}while(0) +#define REQUIRE(v) do{bool ok_=bool(v);CHECK(ok_);if(!ok_)throw Abort{};}while(0) +void near(double a,double b){const bool ok=std::isfinite(a)&&std::isfinite(b)&&std::abs(a-b)<=1e-12*std::max(1.0,std::abs(b)); + if(!ok)std::printf(" actual=%.17g expected=%.17g\n",a,b);CHECK(ok);} +templatestruct Access{friend auto access(Tag){return Member;}}; +#define ACCESS(Tag,Method) struct Tag{friend auto access(Tag);}; template struct Access +ACCESS(Partial,execute_partial_exit_qty); +ACCESS(Percent,execute_partial_exit); +ACCESS(ByEntry,execute_partial_exit_by_entry); +ACCESS(ByQty,execute_partial_exit_by_entry_qty); +ACCESS(ByPercent,execute_partial_exit_by_entry_percent); +ACCESS(Scratch,cover_samebar_market_adds_on_exit); +ACCESS(Flip,flip_market_position_to); +ACCESS(Sequential,sequential_same_tick_reversal_fill); +ACCESS(Add,add_to_pyramid_market); +ACCESS(Entry,execute_market_entry); +ACCESS(Raw,apply_raw_order_fill); +ACCESS(Exit,apply_exit_order_fill); +ACCESS(Range,record_range_end_close_trades); +#undef ACCESS +templatestruct Args; +templatestruct Args{using tuple=std::tuple;}; +using Cause=std::tuple_element_t<2,typename Args::tuple>; +constexpr Cause Script=static_cast(0),Bracket=static_cast(1),Margin=static_cast(2); + +struct Book final:BacktestEngine{ + Book(){initial_capital_=1000;commission_type_=CommissionType::CASH_PER_ORDER;commission_value_=6; + syminfo_.pointvalue=1;syminfo_.mintick=.01;syminfo_mintick_=.01;account_currency_fx_=1; + pyramiding_=100;qty_step_=0;slippage_=0;stream_observe_actions_=true;bar(3,100);} + void on_bar(const Bar&)override{} + void bar(int index,double close){bar_index_=index;current_bar_={close,close+20,close-20,close,1,1736121600000LL+index*60000};} + void open(double q,double price,uint64_t inc,const char* label,double paid=0){ + x::PhysicalExecutionContext c{current_bar_.timestamp,bar_index_,{}, {}}; + REQUIRE(settle_native_execution_at(order_action::Transact{q},x::Fill{price,label,"",inc,paid},c).status==x::Status::Applied); + } + void partial(double p,double q,Cause c=Script){(this->*access(Partial{}))(p,q,c);} + void percent(double p,double q,Cause c=Script){(this->*access(Percent{}))(p,q,c);} + void by_qty(double p,const char* id,double q,Cause c=Script){(this->*access(ByQty{}))(p,std::string(id),q,c);} + void by_all(double p,const char* id,Cause c=Script){(this->*access(ByEntry{}))(p,std::string(id),c);} + void by_percent(double p,const char* id,double q,Cause c=Script){(this->*access(ByPercent{}))(p,std::string(id),q,c);} + double scratch(const PendingOrder& o,double p,Cause c=Bracket){return (this->*access(Scratch{}))(o,p,c);} + void flip(bool buy,double p,double q,int type,bool frozen=false,bool close_only=false){ + (this->*access(Flip{}))(std::string("flip"),buy,p,q,type,frozen,close_only,90); + } + void sequential(bool buy,double p,double q,int type){(this->*access(Sequential{}))(std::string("sequential"),buy,p,q,type,91);} + void old_add(bool buy,double p,double q){(this->*access(Add{}))(std::string("old-add"),buy,p,q,-1,position_side_,false,92);} + void dispatch(bool buy,double p,double q,bool later,bool frozen){ + (this->*access(Entry{}))(std::string("dispatch"),buy,p,q,-1,position_side_,false,false,0,bar_index_,later,false,frozen,93); + } + void raw(PendingOrder& o,double p){double trail=trail_best_price_;int closed_bar=-1;uint64_t closed_inc=0;bool closed_long=false; + (this->*access(Raw{}))(o,p,trail,closed_bar,closed_inc,closed_long);} + void exit(PendingOrder& o,double p){int closed_bar=-1;uint64_t closed_inc=0;bool closed_long=false; + (this->*access(Exit{}))(o,p,closed_bar,closed_inc,closed_long);} + void range(){record_equity_point(current_bar_.timestamp);(this->*access(Range{}))();} + void freeze(){freeze_script_position_view();} void unfreeze(){clear_script_position_view();} + double source_position()const{return signed_position_size();} + double size50(double p)const{return calc_qty_for_type(p,50,static_cast(QtyType::PERCENT_OF_EQUITY));} + x::AccountEffectProjection flatten_quote(double p)const{return project_native_settlement_v1(x::Flatten{},x::Fill{p,"quote","",90,{}});} + double physical()const{return position_side_==PositionSide::SHORT?-position_qty_:position_qty_;} + double qty()const{return position_qty_;}double balance()const{return initial_capital_+net_profit_sum_;} + double marked(double p)const{return marked_equity(p);}int64_t cycle()const{return position_cycle_seq_;} + int64_t next_cycle()const{return next_position_cycle_seq_;}int slots()const{return position_entry_count_;} + void slots(int n){position_entry_count_=n;}void step(double q){qty_step_=q;} + void fee(double q){commission_value_=q;}void pyramid(int n){pyramiding_=n;} + void default_percent(double q){default_qty_type_=QtyType::PERCENT_OF_EQUITY;default_qty_value_=q;} + void financial_scale(double pv,double fx){syminfo_.pointvalue=pv;account_currency_fx_=fx;} + void slip(int n,double tick){slippage_=n;syminfo_mintick_=tick;syminfo_.mintick=tick;} + void block_opposite(bool held_long){risk_direction_=held_long?RiskDirection::LONG_ONLY:RiskDirection::SHORT_ONLY;} + auto& lots(){return pyramid_entries_;}const auto& lots()const{return pyramid_entries_;} + const auto& rows()const{return trades_;}const auto& actions()const{return stream_order_actions_;} + double ledger(const char* id)const{auto at=id_unclosed_qty_.find(id);return at==id_unclosed_qty_.end()?0:at->second;} + void retain_exit(){PendingOrder o;o.id="retained";o.from_entry="old";o.type=OrderType::EXIT;o.incarnation=999;o.created_seq=999;pending_orders_.push_back(std::move(o));} + size_t pending()const{return pending_orders_.size();}const PendingOrder* pending_data()const{return pending_orders_.data();} +}; + +void by_entry_fragments(double sign){ + scenario="caller distinct incarnation construction across fragments";Book b; + b.open(sign,100,11,"A");b.open(sign*2,100,11,"A");b.open(sign*3,100,22,"Other");b.open(sign,100,33,"A"); + const auto other=b.lots()[2];b.by_qty(100+sign*10,"A",2.5); + REQUIRE(b.rows().size()==2);CHECK(b.rows()[0].entry_incarnation==11&&b.rows()[1].entry_incarnation==11); + near(b.rows()[0].qty,1);near(b.rows()[1].qty,1.5); + near(b.rows()[0].commission,2.4);near(b.rows()[1].commission,3.6); + REQUIRE(b.lots().size()==3);near(b.lots()[0].qty,.5);CHECK(b.lots()[1].entry_incarnation==22); + CHECK(b.lots()[1].qty==other.qty&&b.lots()[1].price==other.price&&b.lots()[1].entry_commission_account==other.entry_commission_account); + b.by_all(100+sign*10,"A");REQUIRE(b.lots().size()==1);CHECK(b.lots()[0].entry_incarnation==22); + near(b.physical(),sign*3);CHECK(b.rows().size()==4); + near(b.rows()[2].commission,2);near(b.rows()[3].commission,4); +} + +void refused_provenance(bool zero,bool scratch){ + scenario=scratch?"caller scratch heterogeneous provenance":"caller bound-close invalid provenance";Book b; + b.open(1,100,zero?0:11,"A");b.open(1,100,zero?22:11,"Other"); + b.open(3,100,33,"untouched"); + if(scratch)b.lots()[0].market_pyramid_add=true; + const auto hash=b.broker_state_hash();const auto action_count=b.actions().size();bool threw=false; + try{if(scratch){PendingOrder o;o.from_entry="A";o.legs.set_stop_price(99);(void)b.scratch(o,110);} + else b.by_all(110,"A");}catch(const std::runtime_error& e){threw=true;CHECK(std::string(e.what()).size()>0);} + CHECK(threw);CHECK(b.broker_state_hash()==hash&&b.rows().empty()&&b.actions().size()==action_count); + near(b.qty(),5); +} + +void caller_dust_and_percent(double sign){ + scenario="caller interior dust must not become selected Flatten";Book b;b.fee(0); + b.open(sign,100,11,"A",6);b.open(sign*3,100,22,"B"); + const double amount=1.0-5e-12;b.by_qty(100+sign*10,"A",amount); + REQUIRE(b.lots().size()==2);CHECK(b.rows()[0].qty==amount); + CHECK(b.lots()[0].qty==1.0-amount&&b.lots()[0].qty>0); + CHECK(b.lots()[0].entry_commission_account==6.0-6.0*amount); + CHECK(b.lots()[1].entry_incarnation==22&&b.lots()[1].qty==3); + CHECK(b.qty()==b.lots()[0].qty+b.lots()[1].qty); + scenario="caller percent quantity floor leaves real remainder";Book p;p.fee(0);p.open(sign*5.4103,100,11,"A");p.step(.0001); + p.percent(100,50);REQUIRE(p.rows().size()==1&&p.lots().size()==1); + near(p.rows()[0].qty,2.7051);near(p.qty(),2.7052); +} + +void flips_and_sequential(double held_sign){ + scenario="F7 held plus new size versus F8 total quantity"; + const bool buy=held_sign<0;const double price=100+held_sign*10; + Book flip;flip.open(held_sign*3,100,11,"old");flip.retain_exit();const auto* pending=flip.pending_data(); + flip.flip(buy,price,1,-1,true);REQUIRE(flip.lots().size()==1&&flip.rows().size()==1); + near(flip.physical(),-held_sign);near(flip.rows()[0].qty,3);CHECK(flip.rows()[0].exit_price==price&&flip.lots()[0].price==price); + near(flip.rows()[0].commission+flip.lots()[0].entry_commission_account,6); + CHECK(flip.pending()==1&&flip.pending_data()==pending); + Book small;small.open(held_sign*3,100,11,"old");small.retain_exit();const auto* keep=small.pending_data(); + small.sequential(buy,price,1,-1);CHECK(small.lots().empty()&&small.cycle()==0&&small.rows().size()==1); + CHECK(small.pending()==1&&small.pending_data()==keep);near(small.rows()[0].commission,6); + Book cross;cross.open(held_sign*3,100,11,"old");cross.sequential(buy,price,4,-1); + REQUIRE(cross.lots().size()==1);near(cross.physical(),-held_sign);near(cross.rows()[0].qty,3); + near(cross.rows()[0].commission+cross.lots()[0].entry_commission_account,6); + Book close;close.open(held_sign*3,100,11,"old");close.flip(buy,price,99,-1,true,true); + CHECK(close.lots().empty()&&close.cycle()==0&&close.rows().size()==1); +} + +void projected_percent_flip(bool default_quantity){ + scenario=default_quantity?"F7 default percent consumes close-only quote":"F7 explicit percent consumes close-only quote"; + Book b;b.open(1,100,11,"old");b.open(3,100,22,"old");b.default_percent(50); + const auto hash=b.broker_state_hash();const auto q=b.flatten_quote(110); + REQUIRE(q.status==x::Status::Applied);CHECK(b.broker_state_hash()==hash); + near(q.realized_balance,1034);near(q.current_ticket,6);near(b.size50(110),4.5454545454545459); + b.flip(false,110,default_quantity?std::numeric_limits::quiet_NaN():50, + default_quantity?-1:static_cast(QtyType::PERCENT_OF_EQUITY)); + REQUIRE(b.lots().size()==1&&b.rows().size()==2); + near(b.physical(),-4.7000000000000002);near(b.rows()[0].commission,.68965517241379315); + near(b.rows()[1].commission,2.0689655172413794);near(b.lots()[0].entry_commission_account,3.2413793103448274); + near(b.balance(),1037.2413793103448);near(b.marked(110),1034); +} + +void frozen_quantity_provenance(double sign){ + scenario="F8 frozen off-grid quantity versus retained old wrapper";Book b;b.open(-sign*3,100,11,"old");b.step(1); + b.dispatch(sign>0,100,4.25,true,true);REQUIRE(b.lots().size()==1);CHECK(b.physical()==sign*1.25); + Book old;old.open(-sign*3,100,11,"old");old.step(1);old.sequential(sign>0,100,4.25,-1); + near(old.physical(),sign); // Original no-provenance wrapper intentionally floors4.25 to4. + scenario="F11 frozen same-side add versus retained old wrapper";Book add;add.open(sign*3,100,11,"old");add.step(1); + add.dispatch(sign>0,100,1.25,false,true);REQUIRE(add.lots().size()==2);CHECK(add.lots().back().qty==1.25); + CHECK(add.lots().back().market_pyramid_add);CHECK(add.lots().back().entry_incarnation==93); + CHECK(add.ledger("dispatch")==1.25);near(add.physical(),sign*4.25); + Book wrapper;wrapper.open(sign*3,100,11,"old");wrapper.step(1);wrapper.old_add(sign>0,100,1.25); + REQUIRE(wrapper.lots().size()==2);near(wrapper.lots().back().qty,1); + scenario="F7 cash quantity keeps price pointvalue and FX basis";Book cash;cash.financial_scale(2,2); + cash.open(-sign*3,100,11,"old");cash.flip(sign>0,100,1000,static_cast(QtyType::CASH)); + REQUIRE(cash.lots().size()==1);near(cash.physical(),sign*2.5); + CHECK(cash.lots()[0].price==100&&cash.rows()[0].exit_price==100); + near(cash.rows()[0].commission+cash.lots()[0].entry_commission_account,6); +} + +void raw_cycles_and_noeffect(double sign){ + scenario="RAW one cycle allocator and after-Applied stamps";Book b; + PendingOrder o;o.type=OrderType::RAW_ORDER;o.id="raw";o.is_long=sign>0;o.qty=1.25;o.incarnation=11; + o.created_position_side=PositionSide::FLAT;b.raw(o,100); + REQUIRE(b.lots().size()==1);CHECK(b.cycle()==1&&b.next_cycle()==2);CHECK(b.ledger("raw")==1.25); + PendingOrder add=o;add.id="raw-add";add.incarnation=22;add.created_position_side=sign>0?PositionSide::LONG:PositionSide::SHORT; + b.raw(add,100);REQUIRE(b.lots().size()==2);CHECK(b.cycle()==1&&b.next_cycle()==2); + CHECK(b.lots().back().market_pyramid_add&&b.lots().back().entry_incarnation==22); + CHECK(b.ledger("raw-add")==1.25);near(b.lots().back().entry_commission_account,6); + b.open(sign,100,33,"unrelated-native-lot");REQUIRE(!b.lots().back().market_pyramid_add); + const auto rows=b.rows().size(),actions=b.actions().size(),lots=b.lots().size();const auto last=b.lots().back(); + b.old_add(sign>0,100,0);CHECK(b.lots().size()==lots&&b.rows().size()==rows&&b.actions().size()==actions); + CHECK(b.lots().back().entry_incarnation==last.entry_incarnation&&b.lots().back().market_pyramid_add==last.market_pyramid_add); +} + +void source_slots_and_scratch(){ + scenario="helper-local bracket/script/margin slot restoration"; + for(Cause cause:{Bracket,Script,Margin}){Book b;b.open(1,100,11,"A");b.open(2,100,22,"B");b.slots(7); + b.partial(110,1,cause);CHECK(b.lots().size()==1);CHECK(b.slots()==(cause==Bracket?7:1));} + scenario="R20 release remains after bracket slot restoration";Book unique;unique.pyramid(2); + unique.bar(0,100);unique.open(1,100,11,"A");unique.bar(1,100);unique.open(1,100,22,"B");unique.bar(3,110); + PendingOrder o;o.type=OrderType::EXIT;o.id="X";o.from_entry="A";o.qty=1;o.qty_percent=50;o.incarnation=99;o.created_seq=99; + o.created_bar=2;o.created_position_side=PositionSide::LONG;o.created_position_cycle_seq=unique.cycle(); + o.quantity_request.request(QuantityIntent::units(1));o.quantity_request.reserve(1,2);o.legs.set_limit_price(110); + unique.exit(o,110);REQUIRE(unique.lots().size()==1);CHECK(unique.lots()[0].entry_incarnation==22&&unique.slots()==1); + scenario="KI62 second one-ticket fill after primary bracket";Book scratch; + scratch.bar(1,100);scratch.open(1,100,55,"A");scratch.bar(3,110); + scratch.open(1,100,11,"A");scratch.open(2,100,11,"A");scratch.open(4,100,22,"Other"); + scratch.lots()[1].market_pyramid_add=true;scratch.lots()[2].market_pyramid_add=true;scratch.slots(5); + PendingOrder bracket;bracket.type=OrderType::EXIT;bracket.id="scratch-bracket";bracket.from_entry="A"; + bracket.qty=1;bracket.qty_percent=12.5;bracket.incarnation=100;bracket.created_seq=100; + bracket.quantity_request.request(QuantityIntent::units(1));bracket.quantity_request.reserve(1,8); + bracket.legs.set_limit_price(110); + scratch.exit(bracket,110);REQUIRE(scratch.rows().size()==3&&scratch.lots().size()==1); + CHECK(scratch.rows()[0].entry_incarnation==55&&scratch.rows()[1].entry_incarnation==11&&scratch.rows()[2].entry_incarnation==11); + near(scratch.rows()[0].commission,6);near(scratch.rows()[1].commission,2);near(scratch.rows()[2].commission,4); + CHECK(scratch.rows()[1].entry_bar_index==scratch.rows()[1].exit_bar_index); + CHECK(scratch.rows()[2].entry_bar_index==scratch.rows()[2].exit_bar_index); + CHECK(scratch.slots()==5&&scratch.lots()[0].entry_incarnation==22); + const auto hash=scratch.broker_state_hash();CHECK(scratch.scratch(bracket,110,Bracket)==0);CHECK(scratch.broker_state_hash()==hash); +} + +void direction_blocked_one_slip(double sign){ + scenario="direction-blocked F1 resolves slippage once";Book b;b.open(sign,100,11,"old");b.slip(2,.25);b.block_opposite(sign>0); + b.dispatch(sign<0,100,1,false,false);REQUIRE(b.rows().size()==1&&b.lots().empty()); + CHECK(b.rows()[0].exit_price==100-sign*.5);near(b.rows()[0].pnl,-6.5); +} + +void nonphysical_observations(){ + scenario="range-end and frozen source view do not settle exposure";Book b;b.open(1,100,11,"A",2);b.open(3,100,22,"B",6); + const auto lots=b.lots();const auto rows=b.rows().size(),actions=b.actions().size();const auto cycle=b.cycle(),next=b.next_cycle(); + const double balance=b.balance();b.freeze();CHECK(b.source_position()==4);CHECK(b.physical()==4);b.unfreeze(); + b.bar(4,110);b.range();CHECK(b.rows().size()==rows&&b.actions().size()==actions); + CHECK(b.report_trade_count()==2&&b.get_report_trade(0).open_at_end&&b.get_report_trade(1).open_at_end); + CHECK(b.cycle()==cycle&&b.next_cycle()==next&&b.balance()==balance&&b.lots().size()==lots.size()); + for(size_t i=0;ivoid run(F fn){try{fn();}catch(const Abort&){}catch(const std::exception& e){++failures;std::printf("FAIL %s exception %s\n",scenario,e.what());}} +} +int main(){ + for(double sign:{1.0,-1.0}){run([&]{by_entry_fragments(sign);});run([&]{caller_dust_and_percent(sign);}); + run([&]{flips_and_sequential(sign);});run([&]{frozen_quantity_provenance(sign);}); + run([&]{raw_cycles_and_noeffect(sign);});run([&]{direction_blocked_one_slip(sign);});} + run([]{refused_provenance(true,false);});run([]{refused_provenance(false,false);});run([]{refused_provenance(false,true);}); + run([]{projected_percent_flip(false);});run([]{projected_percent_flip(true);});run(source_slots_and_scratch);run(nonphysical_observations); + std::printf("%s native settlement callers: %d checks %d failures\n",failures?"FAIL":"PASS",checks,failures); + return failures?1:0; +} diff --git a/tests/test_native_settlement_projection.cpp b/tests/test_native_settlement_projection.cpp new file mode 100644 index 000000000..d3f86d4c7 --- /dev/null +++ b/tests/test_native_settlement_projection.cpp @@ -0,0 +1,233 @@ +// Non-applying R3 account projection against the actual allocation/settlement owner. +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge; +namespace x=pineforge::execution; +namespace { +int checks=0,failures=0; +const char* scenario="setup"; +struct Abort{}; +#define CHECK(v) do{++checks;if(!(v)){++failures;std::printf("FAIL %s:%d %s\n",scenario,__LINE__,#v);}}while(0) +#define REQUIRE(v) do{bool ok_=bool(v);CHECK(ok_);if(!ok_)throw Abort{};}while(0) +void near(double a,double b){bool ok=std::isfinite(a)&&std::isfinite(b)&&std::abs(a-b)<=1e-12*std::max(1.0,std::abs(b)); + if(!ok)std::printf(" actual=%.17g expected=%.17g\n",a,b);CHECK(ok);} +struct Book final:BacktestEngine{ + Book(){initial_capital_=1000;commission_type_=CommissionType::CASH_PER_ORDER;commission_value_=6; + syminfo_.pointvalue=1;account_currency_fx_=1;stream_observe_actions_=true; + current_bar_={100,100,100,100,1,1736121600000LL};bar_index_=3;} + void on_bar(const Bar&)override{} + x::PhysicalExecutionContext context()const{return {current_bar_.timestamp,bar_index_,{}, {}};} + x::Result settle(const x::Action& a,const x::Fill& f){return settle_native_execution_at(a,f,context());} + x::Result settle(const x::Action& a,const x::Fill& f,const x::SelectedOpeningSet& s){return settle_native_execution_selected_at(a,f,context(),s);} + x::AccountEffectProjection project(const x::Action& a,const x::Fill& f)const{return project_native_settlement_v1(a,f);} + x::AccountEffectProjection project(const x::Action& a,const x::Fill& f,x::CloseScope s)const{return project_native_settlement_scoped_v1(a,f,s);} + x::AccountEffectProjection project(const x::Action& a,const x::Fill& f,const x::SelectedOpeningSet& s)const{return project_native_settlement_selected_v1(a,f,s);} + x::SettlementInspection inspect(const x::Action& a,const x::Fill& f)const{return inspect_native_settlement(a,f);} + void open(double q,double p,uint64_t inc,double paid=0){REQUIRE(settle(order_action::Transact{q},x::Fill{p,"seed","",inc,paid}).status==x::Status::Applied);} + double balance()const{return initial_capital_+net_profit_sum_;} + double marked(double p)const{return marked_equity(p);} + double paid()const{double v=0;for(const auto& lot:pyramid_entries_)v+=lot.entry_commission_account;return v;} + double signed_qty()const{return position_side_==PositionSide::SHORT?-position_qty_:position_qty_;} + double net()const{return net_profit_sum_;} + int64_t cycle()const{return position_cycle_seq_;} + int64_t next_cycle()const{return next_position_cycle_seq_;} + uint64_t next_order()const{return next_order_incarnation_;} + size_t lots()const{return pyramid_entries_.size();} + size_t rows()const{return trades_.size();} + size_t actions()const{return stream_order_actions_.size();} + const Trade& row(size_t n)const{return trades_[n];} + void initial(double value){initial_capital_=value;} + void cash_fee(double value){commission_value_=value;} + void schedule(CommissionType kind,double value){commission_type_=kind;commission_value_=value;} + void force_next_cycle(int64_t value){next_position_cycle_seq_=value;} + void account_boundary(double initial,double net){initial_capital_=initial;net_profit_sum_=net;} +}; +x::Fill fill(double price,uint64_t inc=90,std::optional fee={}){return {price,"projection","",inc,fee};} +struct Snapshot{ + uint64_t broker,stream,next_order;int64_t cycle,next_cycle;size_t lots,rows,actions; + explicit Snapshot(const Book& b):broker(b.broker_state_hash()),stream(b.stream_state_hash()),next_order(b.next_order()), + cycle(b.cycle()),next_cycle(b.next_cycle()),lots(b.lots()),rows(b.rows()),actions(b.actions()){} + void unchanged(const Book& b)const{CHECK(b.broker_state_hash()==broker&&b.stream_state_hash()==stream); + CHECK(b.next_order()==next_order&&b.cycle()==cycle&&b.next_cycle()==next_cycle); + CHECK(b.lots()==lots&&b.rows()==rows&&b.actions()==actions);} +}; +void quote_facts(const x::AccountEffectProjection& q,x::Status status,double closed,double opened, + double abs_units,size_t lots,double notional,double ticket,double balance, + double paid,double marked,int64_t cycle,double signed_units){ + CHECK(q.status==status);near(q.closed_units,closed);near(q.opened_units,opened); + near(q.resulting_abs_units,abs_units);CHECK(q.resulting_lot_count==lots);near(q.resulting_abs_notional,notional); + near(q.current_ticket,ticket);near(q.realized_balance,balance);near(q.remaining_entry_cost,paid); + near(q.marked_equity,marked);CHECK(q.cycle_after==cycle);near(q.signed_units_after,signed_units); + CHECK(q.would_open==(opened!=0));CHECK(q.incoming_short==(status==x::Status::Applied&&opened<0)); +} +void matches_commit(const Book& b,const x::AccountEffectProjection& q,const x::Result& r,double mark){ + CHECK(r.status==q.status);near(r.closed_units,q.closed_units);near(r.opened_units,q.opened_units); + near(r.current_ticket,q.current_ticket);near(b.balance(),q.realized_balance);near(b.marked(mark),q.marked_equity); + near(b.paid(),q.remaining_entry_cost);near(b.signed_qty(),q.signed_units_after); + CHECK(b.lots()==q.resulting_lot_count&&b.cycle()==q.cycle_after); +} + +void opening_add_flip(double sign){ + scenario="projection flat opening cost once";Book flat; + const auto f=fill(100);const Snapshot before(flat); + const auto q=flat.project(order_action::Transact{sign*2},f);before.unchanged(flat); + quote_facts(q,x::Status::Applied,0,sign*2,2,1,200,6,1000,6,994,1,sign*2); + matches_commit(flat,q,flat.settle(order_action::Transact{sign*2},f),100);CHECK(flat.next_cycle()==2); + + scenario="projection same-side add cycle and paid costs";Book add;add.open(sign,100,11,2); + const double price=100+sign*10;const Snapshot add_before(add); + const auto a=add.project(order_action::Transact{sign*3},fill(price));add_before.unchanged(add); + quote_facts(a,x::Status::Applied,0,sign*3,4,2,4*price,6,1000,8,1002,1,sign*4); + matches_commit(add,a,add.settle(order_action::Transact{sign*3},fill(price)),price);CHECK(add.next_cycle()==2); + + scenario="projection crossing cost allocation and fresh cycle";Book flip;flip.open(-sign,100,11,2); + const double crossed=100+sign*10;const Snapshot flip_before(flip); + const auto p=flip.project(order_action::Transact{sign*3},fill(crossed));flip_before.unchanged(flip); + quote_facts(p,x::Status::Applied,1,sign*2,2,1,2*crossed,6,986,4,982,2,sign*2); + matches_commit(flip,p,flip.settle(order_action::Transact{sign*3},fill(crossed)),crossed);CHECK(flip.next_cycle()==3); +} + +void partial_and_selected_flatten(double sign){ + scenario="projection partial Book close";Book b;b.open(sign,100,11,2);b.open(sign*3,100+sign*10,22,6); + const double price=100+sign*20;const Snapshot before(b); + const auto q=b.project(order_action::Reduce{1},fill(price));before.unchanged(b); + quote_facts(q,x::Status::Applied,1,0,3,1,3*price,6,1012,6,1036,1,sign*3); + matches_commit(b,q,b.settle(order_action::Reduce{1},fill(price)),price); + + scenario="projection selected Flatten leaves sibling";Book selected;selected.open(sign,100,11,2);selected.open(sign*3,100+sign*10,22,6); + const x::SelectedOpeningSet one{selected.cycle(),{22}};const Snapshot original(selected); + const auto set=selected.project(x::Flatten{},fill(price),one); + const auto singleton=selected.project(x::Flatten{},fill(price),x::OpeningExposure{22,selected.cycle()});original.unchanged(selected); + quote_facts(set,x::Status::Applied,3,0,1,1,price,6,1018,2,1036,1,sign); + CHECK(singleton.status==set.status);near(singleton.realized_balance,set.realized_balance); + near(singleton.marked_equity,set.marked_equity);near(singleton.remaining_entry_cost,set.remaining_entry_cost); + matches_commit(selected,set,selected.settle(x::Flatten{},fill(price),one),price); + + scenario="projection selected Flatten entire roster";Book all;all.open(sign,100,11,2);all.open(sign*3,100+sign*10,22,6); + const x::SelectedOpeningSet both{all.cycle(),{22,11}};const Snapshot all_before(all); + const auto p=all.project(x::Flatten{},fill(price),both);all_before.unchanged(all); + quote_facts(p,x::Status::Applied,4,0,0,0,0,6,1036,0,1036,0,0); + matches_commit(all,p,all.settle(x::Flatten{},fill(price),both),price); +} + +void invalid_quote(const x::AccountEffectProjection& p,x::Status status){ + CHECK(p.status==status);CHECK(p.closed_units==0&&p.opened_units==0&&p.resulting_abs_units==0); + CHECK(p.resulting_lot_count==0&&p.resulting_abs_notional==0&&p.current_ticket==0); + CHECK(!p.would_open&&!p.incoming_short);CHECK(p.realized_balance==0&&p.remaining_entry_cost==0&&p.marked_equity==0); + CHECK(p.cycle_after==0&&p.signed_units_after==0); +} +void invalid_inspect(const x::SettlementInspection& p,x::Status status){ + CHECK(p.status==status);CHECK(p.closed_units==0&&p.opened_units==0&&p.resulting_abs_units==0); + CHECK(p.resulting_lot_count==0&&p.resulting_abs_notional==0&&p.current_ticket==0); + CHECK(!p.would_open&&!p.incoming_short); +} +void invalid_result(const x::Result& r,x::Status status){ + CHECK(r.status==status);CHECK(r.closed_units==0&&r.opened_units==0&&r.current_ticket==0); + CHECK(r.first_trade_index==0&&r.closed_trade_count==0&&r.opened_lot_incarnation==0); +} +void noeffect_and_invalid(double sign){ + scenario="projection valid NoEffect live and empty";Book b;b.open(sign*2,100,11,6); + const double mark=100+sign*10;const Snapshot before(b); + const auto p=b.project(order_action::Reduce{0},fill(mark));before.unchanged(b); + quote_facts(p,x::Status::NoEffect,0,0,2,1,2*mark,0,1000,6,1014,1,sign*2); + CHECK(b.project(order_action::Reduce{0},fill(mark,90,0)).status==x::Status::NoEffect); + for(double fee:{1.0,-1.0})invalid_quote(b.project(order_action::Reduce{0},fill(mark,90,fee)),x::Status::InvalidAccounting); + invalid_quote(b.project(order_action::Reduce{-1},fill(mark)),x::Status::InvalidQuantity); + invalid_quote(b.project(x::Flatten{},fill(std::numeric_limits::quiet_NaN())),x::Status::InvalidPrice); + invalid_quote(b.project(x::Flatten{},fill(mark),x::SelectedOpeningSet{b.cycle(),{999}}),x::Status::InvalidCloseTarget); + before.unchanged(b); + Book empty;const Snapshot flat(empty);const auto e=empty.project(x::Flatten{},fill(100));flat.unchanged(empty); + quote_facts(e,x::Status::NoEffect,0,0,0,0,0,0,1000,0,1000,0,0); + scenario="projection added account overflow is not inspection status"; + Book huge;huge.account_boundary(std::numeric_limits::max(),std::numeric_limits::max()); + const Snapshot boundary(huge);CHECK(huge.inspect(order_action::Reduce{0},fill(100)).status==x::Status::NoEffect); + invalid_quote(huge.project(order_action::Reduce{0},fill(100)),x::Status::InvalidAccounting);boundary.unchanged(huge); +} + +void explicit_ticket_projection(double ticket){ + scenario="projection explicit ticket waiver or rebate";Book b;b.open(1,100,11);b.open(3,100,22); + const Snapshot before(b);const auto p=b.project(x::Flatten{},fill(110,90,ticket));before.unchanged(b); + quote_facts(p,x::Status::Applied,4,0,0,0,0,ticket,1040-ticket,0,1040-ticket,0,0); + matches_commit(b,p,b.settle(x::Flatten{},fill(110,90,ticket)),110); +} + +void accumulation_order(bool paid_cost){ + scenario=paid_cost?"projection per-lot marked-equity association":"projection exact commit-order balance"; + Book b;b.initial(1);b.cash_fee(0);const double high=10000000000000100.0; + b.open(1,high,1);REQUIRE(b.settle(x::Flatten{},fill(100,90,0)).status==x::Status::Applied); + CHECK(b.net()==-1e16); + b.open(1,100,2,paid_cost?1:0);b.open(.5,high-2,3); + const Snapshot before(b); + const auto unchanged=b.project(order_action::Reduce{0},fill(high,90,0));before.unchanged(b); + CHECK(unchanged.status==x::Status::NoEffect);CHECK(unchanged.realized_balance==-1e16); + CHECK(unchanged.marked_equity==1);CHECK(unchanged.remaining_entry_cost==(paid_cost?1:0)); + CHECK(unchanged.marked_equity==b.marked(high)); + const auto p=b.project(x::Flatten{},fill(high,90,0));before.unchanged(b); + CHECK(p.status==x::Status::Applied&&p.realized_balance==2&&p.marked_equity==2); + const size_t first=b.rows();const auto r=b.settle(x::Flatten{},fill(high,90,0)); + REQUIRE(r.status==x::Status::Applied&&b.rows()==first+2); + CHECK(b.row(first).pnl==1e16&&b.row(first+1).pnl==1); + CHECK(b.balance()==2&&b.net()==1);matches_commit(b,p,r,high); +} + +void quote_is_not_authority(){ + scenario="projection selection is revalidated at commit";Book b;b.open(1,100,11);b.open(3,110,22); + const x::SelectedOpeningSet selected{b.cycle(),{11}};const Snapshot before(b); + const auto quote=b.project(x::Flatten{},fill(120),selected);REQUIRE(quote.status==x::Status::Applied);before.unchanged(b); + REQUIRE(b.settle(x::Flatten{},fill(120),selected).status==x::Status::Applied); + const Snapshot changed(b);CHECK(b.settle(x::Flatten{},fill(120),selected).status==x::Status::InvalidCloseTarget);changed.unchanged(b); + CHECK(b.lots()==1&&b.signed_qty()==3); + Book smaller;smaller.open(3,100,11);const x::SelectedOpeningSet live{smaller.cycle(),{11}}; + const auto old=smaller.project(x::Flatten{},fill(110),live);CHECK(old.closed_units==3);near(old.realized_balance,1024); + REQUIRE(smaller.settle(order_action::Reduce{1},fill(110)).status==x::Status::Applied); + const auto fresh=smaller.settle(x::Flatten{},fill(110),live);REQUIRE(fresh.status==x::Status::Applied); + CHECK(fresh.closed_units==2&&fresh.closed_units!=old.closed_units);near(smaller.balance(),1018); +} + +void finite_row_infinite_ticket(){ + // Three qty-1 lots @0 with paid 0; per-contract DBL_MAX/2 Flatten at DBL_MAX/2 + // keeps each row PnL finite 0 while 3*(DBL_MAX/2) overflows the current ticket. + scenario="finite row fee cannot apply infinite ticket"; + Book b;b.open(1,0,11,0);b.open(1,0,22,0);b.open(1,0,33,0); + REQUIRE(b.lots()==3&&b.signed_qty()==3&&b.paid()==0&&b.rows()==0&&b.net()==0); + const auto live=b.inspect(x::Flatten{},fill(0)); + REQUIRE(live.status==x::Status::Applied);near(live.current_ticket,6);CHECK(live.closed_units==3); + b.schedule(CommissionType::CASH_PER_CONTRACT,std::numeric_limits::max()/2.0); + const double price=std::numeric_limits::max()/2.0;const Snapshot before(b); + invalid_inspect(b.inspect(x::Flatten{},fill(price)),x::Status::InvalidAccounting);before.unchanged(b); + invalid_quote(b.project(x::Flatten{},fill(price)),x::Status::InvalidAccounting);before.unchanged(b); + invalid_result(b.settle(x::Flatten{},fill(price)),x::Status::InvalidAccounting);before.unchanged(b); + CHECK(b.lots()==3&&b.rows()==0&&b.actions()==before.actions&&b.signed_qty()==3&&b.paid()==0&&b.net()==0); +} + +void cycle_peek_exhaustion(){ + scenario="projection cycle exhaustion without consumption"; + for(int64_t next:{int64_t{0},int64_t{-1},std::numeric_limits::max()}){ + Book flat;flat.force_next_cycle(next);const Snapshot before(flat);bool threw=false; + try{(void)flat.project(order_action::Transact{1},fill(100));}catch(const std::overflow_error&){threw=true;} + CHECK(threw);before.unchanged(flat); + } + Book live;live.open(1,100,11);live.force_next_cycle(std::numeric_limits::max());const Snapshot existing(live); + const auto add=live.project(order_action::Transact{1},fill(100)); + CHECK(add.status==x::Status::Applied&&add.cycle_after==live.cycle()); + const auto zero=live.project(order_action::Reduce{0},fill(100));CHECK(zero.status==x::Status::NoEffect); + existing.unchanged(live); +} +templatevoid run(F fn){try{fn();}catch(const Abort&){}catch(const std::exception& e){++failures;std::printf("FAIL %s exception %s\n",scenario,e.what());}} +} +int main(){ + for(double sign:{1.0,-1.0}){run([&]{opening_add_flip(sign);});run([&]{partial_and_selected_flatten(sign);});run([&]{noeffect_and_invalid(sign);});} + run([]{explicit_ticket_projection(0);});run([]{explicit_ticket_projection(-6);}); + run([]{accumulation_order(false);});run([]{accumulation_order(true);});run(quote_is_not_authority); + run(finite_row_infinite_ticket);run(cycle_peek_exhaustion); + std::printf("%s native settlement projection: %d checks %d failures\n",failures?"FAIL":"PASS",checks,failures); + return failures?1:0; +} From 590ccc455c5dc4e9327f86c133885aff173fdefe Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 13 Sep 2026 04:27:13 +0800 Subject: [PATCH 2/4] Preserve release metadata when fetching the ABI base --- .github/workflows/ci.yml | 8 ++++++-- .github/workflows/native-live.yml | 4 +++- scripts/prepare_settlement_cpp_abi_base.py | 5 ++--- tests/fixtures/settlement_cpp_abi/README.md | 6 ++++-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3d59a7d5..b91a92e23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 + fetch-depth: 1 - name: Install Eigen3 (Linux) if: runner.os == 'Linux' @@ -41,6 +41,8 @@ jobs: - name: Prepare real R2 settlement ABI provider run: | + # Preserve shallow CI version metadata: fetch the old object without tags. + git fetch --no-tags --depth=1 origin e60e57156a54bb1c74c0c0a4f5a338fd924c046d python3 scripts/prepare_settlement_cpp_abi_base.py --source-repo . \ --current-build build --output build/settlement-abi-base --jobs 4 @@ -91,7 +93,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 + fetch-depth: 1 - name: Install Eigen3 run: | @@ -106,6 +108,8 @@ jobs: - name: Prepare real R2 settlement ABI provider run: | + # Preserve shallow CI version metadata: fetch the old object without tags. + git fetch --no-tags --depth=1 origin e60e57156a54bb1c74c0c0a4f5a338fd924c046d python3 scripts/prepare_settlement_cpp_abi_base.py --source-repo . \ --current-build build-asan --output build-asan/settlement-abi-base --jobs 4 diff --git a/.github/workflows/native-live.yml b/.github/workflows/native-live.yml index 129040fe8..298c1b8e2 100644 --- a/.github/workflows/native-live.yml +++ b/.github/workflows/native-live.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: persist-credentials: false - fetch-depth: 0 + fetch-depth: 1 - name: Install native build dependencies run: | sudo apt-get update @@ -46,6 +46,8 @@ jobs: cmake --build build-live -j 4 - name: Prepare real R2 settlement ABI provider run: | + # Preserve shallow CI version metadata: fetch the old object without tags. + git fetch --no-tags --depth=1 origin e60e57156a54bb1c74c0c0a4f5a338fd924c046d python3 scripts/prepare_settlement_cpp_abi_base.py --source-repo . \ --current-build build-live --output build-live/settlement-abi-base --jobs 4 - name: Test native engine, parser, journal and transports diff --git a/scripts/prepare_settlement_cpp_abi_base.py b/scripts/prepare_settlement_cpp_abi_base.py index 2bdd47252..9a2d40855 100644 --- a/scripts/prepare_settlement_cpp_abi_base.py +++ b/scripts/prepare_settlement_cpp_abi_base.py @@ -147,9 +147,8 @@ def main() -> int: commit = run(['git', '-C', str(repo), 'rev-parse', BASE_COMMIT + '^{commit}']).decode().strip() tree = run(['git', '-C', str(repo), 'rev-parse', BASE_COMMIT + '^{tree}']).decode().strip() except RuntimeError as error: - raise RuntimeError(f'exact R2 source is unavailable; use a full-history checkout in CI, ' - f'or fetch the object in a local shallow clone: ' - f'git fetch --no-tags --depth=1 origin {BASE_COMMIT}') from error + raise RuntimeError(f'exact R2 source is unavailable; fetch the pinned object without tags ' + f'before preparation: git fetch --no-tags --depth=1 origin {BASE_COMMIT}') from error if commit != BASE_COMMIT or tree != BASE_TREE: raise RuntimeError('R2 commit/tree mismatch') source_raw = run(['git', '-C', str(repo), 'archive', '--format=tar', BASE_COMMIT, diff --git a/tests/fixtures/settlement_cpp_abi/README.md b/tests/fixtures/settlement_cpp_abi/README.md index 02ef743a0..d08b4fd16 100644 --- a/tests/fixtures/settlement_cpp_abi/README.md +++ b/tests/fixtures/settlement_cpp_abi/README.md @@ -12,8 +12,10 @@ and UBSan RTTI controls. No generated pairing executable is run. ## Prepare the real old provider explicitly After configuring the current candidate build, ensure the exact R2 Git object -is available. CI uses full-history checkouts and needs no separate fetch. For a local shallow -clone, fetch the pinned object explicitly before preparation, outside CTest: +is available. CI retains shallow checkouts and explicitly fetches only the pinned old commit +without tags. This preserves its existing VERSION-based release metadata; a full +history/tag fetch would activate git-describe version selection. The same fetch +works for a local shallow clone, outside CTest: ```sh git fetch --no-tags --depth=1 origin e60e57156a54bb1c74c0c0a4f5a338fd924c046d From 0e18690db3fb6bb4705841c2a86936dfa206380a Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 13 Sep 2026 05:56:06 +0800 Subject: [PATCH 3/4] Share complete local and CI verification with reliable diagnostics --- .github/workflows/ci.yml | 133 ++-- .github/workflows/native-live.yml | 102 ++- .gitignore | 4 + AGENTS.md | 12 +- README.md | 8 +- cmake/PineForgeVersion.cmake | 47 +- docs/ci.md | 77 +++ scripts/ci_verify.py | 710 ++++++++++++++++++++ scripts/collect_ci_diagnostics.py | 88 +++ scripts/prepare_settlement_cpp_abi_base.py | 59 +- scripts/test_ci_verify.py | 710 ++++++++++++++++++++ scripts/test_cmake_version_source.py | 296 ++++++++ tests/CMakeLists.txt | 5 + tests/fixtures/settlement_cpp_abi/README.md | 10 +- 14 files changed, 2128 insertions(+), 133 deletions(-) create mode 100644 docs/ci.md create mode 100644 scripts/ci_verify.py create mode 100644 scripts/collect_ci_diagnostics.py create mode 100644 scripts/test_ci_verify.py create mode 100644 scripts/test_cmake_version_source.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b91a92e23..322f7e345 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,21 @@ on: pull_request: branches: [main] +permissions: + contents: read + +# Cancel superseded PR runs only. Main/post-merge uses a unique run_id group so +# in-progress and queued proof runs are never cancelled. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_COMPILERCHECK: content + CCACHE_COMPRESS: "1" + CCACHE_MAXSIZE: 500M + jobs: build: strategy: @@ -21,66 +36,41 @@ jobs: with: fetch-depth: 1 - - name: Install Eigen3 (Linux) + - name: Install dependencies (Linux) if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install -y libeigen3-dev + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libeigen3-dev ccache - - name: Install Eigen3 (macOS) + - name: Install dependencies (macOS) if: runner.os == 'macOS' run: | - brew install eigen - echo "CMAKE_PREFIX_PATH_EXTRA=$(brew --prefix eigen)" >> "$GITHUB_ENV" - - - name: Configure - run: cmake -B build -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} - - - name: Build - run: cmake --build build -j 4 - - - name: Prepare real R2 settlement ABI provider - run: | - # Preserve shallow CI version metadata: fetch the old object without tags. - git fetch --no-tags --depth=1 origin e60e57156a54bb1c74c0c0a4f5a338fd924c046d - python3 scripts/prepare_settlement_cpp_abi_base.py --source-repo . \ - --current-build build --output build/settlement-abi-base --jobs 4 - - - name: Test - run: ctest --test-dir build --output-on-failure + export HOMEBREW_NO_AUTO_UPDATE=1 + brew install eigen ccache + echo "CMAKE_PREFIX_PATH=$(brew --prefix eigen)${CMAKE_PREFIX_PATH:+:$CMAKE_PREFIX_PATH}" >> "$GITHUB_ENV" - - name: C ABI runtime source check - run: python3 scripts/check_c_abi_runtime.py - - - name: Broker-state hash coverage check - run: python3 scripts/check_broker_state_hash_coverage.py + - name: Restore ccache + uses: actions/cache@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}- - - name: Pending-order POD mirror up-to-date check - run: python3 scripts/gen_pending_order_mirror.py --check + - name: Verify (${{ matrix.build_type }}) + run: python3 scripts/ci_verify.py ${{ matrix.build_type == 'Release' && 'release' || 'debug' }} --build-dir build --jobs 4 --ccache - - name: Install PineForge - run: cmake --install build --prefix "$GITHUB_WORKSPACE/pf_install" + - name: Stage and summarize diagnostics + if: always() + run: python3 scripts/collect_ci_diagnostics.py --build-dir build --profile ${{ matrix.build_type == 'Release' && 'release' || 'debug' }} - - name: find_package smoke build - shell: bash - env: - WORKSPACE: ${{ github.workspace }} - BUILD_TYPE: ${{ matrix.build_type }} - run: | - set -euo pipefail - EXTRA="${CMAKE_PREFIX_PATH_EXTRA:-}" - PREFIX="${WORKSPACE}/pf_install" - # Pass prefix list to CMake with ';' — ':' is for env vars on Unix only. - CMAKE_PFX="${PREFIX}" - if [[ -n "$EXTRA" ]]; then - CMAKE_PFX="${PREFIX};${EXTRA}" - fi - cmake -S cmake/smoke_consumer -B smoke_build \ - -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" \ - -DCMAKE_PREFIX_PATH="${CMAKE_PFX}" - cmake --build smoke_build -j 4 - expected="$(tr -d '[:space:]' < VERSION)" - ./smoke_build/smoke_version | grep -qxF "${expected}" + - name: Retain CI diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: ci-diagnostics-${{ matrix.os }}-${{ matrix.build_type }} + path: ci-diagnostics/ + if-no-files-found: warn # Dedicated AddressSanitizer + UndefinedBehaviorSanitizer lane. Without this, # the determinism / NaN-propagation / handle-reuse / C-ABI-lifecycle probes @@ -90,34 +80,41 @@ jobs: # PINEFORGE_ENABLE_SANITIZERS). sanitizers: runs-on: ubuntu-latest + env: + ASAN_OPTIONS: detect_leaks=1:halt_on_error=1:abort_on_error=1 + UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 steps: - uses: actions/checkout@v4 with: fetch-depth: 1 - - name: Install Eigen3 + - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y libeigen3-dev + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libeigen3-dev ccache - - name: Configure (ASan + UBSan) - run: cmake -B build-asan -DCMAKE_BUILD_TYPE=Debug -DPINEFORGE_ENABLE_SANITIZERS=ON + - name: Restore ccache + uses: actions/cache@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ runner.arch }}-sanitizers-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ runner.arch }}-sanitizers- - - name: Build - run: cmake --build build-asan -j 4 + - name: Verify (sanitizers) + run: python3 scripts/ci_verify.py sanitizers --build-dir build-asan --jobs 4 --ccache - - name: Prepare real R2 settlement ABI provider - run: | - # Preserve shallow CI version metadata: fetch the old object without tags. - git fetch --no-tags --depth=1 origin e60e57156a54bb1c74c0c0a4f5a338fd924c046d - python3 scripts/prepare_settlement_cpp_abi_base.py --source-repo . \ - --current-build build-asan --output build-asan/settlement-abi-base --jobs 4 - - - name: Test under sanitizers - env: - ASAN_OPTIONS: detect_leaks=1:halt_on_error=1:abort_on_error=1 - UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 - run: ctest --test-dir build-asan --output-on-failure + - name: Stage and summarize diagnostics + if: always() + run: python3 scripts/collect_ci_diagnostics.py --build-dir build-asan --profile sanitizers + + - name: Retain CI diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: ci-diagnostics-sanitizers + path: ci-diagnostics/ + if-no-files-found: warn # The branch ruleset requires a status check named `build`, but the matrix # job above reports as `build (ubuntu-latest, Release)` and friends -- a bare diff --git a/.github/workflows/native-live.yml b/.github/workflows/native-live.yml index 298c1b8e2..932ebf58d 100644 --- a/.github/workflows/native-live.yml +++ b/.github/workflows/native-live.yml @@ -10,6 +10,20 @@ on: permissions: contents: read +# Cancel superseded PR runs only. Main/post-merge and manual workflow_dispatch +# use a unique run_id group so proof runs are never cancelled. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_COMPILERCHECK: content + CCACHE_COMPRESS: "1" + CCACHE_MAXSIZE: 500M + CURL_VERSION: "8.14.1" + CURL_SHA256: "f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd" + jobs: native-live: runs-on: ubuntu-latest @@ -19,18 +33,57 @@ jobs: with: persist-credentials: false fetch-depth: 1 + - name: Install native build dependencies run: | sudo apt-get update - sudo apt-get install -y cmake ninja-build libeigen3-dev libsqlite3-dev libssl-dev zlib1g-dev + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cmake ninja-build libeigen3-dev libsqlite3-dev libssl-dev zlib1g-dev ccache + + - name: Restore ccache + uses: actions/cache@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ccache-${{ runner.os }}-${{ runner.arch }}-native-${{ github.sha }} + restore-keys: | + ccache-${{ runner.os }}-${{ runner.arch }}-native- + + - name: Identify native dependency environment + id: native-deps + shell: bash + run: | + mkdir -p build-native-deps + { + printf '%s\n' "${ImageOS:-unknown}" "${ImageVersion:-unknown}" "${GITHUB_WORKSPACE}" + cc --version + cmake --version + dpkg-query -W libssl-dev zlib1g-dev + } > build-native-deps/environment.txt + echo "identity=$(sha256sum build-native-deps/environment.txt | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT" + + - name: Restore pinned WebSocket-enabled libcurl + id: curl-cache + uses: actions/cache@v4 + with: + path: build-native-deps/curl-install + key: curl-${{ env.CURL_VERSION }}-${{ env.CURL_SHA256 }}-${{ steps.native-deps.outputs.identity }}-${{ hashFiles('.github/workflows/native-live.yml') }}-${{ runner.os }}-${{ runner.arch }} + - name: Build pinned WebSocket-enabled libcurl + env: + CURL_CACHE_HIT: ${{ steps.curl-cache.outputs.cache-hit }} run: | + set -euo pipefail + cmake_file="build-native-deps/curl-install/lib/cmake/CURL/CURLConfig.cmake" + if [[ "${CURL_CACHE_HIT}" == "true" && -f "$cmake_file" ]]; then + echo "Using cached libcurl at ${cmake_file}" + exit 0 + fi mkdir -p build-native-deps cd build-native-deps - curl --fail --location --proto '=https' --tlsv1.2 https://curl.se/download/curl-8.14.1.tar.xz -o curl.tar.xz - echo 'f4619a1e2474c4bbfedc88a7c2191209c8334b48fa1f4e53fd584cc12e9120dd curl.tar.xz' | sha256sum -c - + curl --fail --location --proto '=https' --tlsv1.2 \ + "https://curl.se/download/curl-${CURL_VERSION}.tar.xz" -o curl.tar.xz + echo "${CURL_SHA256} curl.tar.xz" | sha256sum -c - tar -xf curl.tar.xz - cmake -S curl-8.14.1 -B curl-build -G Ninja \ + cmake -S "curl-${CURL_VERSION}" -B curl-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$PWD/curl-install" \ -DBUILD_CURL_EXE=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_STATIC_LIBS=ON \ -DBUILD_TESTING=OFF -DCURL_USE_OPENSSL=ON -DENABLE_WEBSOCKETS=ON \ @@ -38,27 +91,20 @@ jobs: -DCURL_USE_LIBSSH2=OFF -DCURL_BROTLI=OFF -DCURL_ZSTD=OFF cmake --build curl-build -j 4 cmake --install curl-build - - name: Build native runner and tests - run: | - cmake -S . -B build-live -G Ninja -DCMAKE_BUILD_TYPE=Release \ - -DPINEFORGE_BUILD_LIVE_RUNNER=ON -DPINEFORGE_BUILD_TUTORIAL=OFF \ - -DCURL_DIR="$PWD/build-native-deps/curl-install/lib/cmake/CURL" - cmake --build build-live -j 4 - - name: Prepare real R2 settlement ABI provider - run: | - # Preserve shallow CI version metadata: fetch the old object without tags. - git fetch --no-tags --depth=1 origin e60e57156a54bb1c74c0c0a4f5a338fd924c046d - python3 scripts/prepare_settlement_cpp_abi_base.py --source-repo . \ - --current-build build-live --output build-live/settlement-abi-base --jobs 4 - - name: Test native engine, parser, journal and transports - run: | - ctest --test-dir build-live --output-on-failure -j 4 - # This required lane uses the pinned WS-enabled curl. A skip is not - # sufficient here even though ordinary system-curl builds may skip. - build-live/bin/test_native_live_websocket - python3 scripts/check_c_abi_runtime.py - python3 scripts/check_broker_state_hash_coverage.py - - name: Install native executable - run: | - cmake --install build-live --prefix "$PWD/build-live/install" - build-live/install/bin/pineforge-live --help + + - name: Verify native live + run: python3 scripts/ci_verify.py native --build-dir build-live --jobs 4 --generator Ninja --curl-dir "${{ github.workspace }}/build-native-deps/curl-install/lib/cmake/CURL" --ccache --require-websocket + + - name: Stage and summarize diagnostics + if: always() + env: + CURL_CACHE_HIT: ${{ steps.curl-cache.outputs.cache-hit }} + run: python3 scripts/collect_ci_diagnostics.py --build-dir build-live --profile native + + - name: Retain CI diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: ci-diagnostics-native-live + path: ci-diagnostics/ + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 31995814c..becb41290 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,7 @@ docs/production-readiness-findings.md # Cloudflare wrangler local cache (never commit) .wrangler/ + +# Reusable CI cache and retained diagnostics (never source bundles) +.ccache/ +ci-diagnostics/ diff --git a/AGENTS.md b/AGENTS.md index e8e949051..ed2a2137b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,15 +7,9 @@ Both C++ unit tests and full corpus verification must pass. ```bash -# 1. Build and run unit tests (ctest) -cmake -B build -S . \ - -DCMAKE_BUILD_TYPE=Release \ - -DPINEFORGE_BUILD_TESTS=ON \ - -DPINEFORGE_BUILD_CORPUS_STRATEGIES=ON -cmake --build build -j$(nproc 2>/dev/null || echo 4) -python3 scripts/prepare_settlement_cpp_abi_base.py --source-repo . \ - --current-build build --output build/settlement-abi-base -ctest --test-dir build --output-on-failure +# 1. Run the same complete verification profile as CI +python3 scripts/ci_verify.py release --build-dir build --jobs 4 +# Relevant additional profiles: debug, sanitizers, native (see docs/ci.md) # 2. Run full validation corpus sweep (against TV exported trades) ./scripts/run_corpus.sh diff --git a/README.md b/README.md index 1fd0f24b9..dfaf8ab18 100644 --- a/README.md +++ b/README.md @@ -74,16 +74,12 @@ Prefer zero install? The hosted server at **[mcp.pineforge.dev/mcp](https://mcp. ```bash git clone https://github.com/pineforge-4pass/pineforge-engine.git && cd pineforge-engine -cmake -B build -DCMAKE_BUILD_TYPE=Release -cmake --build build -j -python3 scripts/prepare_settlement_cpp_abi_base.py --source-repo . \ - --current-build build --output build/settlement-abi-base -ctest --test-dir build --output-on-failure # all enabled unit, replay and ABI checks +python3 scripts/ci_verify.py release --build-dir build --jobs 4 bash tutorial/run.sh # MACD on BTC/USDT, end to end python3 tutorial/run_stream.py # OHLCV warm-up → realtime trades ``` -The ABI check links callers against both the current library and a separately built, pinned historical library. Preparation uses local Git history and the configured compiler; see the [ABI fixture guide](tests/fixtures/settlement_cpp_abi/README.md) for shallow clones and repeated checks. CTest itself stays offline. +The [shared local/CI verifier](docs/ci.md) includes source guards, tests and installed-package smoke checks. Its ABI check links callers against both the current library and a separately built, pinned historical library. Preparation uses local Git history and the configured compiler; see the [ABI fixture guide](tests/fixtures/settlement_cpp_abi/README.md) for shallow clones and repeated checks. CTest itself stays offline. Prerequisites: CMake ≥ 3.16, a C++17 compiler (GCC ≥ 9, Clang ≥ 10, Apple Clang ≥ 12), Eigen 3.3+ (fetched automatically if absent), Python 3 for the tests (`-DPINEFORGE_BUILD_TESTS=OFF` for a library-only build). `cmake --install build --prefix /usr/local` installs `lib/libpineforge.a`, `include/pineforge/`, and the `find_package(PineForge)` config. diff --git a/cmake/PineForgeVersion.cmake b/cmake/PineForgeVersion.cmake index dcc7aae02..ae61b158a 100644 --- a/cmake/PineForgeVersion.cmake +++ b/cmake/PineForgeVersion.cmake @@ -1,21 +1,27 @@ # PineForgeVersion.cmake # -# Resolve PineForge version from two sources, in priority order: -# 1. `git describe --tags --match 'v*' --abbrev=7 --dirty` (when in a git -# checkout). Strips leading 'v', exposes commits-since-tag + sha + dirty -# as PINEFORGE_VERSION_FULL. -# 2. The `VERSION` file at the repo root (always present in tarballs and -# Docker build contexts). +# PINEFORGE_VERSION_SOURCE (CACHE STRING, AUTO|FILE): +# AUTO (default) — historical behavior exactly: prefer +# `git describe --tags --match 'v*' --abbrev=7 --dirty` for MMP/FULL +# when it matches MAJOR.MINOR.PATCH after stripping a leading 'v'; +# otherwise use the VERSION file. Git SHA/dirty are observed the same way. +# FILE — MMP and FULL always come from VERSION (stable release identity). +# Git SHA/dirty are still observed as separate fields; checkout depth or +# missing tags cannot change MMP/FULL. Invalid values fail configure. # # Sets in the calling scope: # PINEFORGE_VERSION_MAJOR integer # PINEFORGE_VERSION_MINOR integer # PINEFORGE_VERSION_PATCH integer # PINEFORGE_VERSION_MMP "MAJOR.MINOR.PATCH" (for project(VERSION ...)) -# PINEFORGE_VERSION_FULL "MAJOR.MINOR.PATCH[-N-gSHA[-dirty]]" +# PINEFORGE_VERSION_FULL AUTO: describe-or-VERSION; FILE: VERSION # PINEFORGE_VERSION_GIT_SHA short sha or "unknown" # PINEFORGE_VERSION_DIRTY ON/OFF +set(PINEFORGE_VERSION_SOURCE "AUTO" CACHE STRING + "Version identity source: AUTO (git describe preferred) or FILE (VERSION MMP/FULL)") +set_property(CACHE PINEFORGE_VERSION_SOURCE PROPERTY STRINGS AUTO FILE) + function(_pineforge_read_version_file _out_mmp) set(_vfile "${CMAKE_CURRENT_SOURCE_DIR}/VERSION") if(NOT EXISTS "${_vfile}") @@ -30,6 +36,12 @@ function(_pineforge_read_version_file _out_mmp) endfunction() function(pineforge_resolve_version) + if(NOT PINEFORGE_VERSION_SOURCE STREQUAL "AUTO" AND + NOT PINEFORGE_VERSION_SOURCE STREQUAL "FILE") + message(FATAL_ERROR + "PINEFORGE_VERSION_SOURCE must be AUTO or FILE (got '${PINEFORGE_VERSION_SOURCE}')") + endif() + _pineforge_read_version_file(_file_mmp) set(_git_full "") @@ -67,9 +79,18 @@ function(pineforge_resolve_version) endif() endif() - set(_mmp "${_file_mmp}") - if(_git_full MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)") - set(_mmp "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") + if(PINEFORGE_VERSION_SOURCE STREQUAL "FILE") + set(_mmp "${_file_mmp}") + set(_full "${_file_mmp}") + else() + set(_mmp "${_file_mmp}") + if(_git_full MATCHES "^([0-9]+)\\.([0-9]+)\\.([0-9]+)") + set(_mmp "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") + endif() + if(NOT _git_full) + set(_git_full "${_mmp}") + endif() + set(_full "${_git_full}") endif() string(REPLACE "." ";" _parts "${_mmp}") @@ -77,15 +98,11 @@ function(pineforge_resolve_version) list(GET _parts 1 _min) list(GET _parts 2 _pat) - if(NOT _git_full) - set(_git_full "${_mmp}") - endif() - set(PINEFORGE_VERSION_MAJOR "${_maj}" PARENT_SCOPE) set(PINEFORGE_VERSION_MINOR "${_min}" PARENT_SCOPE) set(PINEFORGE_VERSION_PATCH "${_pat}" PARENT_SCOPE) set(PINEFORGE_VERSION_MMP "${_mmp}" PARENT_SCOPE) - set(PINEFORGE_VERSION_FULL "${_git_full}" PARENT_SCOPE) + set(PINEFORGE_VERSION_FULL "${_full}" PARENT_SCOPE) set(PINEFORGE_VERSION_GIT_SHA "${_git_sha}" PARENT_SCOPE) set(PINEFORGE_VERSION_DIRTY "${_dirty}" PARENT_SCOPE) endfunction() diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 000000000..7d6566b2c --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,77 @@ +# Local verification and CI + +Use the same entrypoint as GitHub Actions before publishing a change: + +```sh +python3 scripts/ci_verify.py release --build-dir build --jobs 4 +``` + +The command runs source guards, configures and rebuilds every enabled target, +prepares the real historical ABI provider, runs CTest, installs the package, +and builds and runs the `find_package` consumer. The smoke test compares the +installed library's reported version with `VERSION`. After a successful build, +a failing test suite does not hide a separate install or package failure. + +## Profiles + +| Profile | Build | Additional coverage | +| --- | --- | --- | +| `release` | Release, tutorial enabled | Standard CI checks and installed package | +| `debug` | Debug, tutorial enabled | The same checks without Release optimization | +| `sanitizers` | Debug, ASan and UBSan | Instrumented library, tests and installed consumer; Linux CI also requires leak detection | +| `native` | Release, live runner enabled | Parser, journal, transport tests and installed runner help | + +By default each profile uses `build-ci-`. Keep separate build directories +for different profiles and toolchains. The verifier never deletes a build tree +or replaces an incompatible historical ABI receipt. Use a fresh directory when +its configuration no longer matches. + +```sh +python3 scripts/ci_verify.py debug --jobs 4 +python3 scripts/ci_verify.py sanitizers --jobs 4 +python3 scripts/ci_verify.py native --curl-dir /path/to/curl/lib/cmake/CURL \ + --require-websocket --jobs 4 +``` + +The sanitizer profile uses the Linux CI ASan/UBSan environment, including +LeakSanitizer, which needs a supporting runtime. The native CI lane builds +checksum-pinned curl with WebSocket support and passes `--require-websocket`. +That flag turns a transport skip into failure. A local native run using system +curl may report the existing unsupported-WebSocket skip; that is not the required +Linux transport proof. + +Pass `--ccache` when ccache is installed. It caches compiler work, not complete +build directories or verification receipts. Compiler, source, header and flag +checks remain in effect. The native curl cache is separately bound to its source +checksum, workflow configuration, compiler/package environment and install path. + +## Version identity and historical ABI checks + +Verification explicitly uses `PINEFORGE_VERSION_SOURCE=FILE`: package MMP/FULL +identity comes from `VERSION`, while Git revision/dirty observations remain +separate. Git tags and checkout depth cannot change the smoke-test expectation. +Ordinary CMake builds retain `AUTO`, the existing git-describe-first behavior. +No release tag or VERSION value is rewritten by verification. + +The verifier fetches the pinned historical ABI commit without tags only when +that object is missing. It then builds the old archive once or validates and +reuses a matching prepared receipt. CTest itself performs no network fetch. +The [ABI guide](../tests/fixtures/settlement_cpp_abi/README.md) describes the +actual old/new library pairs and their immutable inputs. + +## Failure evidence + +Each build directory contains `ci-summary.json` and full command logs under +`ci-logs/`, plus CTest JUnit when the installed CTest supports it. A version +failure records both actual and expected values. GitHub Actions retains compact +diagnostics, CTest logs and ABI receipts and publishes the stage results in its +job summary. Compiler objects, binaries and dependency caches are not uploaded +as diagnostics. + +Superseded pull-request runs are canceled. Main/post-merge and manual proof runs +use distinct concurrency groups and remain uncanceled. The required `build` +check still depends on every standard matrix lane; sanitizer and native-live +checks remain separate requirements. + +These checks do not run the parity campaign. Fixed-population Cloud measurement, +the actual gate and post-merge evidence remain separate acceptance steps. diff --git a/scripts/ci_verify.py b/scripts/ci_verify.py new file mode 100644 index 000000000..d7f89d9e5 --- /dev/null +++ b/scripts/ci_verify.py @@ -0,0 +1,710 @@ +#!/usr/bin/env python3 +"""Shared local/CI verification driver. Stdlib only. Not a command generator. + +Profiles: release, debug, sanitizers, native. Default build dir build-ci-PROFILE. +Source guards, explicit configure, full rebuild, pinned R2 ABI prepare/reuse, +CTest, install+find_package+VERSION smoke, native help / required WebSocket. +Fail fast on configure/build. After a successful build collect independent +CTest and package failures in the same run. Never deletes source, tests, or +build trees; a mismatched ABI base is a refusal to a fresh --build-dir. +""" +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import sys +import time +from typing import Callable + +from prepare_settlement_cpp_abi_base import ( + BASE_COMMIT, + read_cache, + reusable_prepared_base, +) + +ROOT = Path(__file__).resolve().parents[1] +PROFILES = ('release', 'debug', 'sanitizers', 'native') +DEFAULT_JOBS = 4 +JOBS_MIN, JOBS_MAX = 1, 64 +SCHEMA = 'pineforge-ci-verify/v1' +SANITIZER_FLAG = '-fsanitize=address,undefined' +SANITIZER_RUN_ENV = { + 'ASAN_OPTIONS': 'detect_leaks=1:halt_on_error=1:abort_on_error=1', + 'UBSAN_OPTIONS': 'print_stacktrace=1:halt_on_error=1', +} +SOURCE_GUARD_SCRIPTS = ( + ('source-guard-c-abi', ['scripts/check_c_abi_runtime.py']), + ('source-guard-broker-hash', ['scripts/check_broker_state_hash_coverage.py']), + ('source-guard-pending-mirror', ['scripts/gen_pending_order_mirror.py', '--check']), + ('source-guard-native-versions', ['scripts/check_native_cpp_versions.py']), + ('source-guard-aggregate-versions', ['scripts/check_aggregate_cpp_versions.py']), +) + + +class ConfigError(Exception): + """CLI / profile validation; exit status 2.""" + + +@dataclass(frozen=True) +class Profile: + name: str + build_type: str + sanitizers: bool + live_runner: bool + tutorial: bool + + +PROFILE = { + 'release': Profile('release', 'Release', False, False, True), + 'debug': Profile('debug', 'Debug', False, False, True), + 'sanitizers': Profile('sanitizers', 'Debug', True, False, True), + 'native': Profile('native', 'Release', False, True, False), +} + + +@dataclass +class Completed: + returncode: int + stdout: bytes = b'' + stderr: bytes = b'' + + +Runner = Callable[..., Completed] + + +@dataclass +class VerifyConfig: + profile: Profile + source: Path + build_dir: Path + jobs: int + generator: str | None + curl_dir: Path | None + ccache_path: str | None + require_websocket: bool + runner: Runner + stream_output: bool = True + + +class Parser(argparse.ArgumentParser): + def error(self, message: str) -> None: + raise ConfigError(message) + + +def cmake_on(value: str | None) -> bool: + return (value or '').strip().upper() in {'1', 'ON', 'TRUE', 'YES', 'Y'} + + +def expected_version(source: Path) -> str: + return ''.join((source / 'VERSION').read_text().split()) + + +def default_build_dir(source: Path, profile: str) -> Path: + return source / f'build-ci-{profile}' + + +def call_runner(runner: Runner, argv: list[str], *, extra_env: dict[str, str] | None = None, + timeout: int = 600, combine_stderr: bool = True, + stream_output: bool = False) -> Completed: + return runner(argv, extra_env=extra_env, timeout=timeout, + combine_stderr=combine_stderr, stream_output=stream_output) + + +def ctest_supports_junit(runner: Runner) -> bool: + try: + result = call_runner(runner, ['ctest', '--help'], timeout=30, stream_output=False) + except (OSError, subprocess.TimeoutExpired): + return False + return '--output-junit' in (result.stdout + result.stderr).decode('utf-8', 'replace') + + +def source_guard_commands(source: Path) -> list[tuple[str, list[str]]]: + python = sys.executable + return [(name, [python, str(source / rel[0]), *rel[1:]]) for name, rel in SOURCE_GUARD_SCRIPTS] + + +def cmake_cache_definitions(cfg: VerifyConfig) -> dict[str, str]: + profile = cfg.profile + values = { + 'CMAKE_BUILD_TYPE': profile.build_type, + 'CMAKE_EXPORT_COMPILE_COMMANDS': 'ON', + 'PINEFORGE_BUILD_TESTS': 'ON', + 'PINEFORGE_BUILD_TUTORIAL': 'ON' if profile.tutorial else 'OFF', + 'PINEFORGE_BUILD_LIVE_RUNNER': 'ON' if profile.live_runner else 'OFF', + 'PINEFORGE_ENABLE_SANITIZERS': 'ON' if profile.sanitizers else 'OFF', + 'PINEFORGE_BUILD_CORPUS_STRATEGIES': 'OFF', + 'PINEFORGE_BUILD_BENCH_STRATEGIES': 'OFF', + 'PINEFORGE_BUILD_SPEED_BENCH': 'OFF', + 'PINEFORGE_BUILD_EXAMPLES': 'OFF', + 'PINEFORGE_ENABLE_COVERAGE': 'OFF', + 'PINEFORGE_STRICT_WARNINGS': 'OFF', + 'PINEFORGE_VERSION_SOURCE': 'FILE', + } + if cfg.curl_dir is not None: + values['CURL_DIR'] = str(cfg.curl_dir) + if cfg.ccache_path: + values['CMAKE_C_COMPILER_LAUNCHER'] = cfg.ccache_path + values['CMAKE_CXX_COMPILER_LAUNCHER'] = cfg.ccache_path + return values + + +def cmake_configure_argv(cfg: VerifyConfig) -> list[str]: + argv = ['cmake', '-S', str(cfg.source), '-B', str(cfg.build_dir)] + if cfg.generator: + argv += ['-G', cfg.generator] + argv += [f'-D{key}={value}' for key, value in cmake_cache_definitions(cfg).items()] + return argv + + +def parse_args(argv: list[str] | None, *, source: Path = ROOT) -> argparse.Namespace: + parser = Parser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + 'profile', choices=PROFILES, + help='release/debug keep tutorial ON and native OFF; ' + 'sanitizers enable PUBLIC ASan/UBSan; native enables the live runner') + parser.add_argument('--build-dir', type=Path, default=None, + help='default: /build-ci-PROFILE') + parser.add_argument('--jobs', type=int, default=DEFAULT_JOBS) + parser.add_argument('--generator', default=None) + parser.add_argument('--curl-dir', type=Path, default=None, + help='CURL CMake package dir (native WS-enabled curl install)') + parser.add_argument('--ccache', action='store_true', + help='require installed ccache and bind CMAKE_*_COMPILER_LAUNCHER') + parser.add_argument('--require-websocket', action='store_true', + help='native only: execute test_native_live_websocket and refuse skip (77)') + args = parser.parse_args(argv) + if args.build_dir is None: + args.build_dir = default_build_dir(source, args.profile) + return args + + +def validate_config(args: argparse.Namespace, *, source: Path = ROOT, + which: Callable[[str], str | None] = shutil.which) -> VerifyConfig: + if not JOBS_MIN <= args.jobs <= JOBS_MAX: + raise ConfigError(f'--jobs must be {JOBS_MIN}..{JOBS_MAX}') + if args.require_websocket and args.profile != 'native': + raise ConfigError('--require-websocket is only valid with the native profile') + if args.curl_dir is not None and not args.curl_dir.is_dir(): + raise ConfigError(f'--curl-dir is not a directory: {args.curl_dir}') + ccache_path = None + if args.ccache: + found = which('ccache') + if not found: + raise ConfigError('optional --ccache requires an installed ccache tool') + ccache_path = str(Path(found).resolve()) + for tool in ('cmake', 'ctest', 'git'): + if not which(tool): + raise ConfigError(f'required tool not found: {tool}') + build_dir = args.build_dir.expanduser() + build_dir = build_dir.resolve() if build_dir.is_absolute() else (Path.cwd() / build_dir).resolve() + source = source.resolve() + if build_dir == source: + raise ConfigError('--build-dir cannot be the source root') + return VerifyConfig( + profile=PROFILE[args.profile], + source=source, + build_dir=build_dir, + jobs=args.jobs, + generator=args.generator, + curl_dir=args.curl_dir.resolve() if args.curl_dir is not None else None, + ccache_path=ccache_path, + require_websocket=bool(args.require_websocket), + runner=default_runner, + ) + + +def default_runner(argv: list[str], *, extra_env: dict[str, str] | None = None, + timeout: int = 600, combine_stderr: bool = True, + stream_output: bool = True) -> Completed: + env = os.environ.copy() + if extra_env: + env.update(extra_env) + stderr = subprocess.STDOUT if combine_stderr else subprocess.PIPE + try: + result = subprocess.run( + argv, stdout=subprocess.PIPE, stderr=stderr, env=env, timeout=timeout) + except subprocess.TimeoutExpired as error: + out = error.stdout or b'' + err = b'' if combine_stderr else (error.stderr or b'') + if stream_output and out: + sys.stdout.buffer.write(out) + sys.stdout.buffer.flush() + return Completed(124, out, err) + except OSError as error: + return Completed(127, b'', str(error).encode()) + if stream_output and result.stdout: + sys.stdout.buffer.write(result.stdout) + sys.stdout.buffer.flush() + return Completed(result.returncode, result.stdout or b'', + b'' if combine_stderr else (result.stderr or b'')) + + +def compile_command_text(entry: dict) -> str: + if 'command' in entry: + return entry['command'] + return ' '.join(entry.get('arguments') or []) + + +def sanitizer_flag_on_library(build_dir: Path) -> bool: + path = build_dir / 'compile_commands.json' + if not path.is_file(): + raise RuntimeError('compile_commands.json missing; sanitizer PUBLIC flag cannot be verified') + entries = json.loads(path.read_text()) + library = [] + for entry in entries: + file = (entry.get('file') or '').replace('\\', '/') + if not file.endswith('/src/matrix.cpp') and Path(file).name != 'matrix.cpp': + continue + if '/src/' in file or Path(file).parent.name == 'src': + library.append(compile_command_text(entry)) + if not library: + raise RuntimeError('compile_commands.json has no src/matrix.cpp entry') + return all(SANITIZER_FLAG in command for command in library) + + +def stale_archive_sources(source: Path, archive: Path) -> list[str]: + if not archive.is_file(): + raise RuntimeError(f'full build did not produce {archive}') + newest = archive.stat().st_mtime + stale = [] + for root in (source / 'CMakeLists.txt', source / 'src', source / 'include'): + paths = [root] if root.is_file() else [path for path in root.rglob('*') if path.is_file()] + for path in paths: + if path.stat().st_mtime > newest: + stale.append(str(path.relative_to(source))) + return stale + + +def smoke_prefix(cache: dict[str, str], install_prefix: Path) -> str: + parts = [str(install_prefix)] + extra = cache.get('CMAKE_PREFIX_PATH') + if extra: + for item in extra.split(';'): + if item and item not in parts: + parts.append(item) + eigen = cache.get('Eigen3_DIR') + if eigen and eigen not in parts: + parts.append(eigen) + return ';'.join(parts) + + +def pinned_object_present(source: Path, runner: Runner) -> bool: + result = call_runner( + runner, ['git', '-C', str(source), 'cat-file', '-e', BASE_COMMIT + '^{commit}'], + timeout=30, stream_output=False) + return result.returncode == 0 + + +def tool_version_commands(cfg: VerifyConfig) -> list[list[str]]: + commands = [ + ['cmake', '--version'], + ['ctest', '--version'], + ['git', '--version'], + [sys.executable, '--version'], + ] + compiler = shutil.which('c++') or shutil.which('clang++') or shutil.which('g++') + if compiler: + commands.append([compiler, '--version']) + if cfg.ccache_path: + commands.append([cfg.ccache_path, '--version']) + return commands + + +class Driver: + def __init__(self, cfg: VerifyConfig): + self.cfg = cfg + self.logs = cfg.build_dir / 'ci-logs' + self.install_prefix = cfg.build_dir / 'ci-install' + self.smoke_dir = cfg.build_dir / 'ci-smoke' + self.summary_path = cfg.build_dir / 'ci-summary.json' + self.failures: list[dict] = [] + self.stages: list[dict] = [] + self.expected_version = expected_version(cfg.source) + self.actual_version: str | None = None + self.abi_action = 'not-started' + self.summary: dict = { + 'schemaVersion': SCHEMA, + 'status': 'incomplete', + 'profile': cfg.profile.name, + 'source': str(cfg.source), + 'buildDir': str(cfg.build_dir), + 'installPrefix': str(self.install_prefix), + 'smokeDir': str(self.smoke_dir), + 'jobs': cfg.jobs, + 'generator': cfg.generator, + 'ccache': cfg.ccache_path, + 'requireWebsocket': cfg.require_websocket, + 'curlDir': str(cfg.curl_dir) if cfg.curl_dir else None, + 'versionSource': 'FILE', + 'cmakeDefinitions': cmake_cache_definitions(cfg), + 'expectedVersion': self.expected_version, + 'actualVersion': None, + 'exitCode': None, + 'abi': {'action': self.abi_action}, + 'stages': self.stages, + 'failures': self.failures, + } + + def write_summary(self) -> None: + self.summary['abi'] = {'action': self.abi_action} + self.summary['actualVersion'] = self.actual_version + self.summary['stages'] = self.stages + self.summary['failures'] = self.failures + self.summary_path.parent.mkdir(parents=True, exist_ok=True) + self.summary_path.write_text(json.dumps(self.summary, indent=2, sort_keys=True) + '\n') + + def invoke(self, name: str, argv: list[str], *, extra_env: dict[str, str] | None = None, + timeout: int = 600, combine_stderr: bool = True) -> Completed: + if self.cfg.ccache_path: + extra_env = {**(extra_env or {}), 'CCACHE_COMPILERCHECK': 'content'} + log = self.logs / f'{name}.log' + log.parent.mkdir(parents=True, exist_ok=True) + started = time.time() + header = '+ ' + shlex.join(map(str, argv)) + '\n' + stage = { + 'name': name, 'argv': list(map(str, argv)), 'exitCode': None, + 'status': 'running', 'log': str(log.relative_to(self.cfg.build_dir)), + 'extraEnvKeys': sorted(extra_env) if extra_env else [], + } + self.stages.append(stage) + log.write_text(header) + self.write_summary() + if self.cfg.stream_output: + print(f'ci_verify: {name}: {shlex.join(map(str, argv))}', flush=True) + try: + result = call_runner( + self.cfg.runner, list(map(str, argv)), extra_env=extra_env, timeout=timeout, + combine_stderr=combine_stderr, stream_output=self.cfg.stream_output) + except Exception as error: + result = Completed(1, b'', str(error).encode()) + body = result.stdout + (b'' if not result.stderr else b'\n' + result.stderr) + log.write_bytes(header.encode() + body) + stage.update({ + 'exitCode': result.returncode, + 'status': 'passed' if result.returncode == 0 else 'failed', + 'durationSeconds': round(time.time() - started, 3), + }) + if result.returncode != 0: + self.failures.append({ + 'stage': name, + 'exitCode': result.returncode, + 'log': stage['log'], + }) + self.write_summary() + return result + + def fail_stage(self, name: str, message: str, *, argv: list[str] | None = None) -> None: + log = self.logs / f'{name}.log' + log.parent.mkdir(parents=True, exist_ok=True) + log.write_text(message + '\n') + self.stages.append({ + 'name': name, + 'argv': argv or [], + 'exitCode': 1, + 'status': 'failed', + 'log': str(log.relative_to(self.cfg.build_dir)), + 'error': message, + 'extraEnvKeys': [], + }) + self.failures.append({ + 'stage': name, 'exitCode': 1, 'error': message, + 'log': str(log.relative_to(self.cfg.build_dir)), + }) + self.write_summary() + + def pass_stage(self, name: str, message: str, *, argv: list[str] | None = None) -> None: + log = self.logs / f'{name}.log' + log.parent.mkdir(parents=True, exist_ok=True) + log.write_text(message + '\n') + self.stages.append({ + 'name': name, + 'argv': argv or [], + 'exitCode': 0, + 'status': 'passed', + 'log': str(log.relative_to(self.cfg.build_dir)), + 'extraEnvKeys': [], + }) + self.write_summary() + + def sanitizer_env(self) -> dict[str, str] | None: + return dict(SANITIZER_RUN_ENV) if self.cfg.profile.sanitizers else None + + def collect_tool_versions(self) -> bool: + chunks = [] + failed = False + for argv in tool_version_commands(self.cfg): + result = call_runner( + self.cfg.runner, argv, timeout=30, stream_output=False) + chunks.append('+ ' + shlex.join(argv)) + chunks.append((result.stdout + result.stderr).decode('utf-8', 'replace')) + if result.returncode != 0 and Path(argv[0]).name in {'cmake', 'ctest', 'git'}: + failed = True + self.logs.mkdir(parents=True, exist_ok=True) + (self.logs / 'tool-versions.log').write_text('\n'.join(chunks) + '\n') + self.stages.append({ + 'name': 'tool-versions', + 'argv': ['tool-versions'], + 'exitCode': 1 if failed else 0, + 'status': 'failed' if failed else 'passed', + 'log': 'ci-logs/tool-versions.log', + 'extraEnvKeys': [], + }) + if failed: + self.failures.append({'stage': 'tool-versions', 'exitCode': 1}) + self.write_summary() + return not failed + + def verify_configured_profile(self, cache: dict[str, str]) -> str | None: + profile = self.cfg.profile + if cache.get('CMAKE_BUILD_TYPE') != profile.build_type: + return (f'CMAKE_BUILD_TYPE expected {profile.build_type!r} ' + f'got {cache.get("CMAKE_BUILD_TYPE")!r}') + if cache.get('PINEFORGE_VERSION_SOURCE') != 'FILE': + return (f'PINEFORGE_VERSION_SOURCE expected FILE ' + f'got {cache.get("PINEFORGE_VERSION_SOURCE")!r}') + for key, wanted in ( + ('PINEFORGE_BUILD_TESTS', True), + ('PINEFORGE_BUILD_TUTORIAL', profile.tutorial), + ('PINEFORGE_BUILD_LIVE_RUNNER', profile.live_runner), + ('PINEFORGE_ENABLE_SANITIZERS', profile.sanitizers), + ): + if cmake_on(cache.get(key)) != wanted: + return f'{key} expected {"ON" if wanted else "OFF"} got {cache.get(key)!r}' + if self.cfg.ccache_path: + for launcher in ('CMAKE_C_COMPILER_LAUNCHER', 'CMAKE_CXX_COMPILER_LAUNCHER'): + if cache.get(launcher) != self.cfg.ccache_path: + return f'{launcher} expected {self.cfg.ccache_path!r} got {cache.get(launcher)!r}' + return None + + def ensure_abi_base(self) -> None: + output = self.cfg.build_dir / 'settlement-abi-base' + prepare = [ + sys.executable, str(self.cfg.source / 'scripts/prepare_settlement_cpp_abi_base.py'), + '--source-repo', str(self.cfg.source), + '--current-build', str(self.cfg.build_dir), + '--output', str(output), + '--jobs', str(self.cfg.jobs), + ] + if output.exists(): + try: + receipt = reusable_prepared_base(output, self.cfg.build_dir) + except Exception as error: + self.abi_action = 'refused' + self.fail_stage('abi-base', str(error), argv=['reuse-matching-receipt', str(output)]) + return + self.abi_action = 'reused' + self.pass_stage( + 'abi-base', + json.dumps({'action': 'reused', 'receipt': str(output / 'receipt.json'), + 'archiveSha256': receipt.get('archiveSha256')}, indent=2, sort_keys=True), + argv=['reuse-matching-receipt', str(output / 'receipt.json')]) + return + if not pinned_object_present(self.cfg.source, self.cfg.runner): + fetch = ['git', '-C', str(self.cfg.source), 'fetch', '--no-tags', '--depth=1', + 'origin', BASE_COMMIT] + fetched = self.invoke('abi-fetch', fetch, timeout=120) + if fetched.returncode != 0: + self.abi_action = 'failed' + return + prepared = self.invoke('abi-base', prepare, timeout=1800) + self.abi_action = 'prepared' if prepared.returncode == 0 else 'failed' + + def run_smoke(self, cache: dict[str, str]) -> None: + prefix = smoke_prefix(cache, self.install_prefix) + configure = [ + 'cmake', '-S', str(self.cfg.source / 'cmake/smoke_consumer'), + '-B', str(self.smoke_dir), + f'-DCMAKE_BUILD_TYPE={self.cfg.profile.build_type}', + f'-DCMAKE_PREFIX_PATH={prefix}', + ] + if cache.get('Eigen3_DIR'): + configure.append(f'-DEigen3_DIR={cache["Eigen3_DIR"]}') + if self.invoke('smoke-configure', configure, timeout=120).returncode != 0: + return + if self.invoke( + 'smoke-build', + ['cmake', '--build', str(self.smoke_dir), '--parallel', str(self.cfg.jobs)], + timeout=300).returncode != 0: + return + binary = self.smoke_dir / 'smoke_version' + ran = self.invoke( + 'smoke-version', [str(binary)], extra_env=self.sanitizer_env(), + timeout=60, combine_stderr=False) + actual = ran.stdout.decode('utf-8', 'replace').strip() + self.actual_version = actual + self.summary['actualVersion'] = actual + if ran.returncode != 0: + self.failures[-1]['error'] = ( + f'version smoke exited {ran.returncode}; expected {self.expected_version!r} ' + f'got {actual!r}') + self.write_summary() + return + if actual != self.expected_version: + self.fail_stage( + 'smoke-version-check', + f'version smoke expected {self.expected_version!r} got {actual!r}', + argv=[str(binary)]) + return + self.pass_stage( + 'smoke-version-check', + f'printed VERSION {actual} matches {self.cfg.source / "VERSION"}', + argv=[str(binary)]) + + def run(self) -> int: + self.logs.mkdir(parents=True, exist_ok=True) + self.write_summary() + if not self.collect_tool_versions(): + return self.finish('failed', 1) + guard_failed = False + for name, argv in source_guard_commands(self.cfg.source): + if self.invoke(name, argv, timeout=120).returncode != 0: + guard_failed = True + if guard_failed: + return self.finish('failed', 1) + if self.invoke('configure', cmake_configure_argv(self.cfg), timeout=180).returncode != 0: + return self.finish('failed', 1) + cache_path = self.cfg.build_dir / 'CMakeCache.txt' + if not cache_path.is_file(): + self.fail_stage('configure-cache', f'missing {cache_path}') + return self.finish('failed', 1) + cache = read_cache(cache_path) + profile_error = self.verify_configured_profile(cache) + if profile_error: + self.fail_stage('profile-options', profile_error) + return self.finish('failed', 1) + self.pass_stage('profile-options', json.dumps({ + 'tutorial': cmake_on(cache.get('PINEFORGE_BUILD_TUTORIAL')), + 'liveRunner': cmake_on(cache.get('PINEFORGE_BUILD_LIVE_RUNNER')), + 'sanitizers': cmake_on(cache.get('PINEFORGE_ENABLE_SANITIZERS')), + 'versionSource': cache.get('PINEFORGE_VERSION_SOURCE'), + 'buildType': cache.get('CMAKE_BUILD_TYPE'), + }, indent=2, sort_keys=True)) + if self.cfg.profile.sanitizers: + try: + if not sanitizer_flag_on_library(self.cfg.build_dir): + self.fail_stage( + 'sanitizer-public-flag', + f'src/matrix.cpp compile command missing PUBLIC {SANITIZER_FLAG}') + return self.finish('failed', 1) + except Exception as error: + self.fail_stage('sanitizer-public-flag', str(error)) + return self.finish('failed', 1) + self.pass_stage('sanitizer-public-flag', f'library compile uses {SANITIZER_FLAG}') + if self.invoke( + 'build', + ['cmake', '--build', str(self.cfg.build_dir), '--parallel', str(self.cfg.jobs)], + timeout=1800).returncode != 0: + return self.finish('failed', 1) + archive = self.cfg.build_dir / 'lib' / 'libpineforge.a' + try: + stale = stale_archive_sources(self.cfg.source, archive) + except Exception as error: + self.fail_stage('stale-binaries', str(error)) + return self.finish('failed', 1) + if stale: + self.fail_stage( + 'stale-binaries', + 'libpineforge.a predates source; full rebuild required: ' + ', '.join(stale[:40])) + return self.finish('failed', 1) + self.pass_stage('stale-binaries', f'{archive} is newer than src/, include/, CMakeLists.txt') + live = self.cfg.build_dir / 'bin' / 'pineforge-live' + if self.cfg.profile.live_runner: + if not live.is_file(): + self.fail_stage('native-binary', f'native profile missing {live}') + return self.finish('failed', 1) + self.pass_stage('native-binary', str(live)) + elif live.is_file(): + self.fail_stage( + 'native-binary', + 'non-native profile produced pineforge-live; live runner must stay OFF') + return self.finish('failed', 1) + else: + self.pass_stage('native-binary', 'live runner absent as required for this profile') + + self.ensure_abi_base() + + ctest = ['ctest', '--test-dir', str(self.cfg.build_dir), + '--output-on-failure', '--parallel', str(self.cfg.jobs)] + if ctest_supports_junit(self.cfg.runner): + ctest += ['--output-junit', str(self.cfg.build_dir / 'ctest-junit.xml')] + self.invoke('ctest', ctest, extra_env=self.sanitizer_env(), timeout=1800) + + installed = self.invoke( + 'install', + ['cmake', '--install', str(self.cfg.build_dir), '--prefix', str(self.install_prefix)], + timeout=180) + if installed.returncode == 0: + self.run_smoke(cache) + if self.cfg.profile.live_runner: + help_bin = self.install_prefix / 'bin' / 'pineforge-live' + if not help_bin.is_file(): + self.fail_stage('native-help', f'installed native executable missing: {help_bin}') + else: + self.invoke('native-help', [str(help_bin), '--help'], timeout=30) + if self.cfg.require_websocket: + ws = self.cfg.build_dir / 'bin' / 'test_native_live_websocket' + if not ws.is_file(): + self.fail_stage( + 'require-websocket', + f'--require-websocket cannot accept skip: missing {ws}') + else: + ran = self.invoke( + 'require-websocket', [str(ws)], extra_env=self.sanitizer_env(), timeout=120) + if ran.returncode == 77: + self.fail_stage( + 'require-websocket-skip', + '--require-websocket cannot accept skip (exit 77)', + argv=[str(ws)]) + status = 'passed' if not self.failures else 'failed' + return self.finish(status, 0 if status == 'passed' else 1) + + def finish(self, status: str, code: int) -> int: + self.summary['status'] = status + self.summary['exitCode'] = code + self.write_summary() + return code + + +def build_config(argv: list[str] | None = None, *, source: Path = ROOT, + runner: Runner | None = None, stream_output: bool | None = None, + which: Callable[[str], str | None] = shutil.which) -> VerifyConfig: + args = parse_args(argv, source=source) + cfg = validate_config(args, source=source, which=which) + if runner is not None: + cfg.runner = runner + if stream_output is not None: + cfg.stream_output = stream_output + return cfg + + +def main(argv: list[str] | None = None, *, runner: Runner | None = None, + stream_output: bool | None = None, + which: Callable[[str], str | None] = shutil.which) -> int: + try: + cfg = build_config(argv, runner=runner, stream_output=stream_output, which=which) + except ConfigError as error: + print(f'ci_verify: {error}', file=sys.stderr) + return 2 + driver = Driver(cfg) + try: + return driver.run() + except ConfigError as error: + print(f'ci_verify: {error}', file=sys.stderr) + driver.fail_stage('config', str(error)) + return driver.finish('failed', 2) + except Exception as error: + driver.fail_stage('driver', str(error)) + return driver.finish('failed', 1) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/collect_ci_diagnostics.py b/scripts/collect_ci_diagnostics.py new file mode 100644 index 000000000..4670d5343 --- /dev/null +++ b/scripts/collect_ci_diagnostics.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Retain compact CI diagnostics without copying binaries or dependency caches.""" +from __future__ import annotations +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess + +ARTIFACTS = { + "ci-summary.json": "ci-summary.json", + "ci-logs": "ci-logs", + "ctest-junit.xml": "ctest-junit.xml", + "Testing/Temporary/LastTest.log": "LastTest.log", + "settlement-abi-base/receipt.json": "settlement-abi-base-receipt.json", + "settlement-abi-base/configure.log": "settlement-abi-base-configure.log", + "settlement-abi-base/build.log": "settlement-abi-base-build.log", + "settlement-abi-receipt.json": "settlement-abi-receipt.json", +} +ENV_KEYS = ("GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_SHA", "GITHUB_RUN_ID", + "GITHUB_RUN_ATTEMPT", "GITHUB_JOB", "RUNNER_OS", "RUNNER_ARCH", "CURL_CACHE_HIT") + + +def copy_regular(source: Path, target: Path) -> None: + if source.is_symlink(): + return + if source.is_file(): + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + elif source.is_dir(): + for item in source.iterdir(): + copy_regular(item, target / item.name) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--build-dir", type=Path, required=True) + parser.add_argument("--profile", choices=("release", "debug", "sanitizers", "native"), required=True) + parser.add_argument("--output", type=Path, default=Path("ci-diagnostics")) + args = parser.parse_args() + build = args.build_dir.resolve() + output = args.output.resolve() + if output == build or build in output.parents: + parser.error("--output must be outside --build-dir to avoid copying diagnostics into themselves") + args.output.mkdir(parents=True, exist_ok=True) + info = {key: os.environ.get(key, "") for key in ENV_KEYS} + info.update(buildDir=str(args.build_dir), profile=args.profile) + (args.output / "job-info.json").write_text(json.dumps(info, indent=2) + "\n") + missing = [] + for source, target in ARTIFACTS.items(): + path = args.build_dir / source + if path.exists() and not path.is_symlink(): + copy_regular(path, args.output / target) + else: + missing.append(source) + for directory in args.build_dir.glob("settlement-abi-receipt.artifacts-*"): + if directory.is_symlink() or not directory.is_dir(): + continue + for item in directory.rglob("*"): + if item.suffix in (".log", ".json", ".cpp", ".hpp"): + copy_regular(item, args.output / directory.name / item.relative_to(directory)) + (args.output / "missing.json").write_text(json.dumps(missing, indent=2) + "\n") + if shutil.which("ccache"): + result = subprocess.run(["ccache", "--show-stats"], capture_output=True, text=True) + (args.output / "ccache.log").write_text(result.stdout + result.stderr) + summary = {} + try: + summary = json.loads((args.build_dir / "ci-summary.json").read_text()) + except (OSError, ValueError): + pass + print(json.dumps({"profile": args.profile, "status": summary.get("status", "unavailable"), + "failures": summary.get("failures", []), "diagnostics": str(args.output)}, indent=2)) + destination = os.environ.get("GITHUB_STEP_SUMMARY") + if destination: + status = summary.get("status", "unavailable") + stages = summary.get("stages", []) + lines = [f"### Verification: {args.profile}", "", f"Status: **{status}**", "", + "| Stage | Result |", "| --- | --- |"] + lines += [f"| {stage.get('name', '?')} | {stage.get('status', '?')} |" for stage in stages] + lines += ["", "Full logs and ABI/CTest receipts are attached as CI diagnostics.", ""] + with Path(destination).open("a") as output: + output.write("\n".join(lines)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/prepare_settlement_cpp_abi_base.py b/scripts/prepare_settlement_cpp_abi_base.py index 9a2d40855..2ce3c5ab3 100644 --- a/scripts/prepare_settlement_cpp_abi_base.py +++ b/scripts/prepare_settlement_cpp_abi_base.py @@ -25,6 +25,7 @@ 'CMAKE_BUILD_TYPE', 'CMAKE_C_COMPILER', 'CMAKE_CXX_COMPILER', 'CMAKE_C_COMPILER_TARGET', 'CMAKE_CXX_COMPILER_TARGET', 'CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN', 'CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN', + 'CMAKE_C_COMPILER_LAUNCHER', 'CMAKE_CXX_COMPILER_LAUNCHER', 'CMAKE_C_FLAGS', 'CMAKE_CXX_FLAGS', 'CMAKE_C_FLAGS_DEBUG', 'CMAKE_CXX_FLAGS_DEBUG', 'CMAKE_C_FLAGS_RELEASE', 'CMAKE_CXX_FLAGS_RELEASE', @@ -36,6 +37,7 @@ 'CMAKE_SYSROOT', 'CMAKE_TOOLCHAIN_FILE', 'CMAKE_PREFIX_PATH', 'Eigen3_DIR', 'CMAKE_DISABLE_FIND_PACKAGE_Eigen3', 'FETCHCONTENT_SOURCE_DIR_EIGEN', 'PINEFORGE_ENABLE_SANITIZERS', 'PINEFORGE_ENABLE_COVERAGE', + 'PINEFORGE_VERSION_SOURCE', ) @@ -53,6 +55,59 @@ def read_cache(path: Path) -> dict[str, str]: return found +def copied_cache(cache: dict[str, str]) -> dict[str, str]: + return {key: cache[key] for key in COPY_CACHE if key in cache} + + +def refuse_existing_base(output: Path, reason: str) -> None: + raise RuntimeError(f'{reason}; use a fresh --build-dir (refusing to delete {output})') + + +def receipt_matches_current(receipt: dict, current_cache: dict, compiler: dict) -> None: + """Raise if a prepared base must not be reused with this current build.""" + if receipt.get('schemaVersion') != 'pineforge-settlement-abi-base/v1': + raise RuntimeError('existing ABI base receipt is not portable v1') + if receipt.get('commit') != BASE_COMMIT or receipt.get('tree') != BASE_TREE: + raise RuntimeError('existing ABI base does not pin e60 R2') + old = receipt.get('compiler') or {} + for key in ('target', 'sha256', 'version'): + if old.get(key) != compiler.get(key): + raise RuntimeError('existing ABI base compiler implementation/version/target differs') + if receipt.get('copiedCurrentCache') != copied_cache(current_cache): + raise RuntimeError( + 'existing ABI base compiler/configuration/version-source/launcher differs from this build') + + +def reusable_prepared_base(output: Path, current_build: Path) -> dict: + """Return an existing receipt that is safe to reuse. Never deletes output.""" + receipt_path = output / 'receipt.json' + if not output.exists(): + raise FileNotFoundError(str(output)) + if not receipt_path.is_file(): + refuse_existing_base(output, f'ABI base directory exists without a receipt at {receipt_path}') + receipt = json.loads(receipt_path.read_text()) + cache = read_cache(current_build / 'CMakeCache.txt') + compiler = compiler_identity(cache['CMAKE_CXX_COMPILER']) + try: + receipt_matches_current(receipt, cache, compiler) + except RuntimeError as error: + refuse_existing_base(output, str(error)) + resolve = lambda name: Path(name) if Path(name).is_absolute() else output / name + library = resolve(receipt['archive']) + headers = resolve(receipt.get('headers', 'r2-headers.tar')) + if not library.is_file() or not headers.is_file(): + refuse_existing_base(output, 'existing ABI base is missing archive/header artifacts') + if identity(library)['sha256'] != receipt['archiveSha256'] or identity(headers)['sha256'] != receipt['headersSha256']: + refuse_existing_base(output, 'existing ABI base archive/header bytes do not match receipt') + generated_name = receipt.get('generatedInclude') + expected_generated = receipt.get('generatedHeaderSha256') + if generated_name and expected_generated: + generated = resolve(generated_name) / 'pineforge/version.h' + if not generated.is_file() or identity(generated)['sha256'] != expected_generated: + refuse_existing_base(output, 'existing ABI base generated version header does not match receipt') + return receipt + + def current_eigen_source(cache: dict[str, str], current: Path) -> Path | None: """Reuse the current build's populated dependency without a network fetch.""" candidates = [cache.get(key) for key in @@ -158,7 +213,7 @@ def main() -> int: source, build = output / 'source', output / 'build' extract_tar(source_raw, source) authenticate_headers(source) - settings = {key: cache[key] for key in COPY_CACHE if key in cache} + settings = copied_cache(cache) eigen_source = current_eigen_source(cache, current) if eigen_source is not None: settings['FETCHCONTENT_SOURCE_DIR_EIGEN'] = str(eigen_source) @@ -189,7 +244,7 @@ def main() -> int: 'generatedHeaderSha256': identity(build / 'include/pineforge/version.h')['sha256'], 'sourceSha256': source_identity, 'sourceArchive': identity(output / 'source.tar'), 'compiler': compiler_identity(cache['CMAKE_CXX_COMPILER']), - 'settings': settings, 'copiedCurrentCache': {key: cache[key] for key in COPY_CACHE if key in cache}, + 'settings': settings, 'copiedCurrentCache': copied_cache(cache), 'currentCacheSha256': identity(current / 'CMakeCache.txt')['sha256'], 'baseCacheSha256': identity(build / 'CMakeCache.txt')['sha256'], 'compileCommandsSha256': identity(build / 'compile_commands.json')['sha256'], diff --git a/scripts/test_ci_verify.py b/scripts/test_ci_verify.py new file mode 100644 index 000000000..91b481853 --- /dev/null +++ b/scripts/test_ci_verify.py @@ -0,0 +1,710 @@ +#!/usr/bin/env python3 +"""Driver control-flow, config, aggregation, and receipt-reuse tests for ci_verify. + +Source guards, tool help, git object presence, compiler identity, and VERSION +are real. Full engine configure/build/CTest is not substituted as a passing +root verification — those stages are scripted here so failure aggregation and +ordering can be asserted. Root must still run the actual profiles. +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import shutil +import tempfile +import unittest + +import ci_verify +from ci_verify import ( + Completed, + ConfigError, + ROOT, + SANITIZER_FLAG, + SANITIZER_RUN_ENV, + cmake_cache_definitions, + cmake_configure_argv, + ctest_supports_junit, + default_build_dir, + default_runner, + expected_version, + main, + parse_args, + source_guard_commands, + validate_config, +) +from prepare_settlement_cpp_abi_base import ( + BASE_COMMIT, + BASE_TREE, + COPY_CACHE, + compiler_identity, + copied_cache, + identity, + receipt_matches_current, + reusable_prepared_base, +) + + +def write_cache(path: Path, values: dict[str, str]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + lines = [f'{key}:STRING={value}' for key, value in values.items()] + path.write_text('\n'.join(lines) + '\n') + + +def which_with_ccache(ccache: str | None): + def which(name: str) -> str | None: + if name == 'ccache': + return ccache + return shutil.which(name) + return which + + +class Scripted: + """Programmable command runner that still executes real source guards/tools.""" + + def __init__(self, build_dir: Path, source: Path, profile: str = 'release', **exits): + self.build_dir = Path(build_dir) + self.source = Path(source) + self.profile = profile + self.exits = exits + self.calls: list[list[str]] = [] + self.cxx = shutil.which('c++') or '/usr/bin/c++' + self.secret_seen_in_argv = False + + def __call__(self, argv, *, extra_env=None, timeout=600, combine_stderr=True, + stream_output=False) -> Completed: + argv = list(map(str, argv)) + self.calls.append(argv) + extra_env = extra_env or {} + joined = ' '.join(argv) + secret = os.environ.get('CI_VERIFY_TEST_SECRET') + if secret and secret in joined: + self.secret_seen_in_argv = True + if argv[0] == 'ctest' and '--help' in argv: + if self.exits.get('junit_help') == 'absent': + return Completed(0, b'CTest help without junit\n', b'') + return default_runner(argv, extra_env=None, timeout=timeout, + combine_stderr=True, stream_output=False) + if argv[0] in {'cmake', 'ctest', 'git'} and '--version' in argv: + return default_runner(argv, extra_env=None, timeout=timeout, + combine_stderr=True, stream_output=False) + if argv[0] == 'git' and 'cat-file' in argv: + if 'cat-file' in self.exits: + return Completed(int(self.exits['cat-file']), b'', b'') + return default_runner(argv, extra_env=None, timeout=timeout, + combine_stderr=True, stream_output=False) + if argv[0] == 'git' and 'fetch' in argv: + return Completed(int(self.exits.get('fetch', 0)), b'fetched\n', b'') + if any(Path(part).name == 'prepare_settlement_cpp_abi_base.py' for part in argv): + return Completed(int(self.exits.get('prepare', 0)), b'prepared\n', b'') + if any(Path(part).suffix == '.py' for part in argv): + needles = { + 'source-guard-c-abi': 'check_c_abi_runtime.py', + 'source-guard-broker-hash': 'check_broker_state_hash_coverage.py', + 'source-guard-pending-mirror': 'gen_pending_order_mirror.py', + 'source-guard-native-versions': 'check_native_cpp_versions.py', + 'source-guard-aggregate-versions': 'check_aggregate_cpp_versions.py', + } + for key, needle in needles.items(): + if key in self.exits and any(needle in part for part in argv): + return Completed(int(self.exits[key]), b'', b'injected source-guard failure\n') + return default_runner(argv, extra_env=None, timeout=timeout, + combine_stderr=True, stream_output=False) + if argv[0] == 'cmake' and '-S' in argv and 'smoke_consumer' in joined: + return Completed(int(self.exits.get('smoke-configure', 0)), b'ok\n', b'') + if argv[0] == 'cmake' and '-S' in argv: + return self._configure() + if argv[0] == 'cmake' and '--build' in argv and str(self.build_dir / 'ci-smoke') in argv: + return Completed(int(self.exits.get('smoke-build', 0)), b'ok\n', b'') + if argv[0] == 'cmake' and '--build' in argv: + return self._build() + if argv[0] == 'cmake' and '--install' in argv: + return self._install() + if argv[0] == 'ctest': + env_ok = True + if self.profile == 'sanitizers': + env_ok = extra_env == SANITIZER_RUN_ENV + if not env_ok: + return Completed(1, b'', b'sanitizer env missing\n') + return Completed(int(self.exits.get('ctest', 0)), b'tests\n', b'') + if argv[0].endswith('smoke_version') or Path(argv[0]).name == 'smoke_version': + stdout = str(self.exits.get('smoke-stdout', expected_version(self.source))).encode() + b'\n' + stderr = str(self.exits.get('smoke-stderr', '')).encode() + return Completed(int(self.exits.get('smoke-version', 0)), stdout, stderr) + if Path(argv[0]).name == 'pineforge-live': + return Completed(int(self.exits.get('native-help', 0)), b'usage\n', b'') + if Path(argv[0]).name == 'test_native_live_websocket': + return Completed(int(self.exits.get('require-websocket', 0)), b'ws\n', b'') + if '--version' in argv: + return Completed(0, b'version\n', b'') + return Completed(1, b'', f'unhandled command: {argv}\n'.encode()) + + def _cache_values(self) -> dict[str, str]: + tutorial = 'OFF' if self.profile == 'native' else 'ON' + live = 'ON' if self.profile == 'native' else 'OFF' + sanitizers = 'ON' if self.profile == 'sanitizers' else 'OFF' + build_type = 'Debug' if self.profile in {'debug', 'sanitizers'} else 'Release' + values = { + 'CMAKE_HOME_DIRECTORY': str(self.source), + 'CMAKE_CXX_COMPILER': self.cxx, + 'CMAKE_C_COMPILER': self.cxx, + 'CMAKE_GENERATOR': 'Unix Makefiles', + 'CMAKE_BUILD_TYPE': build_type, + 'PINEFORGE_BUILD_TESTS': 'ON', + 'PINEFORGE_BUILD_TUTORIAL': tutorial, + 'PINEFORGE_BUILD_LIVE_RUNNER': live, + 'PINEFORGE_ENABLE_SANITIZERS': sanitizers, + 'PINEFORGE_VERSION_SOURCE': 'FILE', + } + if self.exits.get('cache_version_source'): + values['PINEFORGE_VERSION_SOURCE'] = self.exits['cache_version_source'] + if self.exits.get('launcher'): + values['CMAKE_C_COMPILER_LAUNCHER'] = self.exits['launcher'] + values['CMAKE_CXX_COMPILER_LAUNCHER'] = self.exits['launcher'] + return values + + def _configure(self) -> Completed: + code = int(self.exits.get('configure', 0)) + if code != 0: + return Completed(code, b'', b'configure failed\n') + write_cache(self.build_dir / 'CMakeCache.txt', self._cache_values()) + if self.profile == 'sanitizers': + commands = [{ + 'directory': str(self.build_dir), + 'file': str(self.source / 'src/matrix.cpp'), + 'command': f'{self.cxx} {SANITIZER_FLAG} -c src/matrix.cpp', + }] + if self.exits.get('sanitizer_flag') == 'absent': + commands[0]['command'] = f'{self.cxx} -c src/matrix.cpp' + (self.build_dir / 'compile_commands.json').write_text(json.dumps(commands)) + self._maybe_seed_abi_base() + return Completed(0, b'configured\n', b'') + + def _build(self) -> Completed: + code = int(self.exits.get('build', 0)) + if code != 0: + return Completed(code, b'', b'build failed\n') + archive = self.build_dir / 'lib' / 'libpineforge.a' + archive.parent.mkdir(parents=True, exist_ok=True) + archive.write_bytes(b'!\nci-verify-test\n') + if self.profile == 'native' or self.exits.get('create_native_binaries'): + binary = self.build_dir / 'bin' / 'pineforge-live' + binary.parent.mkdir(parents=True, exist_ok=True) + binary.write_bytes(b'live') + ws = self.build_dir / 'bin' / 'test_native_live_websocket' + ws.write_bytes(b'ws') + if self.exits.get('omit_ws_binary'): + ws = self.build_dir / 'bin' / 'test_native_live_websocket' + if ws.exists(): + ws.unlink() + return Completed(0, b'built\n', b'') + + def _install(self) -> Completed: + code = int(self.exits.get('install', 0)) + if code != 0: + return Completed(code, b'', b'install failed\n') + prefix = self.build_dir / 'ci-install' + (prefix / 'lib' / 'cmake' / 'PineForge').mkdir(parents=True, exist_ok=True) + if self.profile == 'native': + help_bin = prefix / 'bin' / 'pineforge-live' + help_bin.parent.mkdir(parents=True, exist_ok=True) + help_bin.write_bytes(b'live') + (self.build_dir / 'ci-smoke').mkdir(parents=True, exist_ok=True) + (self.build_dir / 'ci-smoke' / 'smoke_version').write_bytes(b'smoke') + return Completed(0, b'installed\n', b'') + + def _maybe_seed_abi_base(self) -> None: + kind = self.exits.get('preexisting_base') + if not kind: + return + output = self.build_dir / 'settlement-abi-base' + output.mkdir(parents=True, exist_ok=True) + (output / 'sentinel').write_text('keep\n') + if kind == 'no-receipt': + return + archive = output / 'build' / 'lib' / 'libpineforge.a' + headers = output / 'r2-headers.tar' + generated = output / 'build' / 'include' / 'pineforge' / 'version.h' + archive.parent.mkdir(parents=True, exist_ok=True) + generated.parent.mkdir(parents=True, exist_ok=True) + archive.write_bytes(b'!\nbase\n') + headers.write_bytes(b'headers') + generated.write_text('#define PINEFORGE_VERSION_STRING "0.14.0"\n') + compiler = compiler_identity(self.cxx) + cache = copied_cache(self._cache_values()) + if kind == 'mismatch': + cache = dict(cache) + cache['PINEFORGE_VERSION_SOURCE'] = 'AUTO' + receipt = { + 'schemaVersion': 'pineforge-settlement-abi-base/v1', + 'commit': BASE_COMMIT, + 'tree': BASE_TREE, + 'archive': 'build/lib/libpineforge.a', + 'archiveSha256': identity(archive)['sha256'], + 'headers': 'r2-headers.tar', + 'headersSha256': identity(headers)['sha256'], + 'generatedInclude': 'build/include', + 'generatedHeaderSha256': identity(generated)['sha256'], + 'compiler': compiler, + 'copiedCurrentCache': cache, + } + (output / 'receipt.json').write_text(json.dumps(receipt, indent=2, sort_keys=True) + '\n') + + def names(self) -> list[str]: + names = [] + for argv in self.calls: + if argv[0] == 'cmake' and '--build' in argv and 'ci-smoke' not in ''.join(argv): + names.append('build') + elif argv[0] == 'cmake' and '--install' in argv: + names.append('install') + elif argv[0] == 'ctest' and '--test-dir' in argv: + names.append('ctest') + elif argv[0] == 'git' and 'fetch' in argv: + names.append('fetch') + elif any(Path(part).name == 'prepare_settlement_cpp_abi_base.py' for part in argv): + names.append('prepare') + elif argv[0] == 'cmake' and '-S' in argv and 'smoke_consumer' in ''.join(argv): + names.append('smoke-configure') + elif Path(argv[0]).name == 'smoke_version': + names.append('smoke-version') + elif Path(argv[0]).name == 'test_native_live_websocket': + names.append('require-websocket') + return names + + +def run_driver(profile: str, build_dir: Path, scripted: Scripted, extra: list[str] | None = None, + which=shutil.which) -> int: + argv = [profile, '--build-dir', str(build_dir), '--jobs', '2', *(extra or [])] + return main(argv, runner=scripted, stream_output=False, which=which) + + +def load_summary(build_dir: Path) -> dict: + return json.loads((build_dir / 'ci-summary.json').read_text()) + + +def stage_names(summary: dict) -> list[str]: + return [stage['name'] for stage in summary['stages']] + + +def failure_stages(summary: dict) -> list[str]: + return [item['stage'] for item in summary['failures']] + + +class ConfigValidation(unittest.TestCase): + def test_unknown_profile(self): + with self.assertRaises(ConfigError): + parse_args(['relaxed']) + self.assertEqual(main(['relaxed']), 2) + + def test_unknown_argument_is_rejected(self): + with self.assertRaises(ConfigError): + parse_args(['release', '--skip-tests']) + self.assertEqual(main(['release', '--skip-tests']), 2) + + def test_jobs_range(self): + with self.assertRaisesRegex(ConfigError, '--jobs'): + validate_config(parse_args(['release', '--jobs', '0'])) + with self.assertRaisesRegex(ConfigError, '--jobs'): + validate_config(parse_args(['release', '--jobs', '99'])) + self.assertEqual(main(['release', '--jobs', '0']), 2) + + def test_require_websocket_only_native(self): + with self.assertRaisesRegex(ConfigError, 'native'): + validate_config(parse_args(['release', '--require-websocket'])) + self.assertEqual(main(['debug', '--require-websocket']), 2) + + def test_ccache_requires_installed_tool(self): + args = parse_args(['release', '--ccache']) + with self.assertRaisesRegex(ConfigError, 'ccache'): + validate_config(args, which=which_with_ccache(None)) + self.assertEqual(main(['release', '--ccache'], which=which_with_ccache(None)), 2) + + def test_ccache_binds_compiler_launchers(self): + launcher = '/usr/bin/true' + cfg = validate_config(parse_args(['release', '--ccache', '--build-dir', 'build-ci-x']), + which=which_with_ccache(launcher)) + values = cmake_cache_definitions(cfg) + resolved = str(Path(launcher).resolve()) + self.assertEqual(values['CMAKE_C_COMPILER_LAUNCHER'], resolved) + self.assertEqual(values['CMAKE_CXX_COMPILER_LAUNCHER'], resolved) + self.assertEqual(values['PINEFORGE_VERSION_SOURCE'], 'FILE') + + def test_curl_dir_must_exist(self): + missing = ROOT / 'definitely-missing-curl-dir' + with self.assertRaisesRegex(ConfigError, 'curl-dir'): + validate_config(parse_args(['native', '--curl-dir', str(missing)])) + + def test_default_build_dirs_are_profile_isolated(self): + for name in ('release', 'debug', 'sanitizers', 'native'): + self.assertEqual(default_build_dir(ROOT, name), ROOT / f'build-ci-{name}') + args = parse_args([name], source=ROOT) + self.assertEqual(Path(args.build_dir), ROOT / f'build-ci-{name}') + + def test_build_dir_cannot_be_source_root(self): + with self.assertRaisesRegex(ConfigError, 'source root'): + validate_config(parse_args(['release', '--build-dir', str(ROOT)])) + + +class ProfileOptions(unittest.TestCase): + def definitions(self, profile: str, extra=None, which=shutil.which): + args = parse_args([profile, '--build-dir', 'tmp-build', *(extra or [])], source=ROOT) + cfg = validate_config(args, which=which) + return cmake_cache_definitions(cfg), cmake_configure_argv(cfg) + + def test_release_keeps_tutorial_on_native_off(self): + values, argv = self.definitions('release') + self.assertEqual(values['CMAKE_BUILD_TYPE'], 'Release') + self.assertEqual(values['PINEFORGE_BUILD_TUTORIAL'], 'ON') + self.assertEqual(values['PINEFORGE_BUILD_LIVE_RUNNER'], 'OFF') + self.assertEqual(values['PINEFORGE_ENABLE_SANITIZERS'], 'OFF') + self.assertEqual(values['PINEFORGE_BUILD_TESTS'], 'ON') + self.assertEqual(values['PINEFORGE_VERSION_SOURCE'], 'FILE') + self.assertIn('-DPINEFORGE_VERSION_SOURCE=FILE', argv) + self.assertNotIn('AUTO', ''.join(argv)) + + def test_debug_explicit(self): + values, _ = self.definitions('debug') + self.assertEqual(values['CMAKE_BUILD_TYPE'], 'Debug') + self.assertEqual(values['PINEFORGE_BUILD_TUTORIAL'], 'ON') + self.assertEqual(values['PINEFORGE_BUILD_LIVE_RUNNER'], 'OFF') + self.assertEqual(values['PINEFORGE_ENABLE_SANITIZERS'], 'OFF') + + def test_sanitizers_public_flag_expected_and_native_off(self): + values, argv = self.definitions('sanitizers') + self.assertEqual(values['CMAKE_BUILD_TYPE'], 'Debug') + self.assertEqual(values['PINEFORGE_ENABLE_SANITIZERS'], 'ON') + self.assertEqual(values['PINEFORGE_BUILD_LIVE_RUNNER'], 'OFF') + self.assertEqual(values['PINEFORGE_BUILD_TUTORIAL'], 'ON') + self.assertIn('-DPINEFORGE_ENABLE_SANITIZERS=ON', argv) + + def test_native_live_on_tutorial_off(self): + curl = tempfile.TemporaryDirectory() + self.addCleanup(curl.cleanup) + values, argv = self.definitions( + 'native', extra=['--generator', 'Ninja', '--curl-dir', curl.name, '--require-websocket']) + self.assertEqual(values['PINEFORGE_BUILD_LIVE_RUNNER'], 'ON') + self.assertEqual(values['PINEFORGE_BUILD_TUTORIAL'], 'OFF') + self.assertEqual(values['PINEFORGE_ENABLE_SANITIZERS'], 'OFF') + self.assertEqual(values['CURL_DIR'], str(Path(curl.name).resolve())) + self.assertIn('-G', argv) + self.assertIn('Ninja', argv) + + +class CopyCacheIdentity(unittest.TestCase): + def test_copy_cache_includes_version_source_and_launchers(self): + self.assertIn('PINEFORGE_VERSION_SOURCE', COPY_CACHE) + self.assertIn('CMAKE_C_COMPILER_LAUNCHER', COPY_CACHE) + self.assertIn('CMAKE_CXX_COMPILER_LAUNCHER', COPY_CACHE) + + def test_receipt_mismatch_on_version_source(self): + compiler = {'target': 't', 'sha256': 'abc', 'version': 'cc'} + receipt = { + 'schemaVersion': 'pineforge-settlement-abi-base/v1', + 'commit': BASE_COMMIT, + 'tree': BASE_TREE, + 'compiler': compiler, + 'copiedCurrentCache': {'PINEFORGE_VERSION_SOURCE': 'AUTO'}, + } + with self.assertRaisesRegex(RuntimeError, 'version-source/launcher'): + receipt_matches_current(receipt, {'PINEFORGE_VERSION_SOURCE': 'FILE'}, compiler) + + def test_receipt_match_on_identical_identity(self): + compiler = {'target': 't', 'sha256': 'abc', 'version': 'cc'} + cache = {'PINEFORGE_VERSION_SOURCE': 'FILE', 'CMAKE_CXX_COMPILER_LAUNCHER': '/ccache'} + receipt = { + 'schemaVersion': 'pineforge-settlement-abi-base/v1', + 'commit': BASE_COMMIT, + 'tree': BASE_TREE, + 'compiler': compiler, + 'copiedCurrentCache': copied_cache(cache), + } + receipt_matches_current(receipt, cache, compiler) + + +class ReceiptReuse(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + self.build = self.root / 'build' + self.output = self.build / 'settlement-abi-base' + self.cxx = shutil.which('c++') or '/usr/bin/c++' + self.output.mkdir(parents=True) + (self.output / 'sentinel').write_text('keep\n') + + def tearDown(self): + self.temporary.cleanup() + + def seed(self, *, version='FILE', corrupt=False, receipt=True): + archive = self.output / 'build' / 'lib' / 'libpineforge.a' + headers = self.output / 'r2-headers.tar' + generated = self.output / 'build' / 'include' / 'pineforge' / 'version.h' + archive.parent.mkdir(parents=True, exist_ok=True) + generated.parent.mkdir(parents=True, exist_ok=True) + archive.write_bytes(b'!\nbase\n') + headers.write_bytes(b'headers') + generated.write_text('version\n') + write_cache(self.build / 'CMakeCache.txt', { + 'CMAKE_CXX_COMPILER': self.cxx, + 'PINEFORGE_VERSION_SOURCE': 'FILE', + 'CMAKE_BUILD_TYPE': 'Release', + }) + if not receipt: + return + current = copied_cache(read_cache_for_test(self.build / 'CMakeCache.txt')) + payload = { + 'schemaVersion': 'pineforge-settlement-abi-base/v1', + 'commit': BASE_COMMIT, + 'tree': BASE_TREE, + 'archive': 'build/lib/libpineforge.a', + 'archiveSha256': identity(archive)['sha256'], + 'headers': 'r2-headers.tar', + 'headersSha256': identity(headers)['sha256'], + 'generatedInclude': 'build/include', + 'generatedHeaderSha256': identity(generated)['sha256'], + 'compiler': compiler_identity(self.cxx), + 'copiedCurrentCache': dict(current), + } + if version != 'FILE': + payload['copiedCurrentCache']['PINEFORGE_VERSION_SOURCE'] = version + if corrupt: + payload['archiveSha256'] = '0' * 64 + (self.output / 'receipt.json').write_text(json.dumps(payload) + '\n') + + def test_matching_receipt_is_reused(self): + self.seed() + receipt = reusable_prepared_base(self.output, self.build) + self.assertEqual(receipt['commit'], BASE_COMMIT) + self.assertTrue((self.output / 'sentinel').is_file()) + + def test_mismatch_refuses_without_deletion(self): + self.seed(version='AUTO') + with self.assertRaisesRegex(RuntimeError, 'fresh --build-dir'): + reusable_prepared_base(self.output, self.build) + self.assertTrue((self.output / 'sentinel').is_file()) + self.assertTrue((self.output / 'receipt.json').is_file()) + + def test_missing_receipt_refuses_without_deletion(self): + self.seed(receipt=False) + with self.assertRaisesRegex(RuntimeError, 'fresh --build-dir'): + reusable_prepared_base(self.output, self.build) + self.assertTrue((self.output / 'sentinel').is_file()) + + def test_corrupt_archive_hash_refuses_without_deletion(self): + self.seed(corrupt=True) + with self.assertRaisesRegex(RuntimeError, 'fresh --build-dir'): + reusable_prepared_base(self.output, self.build) + self.assertTrue((self.output / 'sentinel').is_file()) + + +def read_cache_for_test(path: Path) -> dict[str, str]: + from prepare_settlement_cpp_abi_base import read_cache + return read_cache(path) + + +class DriverOrderingAndAggregation(unittest.TestCase): + def test_local_ccache_uses_content_identity(self): + seen = [] + def runner(argv, **kwargs): + seen.append(kwargs["extra_env"]) + return Completed(0) + with tempfile.TemporaryDirectory() as temporary: + cfg = ci_verify.build_config( + ["release", "--build-dir", temporary, "--ccache"], runner=runner, + stream_output=False, which=lambda name: "/usr/bin/" + name) + driver = ci_verify.Driver(cfg) + driver.invoke("probe", ["compile"], extra_env={"PRESERVED_SETTING": "yes"}) + self.assertEqual(seen, [{"PRESERVED_SETTING": "yes", "CCACHE_COMPILERCHECK": "content"}]) + + def test_runner_type_error_is_not_retried(self): + calls = [] + def failing_runner(argv, **kwargs): + calls.append(argv) + raise TypeError("runner failure after dispatch") + with self.assertRaisesRegex(TypeError, "after dispatch"): + ci_verify.call_runner(failing_runner, ["build-once"]) + self.assertEqual(calls, [["build-once"]]) + + def run_profile(self, profile='release', extra=None, **exits): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + build_dir = Path(temporary.name) / 'build' + scripted = Scripted(build_dir, ROOT, profile, **exits) + code = run_driver(profile, build_dir, scripted, extra=extra) + summary = load_summary(build_dir) + return code, summary, scripted, build_dir + + def test_source_guards_are_real_and_pass_on_this_tree(self): + for name, argv in source_guard_commands(ROOT): + with self.subTest(name=name): + result = default_runner(argv, extra_env=None, timeout=120, + combine_stderr=True, stream_output=False) + self.assertEqual(result.returncode, 0, result.stdout[-2000:]) + + def test_build_failure_skips_ctest_and_install(self): + code, summary, scripted, _ = self.run_profile(build=1) + self.assertEqual(code, 1) + self.assertIn('build', failure_stages(summary)) + self.assertNotIn('ctest', scripted.names()) + self.assertNotIn('install', scripted.names()) + self.assertNotIn('smoke-version', scripted.names()) + + def test_configure_failure_skips_build(self): + code, summary, scripted, _ = self.run_profile(configure=1) + self.assertEqual(code, 1) + self.assertIn('configure', failure_stages(summary)) + self.assertNotIn('build', scripted.names()) + self.assertNotIn('ctest', scripted.names()) + + def test_source_guard_failure_skips_configure(self): + code, summary, scripted, _ = self.run_profile(**{'source-guard-c-abi': 1}) + self.assertEqual(code, 1) + self.assertIn('source-guard-c-abi', failure_stages(summary)) + self.assertFalse(any(argv[0] == 'cmake' and '-S' in argv for argv in scripted.calls)) + + def test_ctest_and_smoke_failures_are_both_collected(self): + code, summary, scripted, _ = self.run_profile(ctest=8, **{'smoke-stdout': '9.9.9'}) + self.assertEqual(code, 1) + self.assertIn('ctest', scripted.names()) + self.assertIn('install', scripted.names()) + self.assertIn('smoke-version', scripted.names()) + self.assertIn('ctest', failure_stages(summary)) + self.assertIn('smoke-version-check', failure_stages(summary)) + message = next(item['error'] for item in summary['failures'] if item['stage'] == 'smoke-version-check') + self.assertIn(repr(expected_version(ROOT)), message) + self.assertIn(repr('9.9.9'), message) + self.assertEqual(summary['expectedVersion'], expected_version(ROOT)) + self.assertEqual(summary['actualVersion'], '9.9.9') + + def test_ctest_failure_does_not_hide_install_failure(self): + code, summary, scripted, _ = self.run_profile(ctest=8, install=1) + self.assertEqual(code, 1) + self.assertIn('ctest', failure_stages(summary)) + self.assertIn('install', failure_stages(summary)) + self.assertNotIn('smoke-version', scripted.names()) + + def test_successful_scripted_release_exit_zero(self): + code, summary, scripted, build_dir = self.run_profile() + self.assertEqual(code, 0, summary['failures']) + self.assertEqual(summary['status'], 'passed') + self.assertEqual(summary['exitCode'], 0) + self.assertEqual(summary['versionSource'], 'FILE') + self.assertIn('ctest', scripted.names()) + self.assertIn('install', scripted.names()) + self.assertTrue((build_dir / 'ci-logs' / 'ctest.log').is_file()) + self.assertTrue((build_dir / 'ci-summary.json').is_file()) + ctest_argv = next(argv for argv in scripted.calls if argv[0] == 'ctest' and '--test-dir' in argv) + if ctest_supports_junit(scripted): + self.assertIn('--output-junit', ctest_argv) + self.assertEqual(Path(ctest_argv[ctest_argv.index('--output-junit') + 1]).resolve(), + (build_dir / 'ctest-junit.xml').resolve()) + + def test_junit_flag_omitted_when_unsupported(self): + code, summary, scripted, build_dir = self.run_profile(junit_help='absent') + self.assertEqual(code, 0, summary['failures']) + ctest_argv = next(argv for argv in scripted.calls if argv[0] == 'ctest' and '--test-dir' in argv) + self.assertNotIn('--output-junit', ctest_argv) + + def test_matching_base_is_reused_without_fetch_or_prepare(self): + code, summary, scripted, _ = self.run_profile(preexisting_base='match') + self.assertEqual(code, 0, summary['failures']) + self.assertEqual(summary['abi']['action'], 'reused') + self.assertNotIn('fetch', scripted.names()) + self.assertNotIn('prepare', scripted.names()) + + def test_mismatch_base_refuses_without_deletion_and_still_runs_ctest_install(self): + code, summary, scripted, build_dir = self.run_profile(preexisting_base='mismatch') + self.assertEqual(code, 1) + self.assertEqual(summary['abi']['action'], 'refused') + self.assertIn('fresh --build-dir', summary['failures'][0]['error']) + self.assertTrue((build_dir / 'settlement-abi-base' / 'sentinel').is_file()) + self.assertNotIn('prepare', scripted.names()) + self.assertIn('ctest', scripted.names()) + self.assertIn('install', scripted.names()) + + def test_fetch_no_tags_only_when_object_missing(self): + code, summary, scripted, _ = self.run_profile(**{'cat-file': 1}) + self.assertEqual(code, 0, summary['failures']) + fetch = next(argv for argv in scripted.calls if argv[0] == 'git' and 'fetch' in argv) + self.assertIn('--no-tags', fetch) + self.assertNotIn('--tags', fetch) + self.assertIn(BASE_COMMIT, fetch) + self.assertIn('prepare', scripted.names()) + + def test_present_object_does_not_fetch(self): + _, _, scripted, _ = self.run_profile() + self.assertNotIn('fetch', scripted.names()) + self.assertIn('prepare', scripted.names()) + + def test_require_websocket_rejects_skip(self): + code, summary, scripted, _ = self.run_profile( + 'native', extra=['--require-websocket'], **{'require-websocket': 77}) + self.assertEqual(code, 1) + self.assertIn('require-websocket', scripted.names()) + self.assertIn('require-websocket-skip', failure_stages(summary)) + self.assertTrue(any('cannot accept skip' in (item.get('error') or '') + for item in summary['failures'])) + + def test_require_websocket_rejects_missing_binary(self): + code, summary, scripted, _ = self.run_profile( + 'native', extra=['--require-websocket'], omit_ws_binary=True) + self.assertEqual(code, 1) + self.assertIn('require-websocket', failure_stages(summary)) + self.assertNotIn('require-websocket', scripted.names()) + + def test_native_help_runs_after_install(self): + code, summary, scripted, _ = self.run_profile('native') + self.assertEqual(code, 0, summary['failures']) + self.assertIn('native-help', stage_names(summary)) + names = stage_names(summary) + self.assertLess(names.index('install'), names.index('native-help')) + + def test_sanitizer_missing_public_flag_fails_before_build(self): + code, summary, scripted, _ = self.run_profile('sanitizers', sanitizer_flag='absent') + self.assertEqual(code, 1) + self.assertIn('sanitizer-public-flag', failure_stages(summary)) + self.assertNotIn('build', scripted.names()) + + def test_sanitizers_profile_passes_public_flag_and_asan_env(self): + code, summary, scripted, _ = self.run_profile('sanitizers') + self.assertEqual(code, 0, summary['failures']) + self.assertIn('sanitizer-public-flag', stage_names(summary)) + ctest = next(stage for stage in summary['stages'] if stage['name'] == 'ctest') + self.assertEqual(ctest['extraEnvKeys'], ['ASAN_OPTIONS', 'UBSAN_OPTIONS']) + self.assertNotIn('ctest', failure_stages(summary)) + + def test_smoke_ignores_stderr_noise_and_checks_stdout(self): + code, summary, _, _ = self.run_profile(**{'smoke-stderr': 'AddressSanitizer noise'}) + self.assertEqual(code, 0, summary['failures']) + self.assertEqual(summary['actualVersion'], expected_version(ROOT)) + + def test_logs_and_summary_omit_environment_secrets(self): + secret = 's3cret-ci-verify-token-not-for-logs' + previous = os.environ.get('CI_VERIFY_TEST_SECRET') + os.environ['CI_VERIFY_TEST_SECRET'] = secret + os.environ['AWS_SECRET_ACCESS_KEY'] = secret + try: + code, summary, scripted, build_dir = self.run_profile() + self.assertEqual(code, 0, summary['failures']) + dumped = json.dumps(summary) + for path in (build_dir / 'ci-logs').glob('*'): + dumped += path.read_text(errors='replace') + self.assertNotIn(secret, dumped) + self.assertFalse(scripted.secret_seen_in_argv) + self.assertNotIn('AWS_SECRET_ACCESS_KEY', dumped) + finally: + os.environ.pop('AWS_SECRET_ACCESS_KEY', None) + if previous is None: + os.environ.pop('CI_VERIFY_TEST_SECRET', None) + else: + os.environ['CI_VERIFY_TEST_SECRET'] = previous + + def test_ctest_junit_detection_matches_real_ctest(self): + help_text = default_runner(['ctest', '--help'], extra_env=None, timeout=30, + combine_stderr=True, stream_output=False) + self.assertEqual(help_text.returncode, 0) + self.assertEqual(ctest_supports_junit(default_runner), + '--output-junit' in help_text.stdout.decode()) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_cmake_version_source.py b/scripts/test_cmake_version_source.py new file mode 100644 index 000000000..34f864664 --- /dev/null +++ b/scripts/test_cmake_version_source.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 +"""Real CMake/Git fixtures for PINEFORGE_VERSION_SOURCE AUTO|FILE. + +Does not build the engine, bump VERSION, retag the runtime, or invoke git in +the source worktree. Each case is an isolated temporary tree. +""" +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CMAKE_DIR = ROOT / "cmake" +MODULE = CMAKE_DIR / "PineForgeVersion.cmake" +PROBE_NAME = "version_probe.txt" + +PROBE_CMAKELISTS = """cmake_minimum_required(VERSION 3.16) +list(APPEND CMAKE_MODULE_PATH "@CMAKE_DIR@") +include(PineForgeVersion) +pineforge_resolve_version() +file(WRITE "${CMAKE_BINARY_DIR}/version_probe.txt" + "PINEFORGE_VERSION_MAJOR=${PINEFORGE_VERSION_MAJOR}\\n" + "PINEFORGE_VERSION_MINOR=${PINEFORGE_VERSION_MINOR}\\n" + "PINEFORGE_VERSION_PATCH=${PINEFORGE_VERSION_PATCH}\\n" + "PINEFORGE_VERSION_MMP=${PINEFORGE_VERSION_MMP}\\n" + "PINEFORGE_VERSION_FULL=${PINEFORGE_VERSION_FULL}\\n" + "PINEFORGE_VERSION_GIT_SHA=${PINEFORGE_VERSION_GIT_SHA}\\n" + "PINEFORGE_VERSION_DIRTY=${PINEFORGE_VERSION_DIRTY}\\n" + "PINEFORGE_VERSION_SOURCE=${PINEFORGE_VERSION_SOURCE}\\n" +) +project(pineforge_version_probe VERSION ${PINEFORGE_VERSION_MMP} LANGUAGES NONE) +""" + + +def _which(name: str) -> str: + path = shutil.which(name) + if not path: + raise RuntimeError(name + " is required for version-source fixtures") + return path + + +def parse_probe(path: Path) -> dict[str, str]: + found: dict[str, str] = {} + for line in path.read_text().splitlines(): + if not line or "=" not in line: + raise AssertionError("unreadable probe line: " + line) + key, value = line.split("=", 1) + found[key] = value + return found + + +def cache_value(build: Path, name: str) -> str | None: + for line in (build / "CMakeCache.txt").read_text().splitlines(): + if line.startswith(name + ":") and "=" in line: + return line.split("=", 1)[1] + return None + + +class VersionSourceTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.cmake_bin = _which("cmake") + cls.git_bin = _which("git") + if not MODULE.is_file(): + raise RuntimeError("missing " + str(MODULE)) + + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory(prefix="pf-version-source-") + self.base = Path(self._tmp.name) + + def tearDown(self) -> None: + self._tmp.cleanup() + + def git_env(self) -> dict[str, str]: + env = os.environ.copy() + for key in ( + "GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", "GIT_DIR_CEILING", + ): + env.pop(key, None) + env.update({ + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_TERMINAL_PROMPT": "0", + "GIT_AUTHOR_NAME": "version-source-test", + "GIT_AUTHOR_EMAIL": "version-source-test@pineforge.invalid", + "GIT_COMMITTER_NAME": "version-source-test", + "GIT_COMMITTER_EMAIL": "version-source-test@pineforge.invalid", + "GIT_AUTHOR_DATE": "2026-01-01T00:00:00Z", + "GIT_COMMITTER_DATE": "2026-01-01T00:00:00Z", + }) + return env + + def git(self, repo: Path, *args: str) -> str: + result = subprocess.run( + [self.git_bin, "-c", "init.defaultBranch=main", + "-c", "commit.gpgsign=false", "-c", "tag.gpgsign=false", *args], + cwd=repo, env=self.git_env(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False) + if result.returncode: + raise RuntimeError( + "git " + " ".join(args) + " failed (" + str(result.returncode) + "): " + + result.stderr.strip()) + return result.stdout.strip() + + def write_tree(self, source: Path, version: str) -> None: + source.mkdir(parents=True, exist_ok=True) + source.joinpath("VERSION").write_text(version + "\n", encoding="utf-8") + text = PROBE_CMAKELISTS.replace("@CMAKE_DIR@", CMAKE_DIR.as_posix()) + source.joinpath("CMakeLists.txt").write_text(text, encoding="utf-8") + + def init_repo(self, source: Path) -> None: + self.git(source, "init") + self.git(source, "add", "VERSION", "CMakeLists.txt") + self.git(source, "commit", "-m", "fixture") + + def expected_describe(self, repo: Path) -> str: + desc = self.git(repo, "describe", "--tags", "--match", "v*", "--abbrev=7", "--dirty") + if desc.startswith("v"): + desc = desc[1:] + return desc + + def expected_sha(self, repo: Path) -> str: + return self.git(repo, "rev-parse", "--short=7", "HEAD") + + def configure(self, source: Path, build: Path, *defs: str) -> subprocess.CompletedProcess[str]: + build.mkdir(parents=True, exist_ok=True) + return subprocess.run( + [self.cmake_bin, "-S", str(source), "-B", str(build), *defs], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=60) + + def probe(self, source: Path, label: str, *defs: str) -> dict[str, str]: + build = self.base / "build" / label + result = self.configure(source, build, *defs) + probe_path = build / PROBE_NAME + if result.returncode: + raise AssertionError( + "cmake configure failed (" + str(result.returncode) + "):\n" + result.stdout) + self.assertTrue(probe_path.is_file(), result.stdout) + values = parse_probe(probe_path) + values["_cache_source"] = cache_value(build, "PINEFORGE_VERSION_SOURCE") or "" + values["_log"] = result.stdout + values["_build"] = str(build) + return values + + def configure_fail(self, source: Path, label: str, *defs: str) -> str: + build = self.base / "build" / label + result = self.configure(source, build, *defs) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertFalse((build / PROBE_NAME).exists(), result.stdout) + return result.stdout + + def test_default_auto_matches_explicit_auto(self) -> None: + source = self.base / "auto-default" + self.write_tree(source, "0.14.0") + self.init_repo(source) + self.git(source, "tag", "v0.13.1") + omitted = self.probe(source, "omitted") + explicit = self.probe(source, "explicit", "-DPINEFORGE_VERSION_SOURCE=AUTO") + for key in ( + "PINEFORGE_VERSION_MAJOR", "PINEFORGE_VERSION_MINOR", "PINEFORGE_VERSION_PATCH", + "PINEFORGE_VERSION_MMP", "PINEFORGE_VERSION_FULL", "PINEFORGE_VERSION_GIT_SHA", + "PINEFORGE_VERSION_DIRTY", "PINEFORGE_VERSION_SOURCE", + ): + self.assertEqual(omitted[key], explicit[key], key) + full = self.expected_describe(source) + sha = self.expected_sha(source) + self.assertEqual(omitted["PINEFORGE_VERSION_SOURCE"], "AUTO") + self.assertEqual(omitted["_cache_source"], "AUTO") + self.assertEqual(explicit["_cache_source"], "AUTO") + self.assertEqual(omitted["PINEFORGE_VERSION_MMP"], "0.13.1") + self.assertEqual(omitted["PINEFORGE_VERSION_FULL"], full) + self.assertEqual(omitted["PINEFORGE_VERSION_GIT_SHA"], sha) + self.assertEqual(omitted["PINEFORGE_VERSION_DIRTY"], "OFF") + self.assertNotEqual(omitted["PINEFORGE_VERSION_MMP"], "0.14.0") + + def test_mismatched_tag_file_uses_version(self) -> None: + source = self.base / "mismatch" + self.write_tree(source, "0.14.0") + self.init_repo(source) + self.git(source, "tag", "v9.9.9") + source.joinpath("extra.txt").write_text("second commit\n", encoding="utf-8") + self.git(source, "add", "extra.txt") + self.git(source, "commit", "-m", "ahead of tag") + auto = self.probe(source, "mismatch-auto", "-DPINEFORGE_VERSION_SOURCE=AUTO") + file_mode = self.probe(source, "mismatch-file", "-DPINEFORGE_VERSION_SOURCE=FILE") + full = self.expected_describe(source) + sha = self.expected_sha(source) + self.assertEqual(auto["PINEFORGE_VERSION_MMP"], "9.9.9") + self.assertEqual(auto["PINEFORGE_VERSION_FULL"], full) + self.assertTrue(full.startswith("9.9.9-1-g"), full) + self.assertEqual(file_mode["PINEFORGE_VERSION_SOURCE"], "FILE") + self.assertEqual(file_mode["_cache_source"], "FILE") + self.assertEqual(file_mode["PINEFORGE_VERSION_MMP"], "0.14.0") + self.assertEqual(file_mode["PINEFORGE_VERSION_FULL"], "0.14.0") + self.assertEqual(file_mode["PINEFORGE_VERSION_MAJOR"], "0") + self.assertEqual(file_mode["PINEFORGE_VERSION_MINOR"], "14") + self.assertEqual(file_mode["PINEFORGE_VERSION_PATCH"], "0") + self.assertEqual(auto["PINEFORGE_VERSION_GIT_SHA"], sha) + self.assertEqual(file_mode["PINEFORGE_VERSION_GIT_SHA"], sha) + self.assertEqual(auto["PINEFORGE_VERSION_DIRTY"], "OFF") + self.assertEqual(file_mode["PINEFORGE_VERSION_DIRTY"], "OFF") + self.assertNotEqual(file_mode["PINEFORGE_VERSION_FULL"], auto["PINEFORGE_VERSION_FULL"]) + + def test_shallow_no_tag(self) -> None: + origin = self.base / "origin" + self.write_tree(origin, "2.3.4") + self.init_repo(origin) + self.git(origin, "tag", "v1.0.0") + origin.joinpath("later.txt").write_text("later\n", encoding="utf-8") + self.git(origin, "add", "later.txt") + self.git(origin, "commit", "-m", "untagged head") + clone = self.base / "shallow" + # Local clones otherwise copy all objects and are not shallow. + self.git(self.base, "clone", "--depth=1", "--no-tags", "--no-local", + str(origin), str(clone)) + tags = self.git(clone, "tag", "-l") + self.assertEqual(tags, "") + self.assertEqual(self.git(clone, "rev-parse", "--is-shallow-repository"), "true") + self.assertTrue((clone / ".git" / "shallow").is_file()) + describe = subprocess.run( + [self.git_bin, "describe", "--tags", "--match", "v*", "--abbrev=7", "--dirty"], + cwd=clone, env=self.git_env(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + self.assertNotEqual(describe.returncode, 0, describe.stdout + describe.stderr) + self.assertIn("No names found", describe.stderr) + sha = self.expected_sha(clone) + self.assertEqual(len(self.git(clone, "rev-list", "--all").splitlines()), 1) + auto = self.probe(clone, "shallow-auto", "-DPINEFORGE_VERSION_SOURCE=AUTO") + file_mode = self.probe(clone, "shallow-file", "-DPINEFORGE_VERSION_SOURCE=FILE") + for values, source in ((auto, "AUTO"), (file_mode, "FILE")): + self.assertEqual(values["PINEFORGE_VERSION_SOURCE"], source) + self.assertEqual(values["PINEFORGE_VERSION_MMP"], "2.3.4") + self.assertEqual(values["PINEFORGE_VERSION_FULL"], "2.3.4") + self.assertEqual(values["PINEFORGE_VERSION_GIT_SHA"], sha) + self.assertEqual(values["PINEFORGE_VERSION_DIRTY"], "OFF") + + def test_archive_no_git(self) -> None: + source = self.base / "archive" + self.write_tree(source, "0.14.0") + self.assertFalse((source / ".git").exists()) + auto = self.probe(source, "archive-auto") + file_mode = self.probe(source, "archive-file", "-DPINEFORGE_VERSION_SOURCE=FILE") + for values in (auto, file_mode): + self.assertEqual(values["PINEFORGE_VERSION_MMP"], "0.14.0") + self.assertEqual(values["PINEFORGE_VERSION_FULL"], "0.14.0") + self.assertEqual(values["PINEFORGE_VERSION_GIT_SHA"], "unknown") + self.assertEqual(values["PINEFORGE_VERSION_DIRTY"], "OFF") + self.assertEqual(auto["PINEFORGE_VERSION_SOURCE"], "AUTO") + self.assertEqual(file_mode["PINEFORGE_VERSION_SOURCE"], "FILE") + + def test_invalid_mode_fails_clearly(self) -> None: + source = self.base / "invalid" + self.write_tree(source, "0.14.0") + self.init_repo(source) + self.git(source, "tag", "v0.14.0") + for index, mode in enumerate(("BOGUS", "auto", "file", "GIT", "AUTO FILE", "")): + with self.subTest(mode=mode): + log = self.configure_fail( + source, "invalid-" + str(index), + "-DPINEFORGE_VERSION_SOURCE=" + mode) + self.assertIn("PINEFORGE_VERSION_SOURCE must be AUTO or FILE", log) + self.assertIn("got '" + mode + "'", log) + self.assertNotIn("Configuring done", log) + + def test_file_preserves_dirty_without_retagging_full(self) -> None: + source = self.base / "dirty" + self.write_tree(source, "3.2.1") + self.init_repo(source) + self.git(source, "tag", "v8.8.8") + source.joinpath("CMakeLists.txt").write_text( + source.joinpath("CMakeLists.txt").read_text(encoding="utf-8") + "\n# dirty\n", + encoding="utf-8") + full = self.expected_describe(source) + sha = self.expected_sha(source) + self.assertTrue(full.endswith("-dirty"), full) + auto = self.probe(source, "dirty-auto", "-DPINEFORGE_VERSION_SOURCE=AUTO") + file_mode = self.probe(source, "dirty-file", "-DPINEFORGE_VERSION_SOURCE=FILE") + self.assertEqual(auto["PINEFORGE_VERSION_MMP"], "8.8.8") + self.assertEqual(auto["PINEFORGE_VERSION_FULL"], full) + self.assertEqual(auto["PINEFORGE_VERSION_DIRTY"], "ON") + self.assertEqual(file_mode["PINEFORGE_VERSION_MMP"], "3.2.1") + self.assertEqual(file_mode["PINEFORGE_VERSION_FULL"], "3.2.1") + self.assertFalse(file_mode["PINEFORGE_VERSION_FULL"].endswith("-dirty")) + self.assertEqual(file_mode["PINEFORGE_VERSION_DIRTY"], "ON") + self.assertEqual(file_mode["PINEFORGE_VERSION_GIT_SHA"], sha) + self.assertEqual(auto["PINEFORGE_VERSION_GIT_SHA"], sha) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e89815535..379b2ccaa 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -298,6 +298,11 @@ set(TEST_SOURCES find_package(Threads REQUIRED) find_package(Python3 COMPONENTS Interpreter REQUIRED) +add_test(NAME test_cmake_version_source + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/test_cmake_version_source.py) +add_test(NAME test_ci_verify + COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/test_ci_verify.py) + add_test( NAME test_aggregate_cpp_versions COMMAND ${Python3_EXECUTABLE} diff --git a/tests/fixtures/settlement_cpp_abi/README.md b/tests/fixtures/settlement_cpp_abi/README.md index d08b4fd16..77be59171 100644 --- a/tests/fixtures/settlement_cpp_abi/README.md +++ b/tests/fixtures/settlement_cpp_abi/README.md @@ -11,11 +11,11 @@ and UBSan RTTI controls. No generated pairing executable is run. ## Prepare the real old provider explicitly -After configuring the current candidate build, ensure the exact R2 Git object -is available. CI retains shallow checkouts and explicitly fetches only the pinned old commit -without tags. This preserves its existing VERSION-based release metadata; a full -history/tag fetch would activate git-describe version selection. The same fetch -works for a local shallow clone, outside CTest: +The shared `ci_verify.py` profiles use `PINEFORGE_VERSION_SOURCE=FILE`, so +verification version identity is independent of tags or checkout depth. They +fetch the pinned old commit without tags only when it is missing and reuse a +matching prepared archive on subsequent runs. For manual preparation, configure +the current build first, then fetch the old object if needed, outside CTest: ```sh git fetch --no-tags --depth=1 origin e60e57156a54bb1c74c0c0a4f5a338fd924c046d From 824d5d3a6bf95099ed9e095992480796b6f9086c Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 13 Sep 2026 07:58:31 +0800 Subject: [PATCH 4/4] Preserve resolved reversal and source FIFO endpoint intent Keep exact reversal targets and whole-opening source closes on the shared settlement owner. Extend native/source regression coverage and real historical ABI verification through the shared CI driver. --- docs/ci.md | 11 +- docs/native-settlement.md | 69 ++- include/pineforge/engine.hpp | 39 ++ include/pineforge/execution_reverse_to.hpp | 14 + scripts/check_settlement_cpp_abi.py | 156 +++++- scripts/ci_verify.py | 54 +- scripts/collect_ci_diagnostics.py | 3 + scripts/prepare_settlement_cpp_abi_base.py | 70 ++- scripts/test_ci_verify.py | 110 +++- scripts/test_settlement_cpp_abi.py | 63 ++- src/engine_execution.cpp | 251 +++++++-- src/engine_orders.cpp | 73 ++- tests/CMakeLists.txt | 7 + .../settlement_cpp_abi/0e18690/manifest.json | 277 +++++++++ tests/fixtures/settlement_cpp_abi/README.md | 50 +- tests/test_native_reverse_to.cpp | 524 ++++++++++++++++++ tests/test_source_exact_reversal.cpp | 275 +++++++++ tests/test_source_fifo_endpoints.cpp | 509 +++++++++++++++++ 18 files changed, 2377 insertions(+), 178 deletions(-) create mode 100644 include/pineforge/execution_reverse_to.hpp create mode 100644 tests/fixtures/settlement_cpp_abi/0e18690/manifest.json create mode 100644 tests/test_native_reverse_to.cpp create mode 100644 tests/test_source_exact_reversal.cpp create mode 100644 tests/test_source_fifo_endpoints.cpp diff --git a/docs/ci.md b/docs/ci.md index 7d6566b2c..c72357c32 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -53,9 +53,14 @@ separate. Git tags and checkout depth cannot change the smoke-test expectation. Ordinary CMake builds retain `AUTO`, the existing git-describe-first behavior. No release tag or VERSION value is rewritten by verification. -The verifier fetches the pinned historical ABI commit without tags only when -that object is missing. It then builds the old archive once or validates and -reuses a matching prepared receipt. CTest itself performs no network fetch. +The verifier fetches the pinned historical ABI commits `e60e571` (R2) and +`0e18690` (selected settlement, before exact reversal) without tags only when +each object is missing. It builds both full historical static libraries with +tests disabled, or validates and reuses their matching prepared receipts under +`settlement-abi-base/` and `settlement-abi-prior/`. Compiler, configuration and +version-source mismatches refuse reuse without deleting the old evidence. +Each profile needs matching providers; a Mac Release archive cannot replace +a Linux sanitizer build. CTest itself performs no network fetch. The [ABI guide](../tests/fixtures/settlement_cpp_abi/README.md) describes the actual old/new library pairs and their immutable inputs. diff --git a/docs/native-settlement.md b/docs/native-settlement.md index 772f33f42..4ea89c837 100644 --- a/docs/native-settlement.md +++ b/docs/native-settlement.md @@ -36,6 +36,73 @@ epsilon silently discards it. `Flatten` explicitly closes all lots and avoids using rounded aggregate equality to mean a whole-book close. Quantities whose changes cannot be represented are refused before settlement. +The source `execute_partial_exit_qty` adapter retains its existing `1e-10` +FIFO endpoint policy. After its existing whole-book Flatten check, it may +translate a quantity ending at an interior whole-lot prefix into one selected +Flatten. It walks physical lots in FIFO order, stops before the next lot once +the source endpoint is reached, and selects complete opening incarnations, +independently of the logical order label that supplied the close quantity. +A genuine partial lot, an unowned selected lot, or an opening with a fragment +outside the prefix keeps the original scalar Reduce path. Selected Flatten +closes the exact live quantities and all their remaining paid entry costs in +one execution; it does not reproduce historical residual-cost discards. +The native Reduce contract, entry-scoped adapters and transaction-flow requests +retain their existing behavior. This source translation requires separate +compatibility measurement; it does not establish that every observed extra +trade row has the same cause. + +## Reversal to an exact exposure + +`execution::reverse_to_v1::ReverseTo{signed_units}`, declared in +``, closes the entire opposite live book +and opens exactly `abs(signed_units)` units in the requested direction. Positive +targets open long; negative targets open short. It is a separate, call-local +resolved-execution value available to trusted C++ adapters through four protected, +nonvirtual methods: + +| Method | Result and context | +| --- | --- | +| `inspect_native_reversal_v1(reversal, fill)` | Non-mutating `SettlementInspection` of physical effects and the current ticket. | +| `project_native_reversal_v1(reversal, fill)` | Non-mutating `AccountEffectProjection`, including account effects and the next cycle without consuming it. | +| `settle_native_reversal_at_v1(reversal, fill, context)` | `Result` from settlement using the supplied `PhysicalExecutionContext` and empty lifecycle effects. | +| `settle_reversal_with_lifecycle_v1(reversal, fill, lifecycle)` | `Result` from settlement using the engine's current time, interval and preceding exit-path facts with supplied transient lifecycle effects. | + +For example, reversing a held long position of `1` unit to a short target of +`0.1` opens the exact binary64 quantity supplied as `-0.1`. The opening quantity +is never reconstructed by adding the old position to the target and subtracting +the closed position. Every existing lot closes in roster order using whole-lot +Flatten allocation, and one new lot opens at the same resolved `Fill::price`. +Close and open effects share one current ticket and the existing financial +commit owner. No selection or partial-close scope participates in this seam. + +A valid call requires a nonflat book opposite to a finite, nonzero target. +Zero or nonfinite targets return `InvalidQuantity`; a flat or same-side book +returns `InvalidCloseTarget`; malformed physical books return `InvalidBook`. +There is no valid no-effect reversal. Callers use `Flatten` to close without +opening. If finite closed and opening quantities have a nonfinite gross sum, +the call returns `UnrepresentableQuantity` before quoting fees. A tiny target +such as `0.1` against a held quantity of `1e16` remains valid even when their +floating-point sum absorbs the target: after all old lots close, the exact new +quantity is representable on its own. Existing accounting, lifecycle and +sequence preflight checks still apply before effects. + +Inspection, projection and settlement share allocation and fee calculations. +Successful `opened_units` and projected `signed_units_after` preserve the signed +target bits; the resulting book contains one lot of exactly its absolute size. +Projection includes the opening's paid cost once and peeks the fresh cycle; +settlement revalidates the live book and consumes that cycle when it opens. +Invalid projections contain no usable account quote. These values grant no +saved or replayable settlement authority. + +`Transact` continues to describe signed transaction flow, including its existing +opposite-book remainder arithmetic. Native queued requests and their action and +event algebra are unchanged; `ReverseTo` is not an `Action` or request variant. +The source F7 reversal adapter uses this seam after resolving its opening size. +Other flow callers, including F8, retain `Transact`. This opening-intent repair +does not establish that the seven additional trade rows or the EURUSD (ERA) +quantity/margin differences observed during refactor verification are repaired; +those remain separate compatibility acceptance work. + ## Costs and marked equity Every opening lot receives its paid entry commission before its entry @@ -93,7 +160,7 @@ Projection and settlement share allocation and fee logic. Realized balance adds close-row PnL in commit order, and marked equity applies each resulting lot's mark and paid cost in roster order. A projection is data, not commit authority: later settlement revalidates against the current book. Source percent sizing can consume -a close-only projection's equity before resolving one reversal transaction. +a close-only projection's equity before resolving one reversal opening quantity. ## Integration and limits diff --git a/include/pineforge/engine.hpp b/include/pineforge/engine.hpp index 68cc25414..729f246cb 100644 --- a/include/pineforge/engine.hpp +++ b/include/pineforge/engine.hpp @@ -23,6 +23,7 @@ #include "execution_close_scope.hpp" #include "execution_close_selection.hpp" #include "execution_projection.hpp" +#include "execution_reverse_to.hpp" #include "market_admission.hpp" #include "reservation_expansion.hpp" #include "order_cancellation.hpp" @@ -1871,6 +1872,24 @@ class BacktestEngine { const execution::Action& action, const execution::Fill& fill, const execution::SelectedOpeningSet& selection) const; + // Opposite-book reversal to an exact signed exposure. Unlike Transact, + // the opening is not a remainder of transaction units minus held units. + // These synchronous extensions share the settlement owner and do not + // introduce a queued native request or saved commit authority. + execution::SettlementInspection inspect_native_reversal_v1( + const execution::ReverseTo& reversal, + const execution::Fill& fill) const; + execution::AccountEffectProjection project_native_reversal_v1( + const execution::ReverseTo& reversal, + const execution::Fill& fill) const; + execution::Result settle_native_reversal_at_v1( + const execution::ReverseTo& reversal, + const execution::Fill& fill, + const execution::PhysicalExecutionContext& context); + execution::Result settle_reversal_with_lifecycle_v1( + const execution::ReverseTo& reversal, + const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle); // Native account value: realized balance plus marked physical lots minus // their remaining paid entry costs, for every fee type. No Pine sizing or // end-of-range reporting convention participates in this value. @@ -3891,6 +3910,21 @@ class BacktestEngine { execution::CloseScope book_or_opening, const execution::SelectedOpeningSet* selected); struct NativeSettlementStage; + execution::Status validate_native_settlement_book(double& held) const; + execution::Status allocate_native_settlement_closes( + NativeSettlementStage& stage, + const execution::CloseScope& book_or_opening, + double& remaining) const; + void finish_native_settlement_stage( + NativeSettlementStage& stage, const execution::Fill& fill) const; + execution::SettlementInspection inspect_native_settlement_stage( + const NativeSettlementStage& stage, const execution::Fill& fill) const; + execution::AccountEffectProjection project_native_settlement_stage( + const NativeSettlementStage& stage, const execution::Fill& fill) const; + execution::Result commit_native_settlement_stage( + NativeSettlementStage& stage, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::PhysicalExecutionContext& context); void stage_native_settlement( NativeSettlementStage& stage, const execution::Action& action, @@ -3898,6 +3932,11 @@ class BacktestEngine { execution::CloseScope book_or_opening, const execution::SelectedOpeningSet* selected, const execution::LifecycleEffects* lifecycle) const; + void stage_native_settlement( + NativeSettlementStage& stage, + const execution::ReverseTo& reversal, + const execution::Fill& fill, + const execution::LifecycleEffects* lifecycle) const; enum class PositionReductionCause { SCRIPT_ORDER, // strategy.close / close_all / market exit / reversal BRACKET_EXIT, // a strategy.exit bracket leg fill diff --git a/include/pineforge/execution_reverse_to.hpp b/include/pineforge/execution_reverse_to.hpp new file mode 100644 index 000000000..7f9d0cc70 --- /dev/null +++ b/include/pineforge/execution_reverse_to.hpp @@ -0,0 +1,14 @@ +#pragma once + +namespace pineforge::execution { +inline namespace reverse_to_v1 { + +// One resolved execution closes the opposite live book and opens this exact +// signed exposure. This is not a signed transaction flow or a queued request. +// The value is local to inspection/projection/settlement and is never retained. +struct ReverseTo { + double signed_units = 0.0; +}; + +} // inline namespace reverse_to_v1 +} // namespace pineforge::execution diff --git a/scripts/check_settlement_cpp_abi.py b/scripts/check_settlement_cpp_abi.py index dba7c3226..4dddf4767 100644 --- a/scripts/check_settlement_cpp_abi.py +++ b/scripts/check_settlement_cpp_abi.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Actual old/new settlement ABI pairings. Compile and link; NEVER run callers. -Requires a separately prepared real e60 R2 archive. No Git/network/build fallback +Requires separately prepared real e60 R2 and 0e R3 archives. No Git/network/build fallback is performed by this CTest-time checker. Existing native/script ABI guards stay separate and mandatory, including their old epoch and sanitizer RTTI controls. """ @@ -19,7 +19,7 @@ from check_aggregate_cpp_versions import clean, body from check_native_cpp_abi import HOST_EVENTS_CALLER, assembly_layout_values from prepare_settlement_cpp_abi_base import ( - BASE_COMMIT, BASE_TREE, COPY_CACHE, authenticate_headers, compiler_identity, + BASE_COMMIT, BASE_TREE, COPY_CACHE, PROVIDERS, authenticate_headers, compiler_identity, extract_tar, identity, read_cache, run, ) @@ -34,6 +34,9 @@ 'project_native_settlement_scoped_v1', 'project_native_settlement_selected_v1') NEW_PRIVATE = ('add_to_pyramid_market_with_qty_provenance', 'sequential_same_tick_reversal_fill_with_qty_provenance') +REVERSAL_METHODS = ('inspect_native_reversal_v1', 'project_native_reversal_v1', + 'settle_native_reversal_at_v1', 'settle_reversal_with_lifecycle_v1') +REVERSAL_DOMAIN = 'reverse_to_v1::ReverseTo' PRESERVED_ARCHIVE_SHA = 'e13d3d19ad4613c28beddfabb119f1dddadb2c39e474edd6304a2f75c7321f60' PRESERVED_HEADERS_SHA = '1001102a496ae927ae98e111dd7dc68ab6c23ecc41a9eba00995144d9a532109' FROZEN_NATIVE_HEADERS = ('native_order.hpp', 'native_order_identity.hpp', 'native_host.hpp', @@ -54,8 +57,10 @@ using R = ex::Result; using L = ex::LifecycleEffects; static_assert(std::is_same_v); +static_assert(std::variant_size_v == 3); static_assert(std::variant_size_v == 2); static_assert(std::variant_size_v == 16); +static_assert(std::variant_size_v == 3); ''' OLD_CALLER = COMMON + ''' static_assert(std::is_same_v); @@ -102,6 +107,24 @@ return int(i.closed_units+r.closed_units+l.closed_units+p0.realized_balance+p1.remaining_entry_cost+p2.marked_equity); } ''' +REVERSAL_CALLER = COMMON + ''' +using RT = ex::reverse_to_v1::ReverseTo; +using P = ex::AccountEffectProjection; +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +static_assert(std::is_same_v); +int main(int argc, char** argv) { + auto* e = reinterpret_cast(argv); + RT target{0.1}; F f{100,"","",1}; C c{}; L life{}; + auto i = e->inspect_native_reversal_v1(target,f); + auto p = e->project_native_reversal_v1(target,f); + auto r = e->settle_native_reversal_at_v1(target,f,c); + auto l = e->settle_reversal_with_lifecycle_v1(target,f,life); + return int(i.closed_units+p.realized_balance+r.closed_units+l.closed_units); +} +''' PRIVATE_OLD_CALLER = COMMON + ''' using Add = void(E::*)(const std::string&,bool,double,double,int,pineforge::PositionSide,bool,uint64_t); using Seq = void(E::*)(const std::string&,bool,double,double,int,uint64_t); @@ -177,9 +200,12 @@ def storage_declarations(header: str) -> list[str]: return statements -def frozen_shape(old_include: Path, current_include: Path) -> tuple[list[str], dict]: +def frozen_shape(old_include: Path, current_include: Path, *, selected=False) -> tuple[list[str], dict]: old_exec = (old_include/'pineforge/execution.hpp').read_text() cur_exec = (current_include/'pineforge/execution.hpp').read_text() + actions = [re.search(r'using\s+Action\s*=\s*[^;]+;', clean(text)) for text in (old_exec,cur_exec)] + if any(alias is None for alias in actions) or normalized(actions[0].group()) != normalized(actions[1].group()): + raise RuntimeError('Action alternative identities/order changed') for name in ['Result','SettlementInspection','Status','Fill','PhysicalExecutionContext','LifecycleEffects']: pattern = r'(?:struct|enum\s+class)\s+' + name + r'\s*\{' if normalized(body(old_exec, pattern, name)) != normalized(body(cur_exec, pattern, name)): @@ -187,6 +213,10 @@ def frozen_shape(old_include: Path, current_include: Path) -> tuple[list[str], d for name in FROZEN_NATIVE_HEADERS: if normalized((old_include/'pineforge'/name).read_text()) != normalized((current_include/'pineforge'/name).read_text()): raise RuntimeError('R3 must preserve native header layout/contracts: ' + name) + if selected: + for name in ('execution_close_selection.hpp', 'execution_projection.hpp'): + if normalized((old_include/'pineforge'/name).read_text()) != normalized((current_include/'pineforge'/name).read_text()): + raise RuntimeError('selected/projection header layout/contracts changed: ' + name) scopes = [(directory/'pineforge/execution_close_scope.hpp').read_text() for directory in (old_include,current_include)] for name in ['Book','OpeningExposure']: @@ -209,7 +239,7 @@ def frozen_shape(old_include: Path, current_include: Path) -> tuple[list[str], d return members, {'engineStorage': old_storage, 'virtuals': [normalized(v) for v in virtuals(old_engine)]} -def layout_source(members: list[str]) -> tuple[str, int]: +def layout_source(members: list[str], *, selected=False) -> tuple[str, int]: values = [] assertions = [] for name in ['Result','SettlementInspection']: @@ -220,12 +250,16 @@ def layout_source(members: list[str]) -> tuple[str, int]: for index, status in enumerate(['Applied','NoEffect','InvalidPrice','InvalidQuantity','InvalidBook', 'UnrepresentableQuantity','InvalidAccounting','InvalidLifecycle','InvalidCloseTarget']): assertions.append(f'static_assert(int(ex::Status::{status})=={index});') - values += ['sizeof(ex::Status)','sizeof(S)','alignof(S)','std::variant_size_v', + values += ['sizeof(ex::Status)','sizeof(A)','alignof(A)','std::variant_size_v', + 'sizeof(S)','alignof(S)','std::variant_size_v', 'sizeof(E)','alignof(E)','sizeof(pineforge::PendingOrder)', 'sizeof(pineforge::NativeStrategyHost)','sizeof(pineforge::NativeMarketEvent)', 'sizeof(pineforge::NativeStateView)','sizeof(pineforge::native_order::Request)', 'sizeof(pineforge::native_order::WorkingRequestCore)', 'sizeof(pineforge::native_order::CommandEvent)'] + if selected: + values += ['sizeof(ex::SelectedOpeningSet)', 'alignof(ex::SelectedOpeningSet)', + 'sizeof(ex::AccountEffectProjection)', 'alignof(ex::AccountEffectProjection)'] for member in members: values += [f'offsetof(E,{member})', f'sizeof(decltype(E::{member}))', f'alignof(decltype(E::{member}))'] return COMMON + '\n'.join(assertions) + '\nextern "C" const unsigned long long abi_layout[] = {\n' + ',\n'.join(values) + '\n};\n', len(values) @@ -246,8 +280,8 @@ def validate_rejection(diagnostic: str, missing, domain=None) -> list[str]: matching = [symbol for symbol in symbols if symbol.startswith(ENGINE+method+'(')] if not matching: raise RuntimeError('link failure omits expected undefined method: '+method) - if domain and 'selected' in method and any(domain not in symbol for symbol in matching): - raise RuntimeError('selected method has wrong/missing parameter namespace: '+method) + if domain and ('selected' in method or 'reversal' in method) and any(domain not in symbol for symbol in matching): + raise RuntimeError('method has wrong/missing parameter namespace: '+method) unrelated = [symbol for symbol in symbols if not any(symbol.startswith(ENGINE+method+'(') for method in missing)] if unrelated: raise RuntimeError('link failure includes unrelated undefined symbols: '+', '.join(unrelated)) @@ -255,24 +289,44 @@ def validate_rejection(diagnostic: str, missing, domain=None) -> list[str]: def load_base(args, destination: Path, current_cache: dict) -> tuple[Path, Path, Path, dict]: - if not args.base_receipt.is_file(): + return load_provider(args, destination, current_cache, args.base_receipt, PROVIDERS['e60'], + expect_present=(*OLD_METHODS,*OLD_PRIVATE), + expect_absent=(*NEW_METHODS,*NEW_PRIVATE,*REVERSAL_METHODS)) + + +def load_prior(args, destination: Path, current_cache: dict) -> tuple[Path, Path, Path, dict]: + return load_provider(args, destination, current_cache, args.prior_receipt, PROVIDERS['0e'], + expect_present=(*OLD_METHODS,*OLD_PRIVATE,*NEW_METHODS,*NEW_PRIVATE), + expect_absent=REVERSAL_METHODS) + + +def load_provider(args, destination: Path, current_cache: dict, receipt_path: Path | None, + provider: dict, *, expect_present, expect_absent) -> tuple[Path, Path, Path, dict]: + is_base = provider['commit'] == BASE_COMMIT + label = 'R2' if is_base else '0e18690' + if receipt_path is None or not receipt_path.is_file(): + if not is_base: + raise RuntimeError('real 0e18690 archive receipt missing; run scripts/prepare_settlement_cpp_abi_base.py ' + '--source-repo . --current-build BUILD --output BUILD/settlement-abi-prior ' + f'--commit {provider["commit"]} --tree {provider["tree"]} ' + '--header-manifest tests/fixtures/settlement_cpp_abi/0e18690/manifest.json before CTest') raise RuntimeError('real R2 archive receipt missing; run scripts/prepare_settlement_cpp_abi_base.py ' '--source-repo . --current-build BUILD --output BUILD/settlement-abi-base before CTest') - receipt = json.loads(args.base_receipt.read_text()) - if receipt.get('commit') != BASE_COMMIT or receipt.get('tree') != BASE_TREE: - raise RuntimeError('real base receipt does not pin e60 R2') - base_root = args.base_receipt.resolve().parent + receipt = json.loads(receipt_path.read_text()) + if receipt.get('commit') != provider['commit'] or receipt.get('tree') != provider['tree']: + raise RuntimeError('real base receipt does not pin ' + ('e60 R2' if is_base else label)) + base_root = receipt_path.resolve().parent resolve_artifact = lambda name: Path(name) if Path(name).is_absolute() else base_root/name library = resolve_artifact(receipt['archive']) - headers = resolve_artifact(receipt.get('headers','r2-headers.tar')) + headers = resolve_artifact(receipt.get('headers',provider['headers_name'])) if identity(library)['sha256'] != receipt['archiveSha256'] or identity(headers)['sha256'] != receipt['headersSha256']: - raise RuntimeError('real R2 archive/header bytes do not match receipt') + raise RuntimeError('real ' + label + ' archive/header bytes do not match receipt') if not library.read_bytes().startswith(b'!\n'): raise RuntimeError('real base is not a static archive') if len(run(['ar','-t',str(library)]).splitlines()) < 20: raise RuntimeError('real base is not a full product archive') extract_tar(headers.read_bytes(), destination) - authenticate_headers(destination) + authenticate_headers(destination, provider['manifest'], commit=provider['commit'], tree=provider['tree']) if receipt.get('schemaVersion') == 'pineforge-settlement-abi-base/v1': old_compiler = receipt['compiler'] current_compiler = compiler_identity(args.compiler) @@ -285,6 +339,8 @@ def load_base(args, destination: Path, current_cache: dict) -> tuple[Path, Path, if identity(generated/'pineforge/version.h')['sha256'] != receipt['generatedHeaderSha256']: raise RuntimeError('base generated version header changed') else: + if not is_base: + raise RuntimeError('0e18690 provider requires portable v1 receipt; use matching preparation') # Explicit reuse of root's preserved Mac Release artifact, never a stub. if receipt['archiveSha256'] != PRESERVED_ARCHIVE_SHA or receipt['headersSha256'] != PRESERVED_HEADERS_SHA: raise RuntimeError('unrecognized legacy base receipt; use portable preparation') @@ -302,10 +358,10 @@ def load_base(args, destination: Path, current_cache: dict) -> tuple[Path, Path, if identity(generated.parent/'lib/libpineforge.a')['sha256'] != receipt['archiveSha256']: raise RuntimeError('preserved base cache/header directory no longer belongs to its archive') symbols = defined_symbols(library) - for method in (*OLD_METHODS,*OLD_PRIVATE): + for method in expect_present: if ENGINE + method + '(' not in symbols: raise RuntimeError('real old archive omits original symbol: ' + method) - for method in (*NEW_METHODS,*NEW_PRIVATE): + for method in expect_absent: if ENGINE + method + '(' in symbols: raise RuntimeError('supplied old archive already exports new method: ' + method) return library, destination/'include', generated, receipt @@ -318,6 +374,8 @@ def main() -> int: parser.add_argument('--include', type=Path, required=True) parser.add_argument('--generated-include', type=Path, required=True) parser.add_argument('--base-receipt', type=Path, required=True) + parser.add_argument('--prior-receipt', type=Path, + help='required for full proof: prepared real 0e18690 provider receipt') parser.add_argument('--base-generated-include', type=Path) parser.add_argument('--extra-flag', action='append', default=[]) parser.add_argument('--receipt', type=Path, required=True) @@ -338,6 +396,7 @@ def main() -> int: stale = [str(path.relative_to(source)) for path in source_files if path.stat().st_mtime > library.stat().st_mtime] if sum((args.base_only,args.old_rejections_only,args.public_only)) > 1: raise RuntimeError('select only one partial proof mode') + full_matrix = not (args.base_only or args.old_rejections_only or args.public_only) if stale and not args.old_rejections_only: raise RuntimeError('current archive predates source; full rebuild required: '+', '.join(stale)) initial_identity = identity(library) @@ -365,6 +424,13 @@ def main() -> int: 'commit':BASE_COMMIT,'tree':BASE_TREE} members,shape = frozen_shape(old_include,include) report['frozenShape']=shape + if full_matrix: + prior_library,prior_include,prior_generated,prior_receipt = load_prior(args,scratch/'prior',cache) + report['prior']={'receiptSha256':identity(args.prior_receipt)['sha256'], + 'archiveSha256':identity(prior_library)['sha256'], + 'commit':prior_receipt['commit'],'tree':prior_receipt['tree']} + prior_members,prior_shape = frozen_shape(prior_include,include,selected=True) + report['priorFrozenShape']=prior_shape def compile_tu(name,text,headers,generated): path=log_root/(name+'.cpp');path.write_text(text) @@ -386,7 +452,21 @@ def link(name,obj,runtime,missing=(),domain=None): validate_rejection(diagnostic,missing,domain) report['links'].append({'name':name,'argv':argv,'exitCode':result.returncode, 'outcome':'expected-rejection' if missing else 'linked','requiredMissing':list(missing), - 'selectionDomain':domain,'executed':False}) + 'selectionDomain':domain,'parameterDomain':domain,'executed':False}) + + def compare_layout(name,headers,generated,layout_members,*,selected=False): + layout_text,word_count=layout_source(layout_members,selected=selected) + layouts=[] + for label,layout_headers,layout_generated in [(name,headers,generated), + ('current' if name == 'old' else 'current-selected',include,args.generated_include)]: + compile_tu(label+'-layout',layout_text,layout_headers,layout_generated) + src=log_root/(label+'-layout.cpp');asm=log_root/(label+'-layout.s') + run([*common,'-I',str(layout_headers),'-I',str(layout_generated),'-S',str(src),'-o',str(asm)],timeout=120, + log=log_root/(label+'-layout.assembly.log')) + layouts.append(assembly_layout_values(asm.read_text(),word_count)) + if layouts[0]!=layouts[1]: + raise RuntimeError('actual compiler '+name+'/current layout/offset/type-size arrays differ') + return {'wordCount':word_count,'values':layouts[0],'members':layout_members} # Compile every actual caller before interpreting any link outcome. old=compile_tu('old-book-singleton',OLD_CALLER,old_include,old_generated) @@ -395,19 +475,12 @@ def link(name,obj,runtime,missing=(),domain=None): current_private_old=compile_tu('current-old-private-types',PRIVATE_OLD_CALLER,include,args.generated_include) old_events=compile_tu('old-host-events-return',HOST_EVENTS_CALLER,old_include,old_generated) current_events=compile_tu('current-host-events-return',HOST_EVENTS_CALLER,include,args.generated_include) - layout_text,word_count=layout_source(members) - layouts=[] - for label,headers,generated in [('old',old_include,old_generated),('current',include,args.generated_include)]: - compile_tu(label+'-layout',layout_text,headers,generated) - src=log_root/(label+'-layout.cpp');asm=log_root/(label+'-layout.s') - run([*common,'-I',str(headers),'-I',str(generated),'-S',str(src),'-o',str(asm)],timeout=120, - log=log_root/(label+'-layout.assembly.log')) - layouts.append(assembly_layout_values(asm.read_text(),word_count)) - if layouts[0]!=layouts[1]: raise RuntimeError('actual compiler old/current layout/offset/type-size arrays differ') - report['layout']={'wordCount':word_count,'values':layouts[0],'members':members} + report['layout']=compare_layout('old',old_include,old_generated,members) + if full_matrix: + report['priorLayout']=compare_layout('prior',prior_include,prior_generated,prior_members,selected=True) if not args.base_only: current_header=clean((include/'pineforge/engine.hpp').read_text()) - for method in NEW_METHODS: + for method in (*NEW_METHODS,*(REVERSAL_METHODS if full_matrix else ())): match=re.search(r'[^;{}]*\b'+method+r'\s*\([^;{}]*;',current_header) if not match or re.search(r'\bvirtual\b',match.group()): raise RuntimeError('new method must have one nonvirtual declaration: '+method) @@ -435,6 +508,23 @@ def link(name,obj,runtime,missing=(),domain=None): if require_name not in changed: raise RuntimeError('selection namespace missing from real header') header.write_text(changed.replace(require_name,'close_selection_v2')) wrong_set=compile_tu('synthetic-selection-v2',NEW_CALLER.replace(require_name,'close_selection_v2'),wrong_selection,args.generated_include) + if full_matrix: + prior_selected=compile_tu('prior-selected-project',NEW_CALLER,prior_include,prior_generated) + reversal=compile_tu('new-four-reversal-methods',REVERSAL_CALLER,include,args.generated_include) + wrong_reversal=scratch/'wrong-reversal';shutil.copytree(include,wrong_reversal) + header=wrong_reversal/'pineforge/engine.hpp';changed=header.read_text();wrong_source=REVERSAL_CALLER + wrong_reversal_names=[] + for method in REVERSAL_METHODS: + future=method[:-2]+'v2';wrong_reversal_names.append(future) + changed=changed.replace(method,future);wrong_source=wrong_source.replace(method,future) + header.write_text(changed) + wrong_reversal_methods=compile_tu('synthetic-reversal-method-v2',wrong_source,wrong_reversal,args.generated_include) + wrong_target=scratch/'wrong-target';shutil.copytree(include,wrong_target) + header=wrong_target/'pineforge/execution_reverse_to.hpp';changed=header.read_text() + if 'reverse_to_v1' not in changed: + raise RuntimeError('reversal namespace missing from real header') + header.write_text(changed.replace('reverse_to_v1','reverse_to_v2')) + wrong_reverse_to=compile_tu('synthetic-reverse-to-v2',REVERSAL_CALLER.replace('reverse_to_v1','reverse_to_v2'),wrong_target,args.generated_include) link('old-api-old-real',old,old_library) link('old-private-old-real',private_old,old_library) link('old-events-old-real',old_events,old_library) @@ -455,6 +545,14 @@ def link(name,obj,runtime,missing=(),domain=None): link('synthetic-project-v2-rejected',wrong_projection,library,wrong_names) link('synthetic-selection-v2-rejected',wrong_set,library, (NEW_METHODS[0],NEW_METHODS[1],NEW_METHODS[2],NEW_METHODS[5]),'close_selection_v2::SelectedOpeningSet') + if full_matrix: + link('prior-selected-prior-real',prior_selected,prior_library) + link('prior-selected-new-real',prior_selected,library) + link('current-selected-prior-real',new,prior_library) + link('new-reversal-new-real',reversal,library) + link('new-reversal-prior-real-rejected',reversal,prior_library,REVERSAL_METHODS,REVERSAL_DOMAIN) + link('synthetic-reversal-method-v2-rejected',wrong_reversal_methods,library,wrong_reversal_names,REVERSAL_DOMAIN) + link('synthetic-reverse-to-v2-rejected',wrong_reverse_to,library,REVERSAL_METHODS,'reverse_to_v2::ReverseTo') if identity(library)!=initial_identity: raise RuntimeError('current archive changed during ABI proof') if any(identity(source/name)['sha256']!=sha for name,sha in report['sourceSha256'].items()): raise RuntimeError('source changed during ABI proof') diff --git a/scripts/ci_verify.py b/scripts/ci_verify.py index d7f89d9e5..7dbe0d417 100644 --- a/scripts/ci_verify.py +++ b/scripts/ci_verify.py @@ -2,7 +2,7 @@ """Shared local/CI verification driver. Stdlib only. Not a command generator. Profiles: release, debug, sanitizers, native. Default build dir build-ci-PROFILE. -Source guards, explicit configure, full rebuild, pinned R2 ABI prepare/reuse, +Source guards, explicit configure, full rebuild, pinned e60 and 0e ABI prepare/reuse, CTest, install+find_package+VERSION smoke, native help / required WebSocket. Fail fast on configure/build. After a successful build collect independent CTest and package failures in the same run. Never deletes source, tests, or @@ -24,6 +24,8 @@ from prepare_settlement_cpp_abi_base import ( BASE_COMMIT, + BASE_TREE, + PROVIDERS, read_cache, reusable_prepared_base, ) @@ -297,9 +299,9 @@ def smoke_prefix(cache: dict[str, str], install_prefix: Path) -> str: return ';'.join(parts) -def pinned_object_present(source: Path, runner: Runner) -> bool: +def pinned_object_present(source: Path, runner: Runner, commit: str = BASE_COMMIT) -> bool: result = call_runner( - runner, ['git', '-C', str(source), 'cat-file', '-e', BASE_COMMIT + '^{commit}'], + runner, ['git', '-C', str(source), 'cat-file', '-e', commit + '^{commit}'], timeout=30, stream_output=False) return result.returncode == 0 @@ -331,6 +333,7 @@ def __init__(self, cfg: VerifyConfig): self.expected_version = expected_version(cfg.source) self.actual_version: str | None = None self.abi_action = 'not-started' + self.abi_prior_action = 'not-started' self.summary: dict = { 'schemaVersion': SCHEMA, 'status': 'incomplete', @@ -350,12 +353,14 @@ def __init__(self, cfg: VerifyConfig): 'actualVersion': None, 'exitCode': None, 'abi': {'action': self.abi_action}, + 'abiPrior': {'action': self.abi_prior_action}, 'stages': self.stages, 'failures': self.failures, } def write_summary(self) -> None: self.summary['abi'] = {'action': self.abi_action} + self.summary['abiPrior'] = {'action': self.abi_prior_action} self.summary['actualVersion'] = self.actual_version self.summary['stages'] = self.stages self.summary['failures'] = self.failures @@ -486,37 +491,49 @@ def verify_configured_profile(self, cache: dict[str, str]) -> str | None: return None def ensure_abi_base(self) -> None: - output = self.cfg.build_dir / 'settlement-abi-base' + self.abi_action = self.ensure_prepared_provider( + self.cfg.build_dir / 'settlement-abi-base', BASE_COMMIT, BASE_TREE, + extra_argv=[], stage='abi-base', fetch_stage='abi-fetch') + + def ensure_abi_prior(self) -> None: + provider = PROVIDERS['0e'] + manifest = self.cfg.source / provider['manifest'].relative_to(ROOT) + self.abi_prior_action = self.ensure_prepared_provider( + self.cfg.build_dir / provider['default_output'], provider['commit'], provider['tree'], + extra_argv=['--commit', provider['commit'], '--tree', provider['tree'], + '--header-manifest', str(manifest)], + stage='abi-prior', fetch_stage='abi-prior-fetch') + + def ensure_prepared_provider(self, output: Path, commit: str, tree: str, *, + extra_argv: list[str], stage: str, fetch_stage: str) -> str: prepare = [ sys.executable, str(self.cfg.source / 'scripts/prepare_settlement_cpp_abi_base.py'), '--source-repo', str(self.cfg.source), '--current-build', str(self.cfg.build_dir), '--output', str(output), '--jobs', str(self.cfg.jobs), + *extra_argv, ] if output.exists(): try: - receipt = reusable_prepared_base(output, self.cfg.build_dir) + receipt = reusable_prepared_base(output, self.cfg.build_dir, commit=commit, tree=tree) except Exception as error: - self.abi_action = 'refused' - self.fail_stage('abi-base', str(error), argv=['reuse-matching-receipt', str(output)]) - return - self.abi_action = 'reused' + self.fail_stage(stage, str(error), argv=['reuse-matching-receipt', str(output)]) + return 'refused' self.pass_stage( - 'abi-base', + stage, json.dumps({'action': 'reused', 'receipt': str(output / 'receipt.json'), 'archiveSha256': receipt.get('archiveSha256')}, indent=2, sort_keys=True), argv=['reuse-matching-receipt', str(output / 'receipt.json')]) - return - if not pinned_object_present(self.cfg.source, self.cfg.runner): + return 'reused' + if not pinned_object_present(self.cfg.source, self.cfg.runner, commit): fetch = ['git', '-C', str(self.cfg.source), 'fetch', '--no-tags', '--depth=1', - 'origin', BASE_COMMIT] - fetched = self.invoke('abi-fetch', fetch, timeout=120) + 'origin', commit] + fetched = self.invoke(fetch_stage, fetch, timeout=120) if fetched.returncode != 0: - self.abi_action = 'failed' - return - prepared = self.invoke('abi-base', prepare, timeout=1800) - self.abi_action = 'prepared' if prepared.returncode == 0 else 'failed' + return 'failed' + prepared = self.invoke(stage, prepare, timeout=1800) + return 'prepared' if prepared.returncode == 0 else 'failed' def run_smoke(self, cache: dict[str, str]) -> None: prefix = smoke_prefix(cache, self.install_prefix) @@ -631,6 +648,7 @@ def run(self) -> int: self.pass_stage('native-binary', 'live runner absent as required for this profile') self.ensure_abi_base() + self.ensure_abi_prior() ctest = ['ctest', '--test-dir', str(self.cfg.build_dir), '--output-on-failure', '--parallel', str(self.cfg.jobs)] diff --git a/scripts/collect_ci_diagnostics.py b/scripts/collect_ci_diagnostics.py index 4670d5343..4cfe2f604 100644 --- a/scripts/collect_ci_diagnostics.py +++ b/scripts/collect_ci_diagnostics.py @@ -16,6 +16,9 @@ "settlement-abi-base/receipt.json": "settlement-abi-base-receipt.json", "settlement-abi-base/configure.log": "settlement-abi-base-configure.log", "settlement-abi-base/build.log": "settlement-abi-base-build.log", + "settlement-abi-prior/receipt.json": "settlement-abi-prior-receipt.json", + "settlement-abi-prior/configure.log": "settlement-abi-prior-configure.log", + "settlement-abi-prior/build.log": "settlement-abi-prior-build.log", "settlement-abi-receipt.json": "settlement-abi-receipt.json", } ENV_KEYS = ("GITHUB_EVENT_NAME", "GITHUB_REF", "GITHUB_SHA", "GITHUB_RUN_ID", diff --git a/scripts/prepare_settlement_cpp_abi_base.py b/scripts/prepare_settlement_cpp_abi_base.py index 2ce3c5ab3..cb97a9ad1 100644 --- a/scripts/prepare_settlement_cpp_abi_base.py +++ b/scripts/prepare_settlement_cpp_abi_base.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Build the exact R2 settlement ABI provider from local Git objects, without tests. +"""Build an exact historical settlement ABI provider from local Git objects. Run explicitly before CTest. This script never fetches Git history or dependencies. The exact Git object and Eigen must already be available to the current build. @@ -21,6 +21,15 @@ BASE_TREE = 'e209c89035ec884774f8f3e68e1dc118f26aaf51' ROOT = Path(__file__).resolve().parents[1] HEADER_MANIFEST = ROOT / 'tests/fixtures/settlement_cpp_abi/e60e571/manifest.json' +PRIOR_COMMIT = '0e18690db3fb6bb4705841c2a86936dfa206380a' +PRIOR_TREE = 'c29545d262bf6dcc22e46aa3382422f2e9c68378' +PRIOR_HEADER_MANIFEST = ROOT / 'tests/fixtures/settlement_cpp_abi/0e18690/manifest.json' +PROVIDERS = { + 'e60': {'commit': BASE_COMMIT, 'tree': BASE_TREE, 'manifest': HEADER_MANIFEST, + 'default_output': 'settlement-abi-base', 'headers_name': 'r2-headers.tar'}, + '0e': {'commit': PRIOR_COMMIT, 'tree': PRIOR_TREE, 'manifest': PRIOR_HEADER_MANIFEST, + 'default_output': 'settlement-abi-prior', 'headers_name': 'headers.tar'}, +} COPY_CACHE = ( 'CMAKE_BUILD_TYPE', 'CMAKE_C_COMPILER', 'CMAKE_CXX_COMPILER', 'CMAKE_C_COMPILER_TARGET', 'CMAKE_CXX_COMPILER_TARGET', @@ -63,12 +72,14 @@ def refuse_existing_base(output: Path, reason: str) -> None: raise RuntimeError(f'{reason}; use a fresh --build-dir (refusing to delete {output})') -def receipt_matches_current(receipt: dict, current_cache: dict, compiler: dict) -> None: +def receipt_matches_current(receipt: dict, current_cache: dict, compiler: dict, *, + commit: str = BASE_COMMIT, tree: str = BASE_TREE) -> None: """Raise if a prepared base must not be reused with this current build.""" if receipt.get('schemaVersion') != 'pineforge-settlement-abi-base/v1': raise RuntimeError('existing ABI base receipt is not portable v1') - if receipt.get('commit') != BASE_COMMIT or receipt.get('tree') != BASE_TREE: - raise RuntimeError('existing ABI base does not pin e60 R2') + if receipt.get('commit') != commit or receipt.get('tree') != tree: + label = 'e60 R2' if commit == BASE_COMMIT else commit + raise RuntimeError('existing ABI base does not pin ' + label) old = receipt.get('compiler') or {} for key in ('target', 'sha256', 'version'): if old.get(key) != compiler.get(key): @@ -78,7 +89,8 @@ def receipt_matches_current(receipt: dict, current_cache: dict, compiler: dict) 'existing ABI base compiler/configuration/version-source/launcher differs from this build') -def reusable_prepared_base(output: Path, current_build: Path) -> dict: +def reusable_prepared_base(output: Path, current_build: Path, *, + commit: str = BASE_COMMIT, tree: str = BASE_TREE) -> dict: """Return an existing receipt that is safe to reuse. Never deletes output.""" receipt_path = output / 'receipt.json' if not output.exists(): @@ -89,7 +101,7 @@ def reusable_prepared_base(output: Path, current_build: Path) -> dict: cache = read_cache(current_build / 'CMakeCache.txt') compiler = compiler_identity(cache['CMAKE_CXX_COMPILER']) try: - receipt_matches_current(receipt, cache, compiler) + receipt_matches_current(receipt, cache, compiler, commit=commit, tree=tree) except RuntimeError as error: refuse_existing_base(output, str(error)) resolve = lambda name: Path(name) if Path(name).is_absolute() else output / name @@ -156,18 +168,20 @@ def extract_tar(raw: bytes, destination: Path) -> None: target.chmod(member.mode & 0o777) -def authenticate_headers(directory: Path) -> dict: - manifest = json.loads(HEADER_MANIFEST.read_text()) - if manifest['commit'] != BASE_COMMIT or manifest['tree'] != BASE_TREE: - raise RuntimeError('tracked R2 header pin changed') +def authenticate_headers(directory: Path, manifest_path: Path = HEADER_MANIFEST, *, + commit: str = BASE_COMMIT, tree: str = BASE_TREE) -> dict: + manifest = json.loads(manifest_path.read_text()) + label = 'R2' if commit == BASE_COMMIT else commit[:7] + if manifest['commit'] != commit or manifest['tree'] != tree: + raise RuntimeError('tracked ' + label + ' header pin changed') actual = {str(path.relative_to(directory)) for path in (directory / 'include').rglob('*') if path.is_file()} if actual != set(manifest['files']): - raise RuntimeError('R2 header closure differs from the tracked exact file inventory') + raise RuntimeError(label + ' header closure differs from the tracked exact file inventory') for name, expected in manifest['files'].items(): raw = (directory / name).read_bytes() blob = hashlib.sha1(b'blob ' + str(len(raw)).encode() + b'\0' + raw).hexdigest() if identity(directory / name) != {'sha256': expected['sha256'], 'bytes': expected['bytes']} or blob != expected['git_blob']: - raise RuntimeError('R2 header bytes differ: ' + name) + raise RuntimeError(label + ' header bytes differ: ' + name) return manifest @@ -189,7 +203,15 @@ def main() -> int: parser.add_argument('--current-build', type=Path, required=True) parser.add_argument('--output', type=Path, required=True) parser.add_argument('--jobs', type=int, default=4) + parser.add_argument('--commit', default=BASE_COMMIT) + parser.add_argument('--tree', default=BASE_TREE) + parser.add_argument('--header-manifest', type=Path, default=HEADER_MANIFEST) args = parser.parse_args() + provider = next((pin for pin in PROVIDERS.values() + if (pin['commit'], pin['tree']) == (args.commit, args.tree)), None) + if provider is None: + raise RuntimeError('unsupported historical ABI commit/tree pin') + label = 'R2' if args.commit == BASE_COMMIT else args.commit[:7] repo, current, output = args.source_repo.resolve(), args.current_build.resolve(), args.output.resolve() if output.exists(): raise RuntimeError('output must be new; reuse its existing receipt explicitly instead of rebuilding over it') @@ -199,20 +221,20 @@ def main() -> int: if Path(cache['CMAKE_HOME_DIRECTORY']).resolve() != repo: raise RuntimeError('current CMake cache belongs to a different source repository') try: - commit = run(['git', '-C', str(repo), 'rev-parse', BASE_COMMIT + '^{commit}']).decode().strip() - tree = run(['git', '-C', str(repo), 'rev-parse', BASE_COMMIT + '^{tree}']).decode().strip() + commit = run(['git', '-C', str(repo), 'rev-parse', args.commit + '^{commit}']).decode().strip() + tree = run(['git', '-C', str(repo), 'rev-parse', args.commit + '^{tree}']).decode().strip() except RuntimeError as error: - raise RuntimeError(f'exact R2 source is unavailable; fetch the pinned object without tags ' - f'before preparation: git fetch --no-tags --depth=1 origin {BASE_COMMIT}') from error - if commit != BASE_COMMIT or tree != BASE_TREE: - raise RuntimeError('R2 commit/tree mismatch') - source_raw = run(['git', '-C', str(repo), 'archive', '--format=tar', BASE_COMMIT, + raise RuntimeError(f'exact {label} source is unavailable; fetch the pinned object without tags ' + f'before preparation: git fetch --no-tags --depth=1 origin {args.commit}') from error + if commit != args.commit or tree != args.tree: + raise RuntimeError(label + ' commit/tree mismatch') + source_raw = run(['git', '-C', str(repo), 'archive', '--format=tar', args.commit, 'CMakeLists.txt', 'VERSION', 'cmake', 'include', 'src']) output.mkdir(parents=True) (output / 'source.tar').write_bytes(source_raw) source, build = output / 'source', output / 'build' extract_tar(source_raw, source) - authenticate_headers(source) + authenticate_headers(source, args.header_manifest, commit=args.commit, tree=args.tree) settings = copied_cache(cache) eigen_source = current_eigen_source(cache, current) if eigen_source is not None: @@ -231,15 +253,15 @@ def main() -> int: raise RuntimeError('base library is not a full static archive') archive_members = run(['ar', '-t', str(library)]).decode().splitlines() if len(archive_members) < 20: - raise RuntimeError('base archive is not the full R2 library') - headers = output / 'r2-headers.tar' + raise RuntimeError('base archive is not the full ' + label + ' library') + headers = output / provider['headers_name'] with tarfile.open(headers, 'w') as archive: archive.add(source / 'include', arcname='include', recursive=True) source_identity = {str(path.relative_to(source)): identity(path)['sha256'] for path in sorted(source.rglob('*')) if path.is_file()} - receipt = {'schemaVersion': 'pineforge-settlement-abi-base/v1', 'commit': BASE_COMMIT, 'tree': BASE_TREE, + receipt = {'schemaVersion': 'pineforge-settlement-abi-base/v1', 'commit': args.commit, 'tree': args.tree, 'archive': str(library.relative_to(output)), 'archiveSha256': identity(library)['sha256'], - 'headers': 'r2-headers.tar', 'headersSha256': identity(headers)['sha256'], + 'headers': headers.name, 'headersSha256': identity(headers)['sha256'], 'generatedInclude': 'build/include', 'generatedHeaderSha256': identity(build / 'include/pineforge/version.h')['sha256'], 'sourceSha256': source_identity, 'sourceArchive': identity(output / 'source.tar'), diff --git a/scripts/test_ci_verify.py b/scripts/test_ci_verify.py index 91b481853..0046764a8 100644 --- a/scripts/test_ci_verify.py +++ b/scripts/test_ci_verify.py @@ -36,6 +36,8 @@ from prepare_settlement_cpp_abi_base import ( BASE_COMMIT, BASE_TREE, + PRIOR_COMMIT, + PRIOR_TREE, COPY_CACHE, compiler_identity, copied_cache, @@ -89,14 +91,17 @@ def __call__(self, argv, *, extra_env=None, timeout=600, combine_stderr=True, return default_runner(argv, extra_env=None, timeout=timeout, combine_stderr=True, stream_output=False) if argv[0] == 'git' and 'cat-file' in argv: - if 'cat-file' in self.exits: - return Completed(int(self.exits['cat-file']), b'', b'') + key = 'prior-cat-file' if PRIOR_COMMIT + '^{commit}' in argv else 'cat-file' + if key in self.exits: + return Completed(int(self.exits[key]), b'', b'') return default_runner(argv, extra_env=None, timeout=timeout, combine_stderr=True, stream_output=False) if argv[0] == 'git' and 'fetch' in argv: - return Completed(int(self.exits.get('fetch', 0)), b'fetched\n', b'') + key = 'prior-fetch' if PRIOR_COMMIT in argv else 'fetch' + return Completed(int(self.exits.get(key, 0)), b'fetched\n', b'') if any(Path(part).name == 'prepare_settlement_cpp_abi_base.py' for part in argv): - return Completed(int(self.exits.get('prepare', 0)), b'prepared\n', b'') + key = 'prior-prepare' if PRIOR_COMMIT in argv else 'prepare' + return Completed(int(self.exits.get(key, 0)), b'prepared\n', b'') if any(Path(part).suffix == '.py' for part in argv): needles = { 'source-guard-c-abi': 'check_c_abi_runtime.py', @@ -178,6 +183,7 @@ def _configure(self) -> Completed: commands[0]['command'] = f'{self.cxx} -c src/matrix.cpp' (self.build_dir / 'compile_commands.json').write_text(json.dumps(commands)) self._maybe_seed_abi_base() + self._maybe_seed_abi_base(prior=True) return Completed(0, b'configured\n', b'') def _build(self) -> Completed: @@ -213,17 +219,17 @@ def _install(self) -> Completed: (self.build_dir / 'ci-smoke' / 'smoke_version').write_bytes(b'smoke') return Completed(0, b'installed\n', b'') - def _maybe_seed_abi_base(self) -> None: - kind = self.exits.get('preexisting_base') + def _maybe_seed_abi_base(self, *, prior=False) -> None: + kind = self.exits.get('preexisting_prior' if prior else 'preexisting_base') if not kind: return - output = self.build_dir / 'settlement-abi-base' + output = self.build_dir / ('settlement-abi-prior' if prior else 'settlement-abi-base') output.mkdir(parents=True, exist_ok=True) (output / 'sentinel').write_text('keep\n') if kind == 'no-receipt': return archive = output / 'build' / 'lib' / 'libpineforge.a' - headers = output / 'r2-headers.tar' + headers = output / ('headers.tar' if prior else 'r2-headers.tar') generated = output / 'build' / 'include' / 'pineforge' / 'version.h' archive.parent.mkdir(parents=True, exist_ok=True) generated.parent.mkdir(parents=True, exist_ok=True) @@ -237,11 +243,11 @@ def _maybe_seed_abi_base(self) -> None: cache['PINEFORGE_VERSION_SOURCE'] = 'AUTO' receipt = { 'schemaVersion': 'pineforge-settlement-abi-base/v1', - 'commit': BASE_COMMIT, - 'tree': BASE_TREE, + 'commit': PRIOR_COMMIT if prior else BASE_COMMIT, + 'tree': PRIOR_TREE if prior else BASE_TREE, 'archive': 'build/lib/libpineforge.a', 'archiveSha256': identity(archive)['sha256'], - 'headers': 'r2-headers.tar', + 'headers': headers.name, 'headersSha256': identity(headers)['sha256'], 'generatedInclude': 'build/include', 'generatedHeaderSha256': identity(generated)['sha256'], @@ -260,9 +266,9 @@ def names(self) -> list[str]: elif argv[0] == 'ctest' and '--test-dir' in argv: names.append('ctest') elif argv[0] == 'git' and 'fetch' in argv: - names.append('fetch') + names.append('prior-fetch' if PRIOR_COMMIT in argv else 'fetch') elif any(Path(part).name == 'prepare_settlement_cpp_abi_base.py' for part in argv): - names.append('prepare') + names.append('prior-prepare' if PRIOR_COMMIT in argv else 'prepare') elif argv[0] == 'cmake' and '-S' in argv and 'smoke_consumer' in ''.join(argv): names.append('smoke-configure') elif Path(argv[0]).name == 'smoke_version': @@ -434,7 +440,7 @@ def setUp(self): def tearDown(self): self.temporary.cleanup() - def seed(self, *, version='FILE', corrupt=False, receipt=True): + def seed(self, *, version='FILE', corrupt=False, receipt=True, prior=False): archive = self.output / 'build' / 'lib' / 'libpineforge.a' headers = self.output / 'r2-headers.tar' generated = self.output / 'build' / 'include' / 'pineforge' / 'version.h' @@ -453,8 +459,8 @@ def seed(self, *, version='FILE', corrupt=False, receipt=True): current = copied_cache(read_cache_for_test(self.build / 'CMakeCache.txt')) payload = { 'schemaVersion': 'pineforge-settlement-abi-base/v1', - 'commit': BASE_COMMIT, - 'tree': BASE_TREE, + 'commit': PRIOR_COMMIT if prior else BASE_COMMIT, + 'tree': PRIOR_TREE if prior else BASE_TREE, 'archive': 'build/lib/libpineforge.a', 'archiveSha256': identity(archive)['sha256'], 'headers': 'r2-headers.tar', @@ -495,6 +501,22 @@ def test_corrupt_archive_hash_refuses_without_deletion(self): reusable_prepared_base(self.output, self.build) self.assertTrue((self.output / 'sentinel').is_file()) + def test_prior_reuse_pins_its_own_commit_and_tree(self): + self.seed(prior=True) + receipt = reusable_prepared_base(self.output, self.build, commit=PRIOR_COMMIT, tree=PRIOR_TREE) + self.assertEqual(receipt['commit'], PRIOR_COMMIT) + with self.assertRaisesRegex(RuntimeError, 'does not pin e60 R2'): + reusable_prepared_base(self.output, self.build) + with self.assertRaisesRegex(RuntimeError, 'does not pin 0e18690'): + reusable_prepared_base(self.output, self.build, commit=PRIOR_COMMIT, tree=BASE_TREE) + self.assertTrue((self.output / 'sentinel').is_file()) + + def test_prior_reuse_also_refuses_version_source_mismatch(self): + self.seed(prior=True, version='AUTO') + with self.assertRaisesRegex(RuntimeError, 'fresh --build-dir'): + reusable_prepared_base(self.output, self.build, commit=PRIOR_COMMIT, tree=PRIOR_TREE) + self.assertTrue((self.output / 'sentinel').is_file()) + def read_cache_for_test(path: Path) -> dict[str, str]: from prepare_settlement_cpp_abi_base import read_cache @@ -588,6 +610,12 @@ def test_successful_scripted_release_exit_zero(self): self.assertEqual(summary['status'], 'passed') self.assertEqual(summary['exitCode'], 0) self.assertEqual(summary['versionSource'], 'FILE') + self.assertIn('abi-base', stage_names(summary)) + self.assertIn('abi-prior', stage_names(summary)) + names = stage_names(summary) + self.assertLess(names.index('build'), names.index('abi-base')) + self.assertLess(names.index('abi-base'), names.index('abi-prior')) + self.assertLess(names.index('abi-prior'), names.index('ctest')) self.assertIn('ctest', scripted.names()) self.assertIn('install', scripted.names()) self.assertTrue((build_dir / 'ci-logs' / 'ctest.log').is_file()) @@ -635,6 +663,56 @@ def test_present_object_does_not_fetch(self): self.assertNotIn('fetch', scripted.names()) self.assertIn('prepare', scripted.names()) + def test_matching_prior_is_reused_without_fetch_or_prepare(self): + code, summary, scripted, _ = self.run_profile(preexisting_prior='match') + self.assertEqual(code, 0, summary['failures']) + self.assertEqual(summary['abiPrior']['action'], 'reused') + self.assertNotIn('prior-fetch', scripted.names()) + self.assertNotIn('prior-prepare', scripted.names()) + + def test_mismatch_prior_refuses_without_deletion_and_keeps_other_checks(self): + code, summary, scripted, build_dir = self.run_profile(preexisting_prior='mismatch') + self.assertEqual(code, 1) + self.assertEqual(summary['abiPrior']['action'], 'refused') + self.assertIn('abi-prior', failure_stages(summary)) + self.assertIn('fresh --build-dir', summary['failures'][0]['error']) + self.assertTrue((build_dir / 'settlement-abi-prior' / 'sentinel').is_file()) + self.assertTrue((build_dir / 'settlement-abi-prior' / 'receipt.json').is_file()) + self.assertNotIn('prior-prepare', scripted.names()) + self.assertIn('ctest', scripted.names()) + self.assertIn('install', scripted.names()) + + def test_prior_fetch_is_exact_and_only_when_object_missing(self): + code, summary, scripted, build_dir = self.run_profile(**{'prior-cat-file': 1}) + self.assertEqual(code, 0, summary['failures']) + fetches = [argv for argv in scripted.calls if argv[0] == 'git' and 'fetch' in argv] + self.assertEqual(fetches, [['git', '-C', str(ROOT), 'fetch', '--no-tags', '--depth=1', + 'origin', PRIOR_COMMIT]]) + self.assertIn('abi-prior-fetch', stage_names(summary)) + prepare = next(argv for argv in scripted.calls + if any(Path(part).name == 'prepare_settlement_cpp_abi_base.py' for part in argv) + and PRIOR_COMMIT in argv) + self.assertEqual(prepare[prepare.index('--tree') + 1], PRIOR_TREE) + self.assertEqual(Path(prepare[prepare.index('--output') + 1]).resolve(), + (build_dir / 'settlement-abi-prior').resolve()) + self.assertEqual(prepare[prepare.index('--header-manifest') + 1], + str(ROOT / 'tests/fixtures/settlement_cpp_abi/0e18690/manifest.json')) + + def test_present_prior_object_does_not_fetch(self): + code, summary, scripted, _ = self.run_profile(**{'prior-cat-file': 0}) + self.assertEqual(code, 0, summary['failures']) + self.assertNotIn('prior-fetch', scripted.names()) + self.assertIn('prior-prepare', scripted.names()) + + def test_prior_fetch_failure_does_not_prepare_and_keeps_other_checks(self): + code, summary, scripted, _ = self.run_profile(**{'prior-cat-file': 1, 'prior-fetch': 1}) + self.assertEqual(code, 1) + self.assertEqual(summary['abiPrior']['action'], 'failed') + self.assertIn('abi-prior-fetch', failure_stages(summary)) + self.assertNotIn('prior-prepare', scripted.names()) + self.assertIn('ctest', scripted.names()) + self.assertIn('install', scripted.names()) + def test_require_websocket_rejects_skip(self): code, summary, scripted, _ = self.run_profile( 'native', extra=['--require-websocket'], **{'require-websocket': 77}) diff --git a/scripts/test_settlement_cpp_abi.py b/scripts/test_settlement_cpp_abi.py index a3ae51514..d6376ac7d 100644 --- a/scripts/test_settlement_cpp_abi.py +++ b/scripts/test_settlement_cpp_abi.py @@ -1,13 +1,19 @@ #!/usr/bin/env python3 """Narrow offline refusal tests for the settlement ABI tooling; no C++ execution.""" import io +import json from pathlib import Path +import shutil import tempfile import tarfile +from types import SimpleNamespace import unittest -from check_settlement_cpp_abi import ENGINE, ROOT, storage_declarations, validate_rejection -from prepare_settlement_cpp_abi_base import extract_tar, read_cache +from check_settlement_cpp_abi import ( + ENGINE, ROOT, REVERSAL_METHODS, REVERSAL_DOMAIN, frozen_shape, load_prior, + storage_declarations, validate_rejection, +) +from prepare_settlement_cpp_abi_base import BASE_COMMIT, BASE_TREE, extract_tar, read_cache class AbiToolingTests(unittest.TestCase): @@ -33,6 +39,59 @@ def test_generic_failure_or_missing_one_method_cannot_pass(self): with self.assertRaisesRegex(RuntimeError,'omits expected'): validate_rejection(valid,['project_native_settlement_v1','project_native_settlement_scoped_v1']) + def reversal_diagnostic(self, style, methods=REVERSAL_METHODS, domain=REVERSAL_DOMAIN): + symbols=[ENGINE+method+'(pineforge::execution::'+domain+' const&, pineforge::execution::Fill const&)' + for method in methods] + formats={'mac': lambda symbol: ' "'+symbol+'", referenced from:\n _main', + 'gnu': lambda symbol: "caller.cpp: undefined reference to `"+symbol+"'", + 'lld': lambda symbol: 'ld.lld: error: undefined symbol: '+symbol} + return '\n'.join(formats[style](symbol) for symbol in symbols) + + def test_reversal_rejection_requires_all_four_names_and_parameter_domain(self): + for style in ('mac','gnu','lld'): + with self.subTest(style=style): + valid=self.reversal_diagnostic(style) + self.assertEqual(len(validate_rejection(valid,REVERSAL_METHODS,REVERSAL_DOMAIN)),4) + for omitted in REVERSAL_METHODS: + with self.assertRaisesRegex(RuntimeError,'omits expected'): + validate_rejection(self.reversal_diagnostic(style,[m for m in REVERSAL_METHODS if m!=omitted]), + REVERSAL_METHODS,REVERSAL_DOMAIN) + for bad_domain in ('reverse_to_v2::ReverseTo','ReverseTo'): + with self.assertRaisesRegex(RuntimeError,'namespace'): + validate_rejection(self.reversal_diagnostic(style,domain=bad_domain), + REVERSAL_METHODS,REVERSAL_DOMAIN) + with self.assertRaisesRegex(RuntimeError,'unrelated'): + validate_rejection(valid+"\nundefined reference to `other_dependency()'", + REVERSAL_METHODS,REVERSAL_DOMAIN) + + def test_missing_prior_receipt_fails_with_exact_prepare_remedy(self): + with tempfile.TemporaryDirectory() as temporary: + root=Path(temporary) + for receipt in (None,root/'missing.json'): + with self.assertRaisesRegex(RuntimeError,'settlement-abi-prior .*--commit 0e18690'): + load_prior(SimpleNamespace(prior_receipt=receipt),root/'headers',{}) + self.assertFalse((root/'headers').exists()) + + def test_e60_receipt_cannot_be_used_for_reversal_prior(self): + with tempfile.TemporaryDirectory() as temporary: + root=Path(temporary);receipt=root/'wrong-provider.json' + receipt.write_text(json.dumps({'commit':BASE_COMMIT,'tree':BASE_TREE})) + with self.assertRaisesRegex(RuntimeError,'does not pin 0e18690'): + load_prior(SimpleNamespace(prior_receipt=receipt),root/'headers',{}) + self.assertFalse((root/'headers').exists()) + + def test_action_alternative_changes_are_frozen(self): + with tempfile.TemporaryDirectory() as temporary: + copied=Path(temporary)/'include' + shutil.copytree(ROOT/'include',copied) + header=copied/'pineforge/execution.hpp' + source=header.read_text() + original='std::variant' + self.assertIn(original,source) + header.write_text(source.replace(original,original[:-1]+', int>')) + with self.assertRaisesRegex(RuntimeError,'Action alternative'): + frozen_shape(ROOT/'include',copied) + def test_same_size_member_change_or_added_padding_member_is_visible(self): source=(ROOT/'include/pineforge/engine.hpp').read_text() original=storage_declarations(source) diff --git a/src/engine_execution.cpp b/src/engine_execution.cpp index e3e1cfa8a..fd83b629c 100644 --- a/src/engine_execution.cpp +++ b/src/engine_execution.cpp @@ -156,6 +156,64 @@ struct BacktestEngine::NativeSettlementStage { std::unordered_set selected_ids; }; +execution::Status BacktestEngine::validate_native_settlement_book(double& held) const { + using execution::Status; + held = 0.0; + if (position_side_ != PositionSide::FLAT + && position_side_ != PositionSide::LONG + && position_side_ != PositionSide::SHORT) + return Status::InvalidBook; + if ((position_side_ == PositionSide::FLAT) != pyramid_entries_.empty()) + return Status::InvalidBook; + for (const auto& lot : pyramid_entries_) { + if (!std::isfinite(lot.qty) || lot.qty <= 0.0 || !std::isfinite(lot.price)) + return Status::InvalidBook; + held += lot.qty; + if (!std::isfinite(held)) return Status::InvalidBook; + } + return Status::Applied; +} + +execution::Status BacktestEngine::allocate_native_settlement_closes( + NativeSettlementStage& stage, + const execution::CloseScope& book_or_opening, + double& remaining) const { + using execution::Status; + if (!stage.flatten || stage.scoped) + stage.survivors.reserve(pyramid_entries_.size()); + stage.closing_indices.reserve(stage.closes ? pyramid_entries_.size() : 0); + stage.closing_quantities.reserve(stage.closes ? pyramid_entries_.size() : 0); + double closed = 0.0; + for (size_t index = 0; index < pyramid_entries_.size(); ++index) { + const auto& lot = pyramid_entries_[index]; + const bool member = stage.use_selected + ? stage.selected_ids.count(lot.entry_incarnation) != 0 + : selected_for_close(book_or_opening, lot); + const auto split = next_close_split( + lot, stage.closes && member, stage.flatten, + stage.allocation_requested, closed, remaining); + if (split.status != Status::Applied) return split.status; + if (split.amount == 0.0) { + stage.survivors.push_back(lot); + continue; + } + stage.closing_indices.push_back(index); + stage.closing_quantities.push_back(split.amount); + if (split.kept > 0.0) { + auto survivor = lot; + const double scale = split.kept / lot.qty; + survivor.qty = split.kept; + survivor.max_runup *= scale; + survivor.max_drawdown *= scale; + survivor.entry_commission_account = open_entry_commission(lot) + - allocated_entry_commission(lot, split.amount); + stage.survivors.push_back(std::move(survivor)); + } + } + stage.closed = closed; + return Status::Applied; +} + void BacktestEngine::stage_native_settlement( NativeSettlementStage& stage, const execution::Action& action, @@ -190,28 +248,11 @@ void BacktestEngine::stage_native_settlement( fail(Status::InvalidQuantity); return; } - if (position_side_ != PositionSide::FLAT - && position_side_ != PositionSide::LONG - && position_side_ != PositionSide::SHORT) { - fail(Status::InvalidBook); - return; - } - if ((position_side_ == PositionSide::FLAT) != pyramid_entries_.empty()) { - fail(Status::InvalidBook); - return; - } - double held = 0.0; - for (const auto& lot : pyramid_entries_) { - if (!std::isfinite(lot.qty) || lot.qty <= 0.0 || !std::isfinite(lot.price)) { - fail(Status::InvalidBook); - return; - } - held += lot.qty; - if (!std::isfinite(held)) { - fail(Status::InvalidBook); - return; - } + if (const auto status = validate_native_settlement_book(held); + status != Status::Applied) { + fail(status); + return; } if (lifecycle) { if (auto invalid = validate_lifecycle_effects(*lifecycle)) { @@ -264,47 +305,86 @@ void BacktestEngine::stage_native_settlement( stage.closes = stage.flatten || reduce || stage.opposite; stage.was_long = position_side_ == PositionSide::LONG; - if (!stage.flatten || stage.scoped) - stage.survivors.reserve(pyramid_entries_.size()); - stage.closing_indices.reserve(stage.closes ? pyramid_entries_.size() : 0); - stage.closing_quantities.reserve(stage.closes ? pyramid_entries_.size() : 0); - double closed = 0.0; double remaining = stage.allocation_requested; - for (size_t index = 0; index < pyramid_entries_.size(); ++index) { - const auto& lot = pyramid_entries_[index]; - const bool member = stage.use_selected - ? stage.selected_ids.count(lot.entry_incarnation) != 0 - : selected_for_close(book_or_opening, lot); - const auto split = next_close_split( - lot, stage.closes && member, stage.flatten, - stage.allocation_requested, closed, remaining); - if (split.status != Status::Applied) { - fail(split.status); - return; - } - if (split.amount == 0.0) { - stage.survivors.push_back(lot); - continue; - } - stage.closing_indices.push_back(index); - stage.closing_quantities.push_back(split.amount); - if (split.kept > 0.0) { - auto survivor = lot; - const double scale = split.kept / lot.qty; - survivor.qty = split.kept; - survivor.max_runup *= scale; - survivor.max_drawdown *= scale; - survivor.entry_commission_account = open_entry_commission(lot) - - allocated_entry_commission(lot, split.amount); - stage.survivors.push_back(std::move(survivor)); - } + if (const auto status = allocate_native_settlement_closes( + stage, book_or_opening, remaining); status != Status::Applied) { + fail(status); + return; } - stage.closed = closed; stage.opening = !transact ? 0.0 : stage.opposite ? remaining : stage.requested; if (stage.opening > 0.0 && stage.opposite && !stage.survivors.empty()) { fail(Status::UnrepresentableQuantity); return; } + finish_native_settlement_stage(stage, fill); +} + +void BacktestEngine::stage_native_settlement( + NativeSettlementStage& stage, + const execution::ReverseTo& reversal, + const execution::Fill& fill, + const execution::LifecycleEffects* lifecycle) const { + using execution::Status; + stage = NativeSettlementStage{}; + auto fail = [&](Status status) { + stage.phase = NativeSettlementStage::Phase::Invalid; + stage.status = status; + }; + if (!std::isfinite(fill.price)) { fail(Status::InvalidPrice); return; } + if (fill.commission_account && !std::isfinite(*fill.commission_account)) { + fail(Status::InvalidAccounting); + return; + } + if (!std::isfinite(reversal.signed_units) || reversal.signed_units == 0.0) { + fail(Status::InvalidQuantity); + return; + } + double held = 0.0; + if (const auto status = validate_native_settlement_book(held); + status != Status::Applied) { + fail(status); + return; + } + if (lifecycle) { + if (auto invalid = validate_lifecycle_effects(*lifecycle)) { + fail(*invalid); + return; + } + } + stage.incoming = reversal.signed_units < 0.0 + ? PositionSide::SHORT : PositionSide::LONG; + if (position_side_ == PositionSide::FLAT || position_side_ == stage.incoming) { + fail(Status::InvalidCloseTarget); + return; + } + stage.flatten = true; + stage.opposite = true; + stage.closes = true; + stage.was_long = position_side_ == PositionSide::LONG; + stage.selected_held = held; + double remaining = 0.0; + if (const auto status = allocate_native_settlement_closes( + stage, execution::Book{}, remaining); status != Status::Applied) { + fail(status); + return; + } + // ReverseTo carries the exact new exposure independently of all closing + // lots. Neither transaction-flow subtraction nor absorption applies here. + stage.opening = std::abs(reversal.signed_units); + if (!std::isfinite(stage.closed + stage.opening)) { + fail(Status::UnrepresentableQuantity); + return; + } + finish_native_settlement_stage(stage, fill); +} + +void BacktestEngine::finish_native_settlement_stage( + NativeSettlementStage& stage, const execution::Fill& fill) const { + using execution::Status; + auto fail = [&](Status status) { + stage.phase = NativeSettlementStage::Phase::Invalid; + stage.status = status; + }; stage.current_costs = quote_execution_commissions( stage.closing_quantities, stage.opening, fill); stage.ticket = 0.0; @@ -344,6 +424,44 @@ void BacktestEngine::stage_native_settlement( stage.status = Status::Applied; } +execution::SettlementInspection BacktestEngine::inspect_native_reversal_v1( + const execution::ReverseTo& reversal, const execution::Fill& fill) const { + NativeSettlementStage stage; + stage_native_settlement(stage, reversal, fill, nullptr); + return inspect_native_settlement_stage(stage, fill); +} + +execution::AccountEffectProjection BacktestEngine::project_native_reversal_v1( + const execution::ReverseTo& reversal, const execution::Fill& fill) const { + NativeSettlementStage stage; + stage_native_settlement(stage, reversal, fill, nullptr); + return project_native_settlement_stage(stage, fill); +} + +execution::Result BacktestEngine::settle_native_reversal_at_v1( + const execution::ReverseTo& reversal, const execution::Fill& fill, + const execution::PhysicalExecutionContext& context) { + const execution::LifecycleEffects lifecycle; + NativeSettlementStage stage; + stage_native_settlement(stage, reversal, fill, &lifecycle); + return commit_native_settlement_stage(stage, fill, lifecycle, context); +} + +execution::Result BacktestEngine::settle_reversal_with_lifecycle_v1( + const execution::ReverseTo& reversal, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle) { + execution::PhysicalExecutionContext context; + context.effective_time_ms = current_bar_.timestamp; + context.interval_index = bar_index_; + context.preceding_exit_path_prefix = fold_exit_path_extremes_; + if (!std::isnan(fold_exit_trail_peak_)) { + context.preceding_exit_trail_peak = fold_exit_trail_peak_; + } + NativeSettlementStage stage; + stage_native_settlement(stage, reversal, fill, &lifecycle); + return commit_native_settlement_stage(stage, fill, lifecycle, context); +} + execution::Result BacktestEngine::settle_resolved_execution( const execution::Action& action, const execution::Fill& fill) { return settle_execution_with_lifecycle(action, fill, {}); @@ -426,10 +544,17 @@ execution::Result BacktestEngine::settle_with_membership( const execution::PhysicalExecutionContext& context, execution::CloseScope book_or_opening, const execution::SelectedOpeningSet* selected) { - using execution::Status; NativeSettlementStage stage; stage_native_settlement( stage, action, fill, book_or_opening, selected, &lifecycle); + return commit_native_settlement_stage(stage, fill, lifecycle, context); +} + +execution::Result BacktestEngine::commit_native_settlement_stage( + NativeSettlementStage& stage, const execution::Fill& fill, + const execution::LifecycleEffects& lifecycle, + const execution::PhysicalExecutionContext& context) { + using execution::Status; if (stage.phase != NativeSettlementStage::Phase::Ready) return {stage.status}; const auto no_effect = [&]() -> execution::Result { @@ -561,10 +686,15 @@ execution::SettlementInspection BacktestEngine::inspect_with_membership( const execution::Action& action, const execution::Fill& fill, execution::CloseScope book_or_opening, const execution::SelectedOpeningSet* selected) const { - using execution::Status; NativeSettlementStage stage; stage_native_settlement( stage, action, fill, book_or_opening, selected, nullptr); + return inspect_native_settlement_stage(stage, fill); +} + +execution::SettlementInspection BacktestEngine::inspect_native_settlement_stage( + const NativeSettlementStage& stage, const execution::Fill& fill) const { + using execution::Status; execution::SettlementInspection out; if (stage.phase != NativeSettlementStage::Phase::Ready) { out.status = stage.status; @@ -611,10 +741,15 @@ execution::AccountEffectProjection BacktestEngine::project_with_membership( const execution::Action& action, const execution::Fill& fill, execution::CloseScope book_or_opening, const execution::SelectedOpeningSet* selected) const { - using execution::Status; NativeSettlementStage stage; stage_native_settlement( stage, action, fill, book_or_opening, selected, nullptr); + return project_native_settlement_stage(stage, fill); +} + +execution::AccountEffectProjection BacktestEngine::project_native_settlement_stage( + const NativeSettlementStage& stage, const execution::Fill& fill) const { + using execution::Status; if (stage.phase == NativeSettlementStage::Phase::Invalid) return invalid_projection(stage.status); diff --git a/src/engine_orders.cpp b/src/engine_orders.cpp index 13df2f1e9..0eb95db9d 100644 --- a/src/engine_orders.cpp +++ b/src/engine_orders.cpp @@ -20,6 +20,51 @@ namespace pineforge { using namespace internal; namespace { +// Existing source FIFO endpoint policy; never a native quantity tolerance. +// Keep the R2 stop/whole-lot interpretation at 1e-10 in this adapter. +constexpr double kSourceFifoEndpointEpsilon = kQtyEpsilon; + +std::optional source_fifo_prefix_membership( + const std::vector& lots, double qty_limit, + int64_t cycle) { + if (cycle <= 0 || !std::isfinite(qty_limit) || qty_limit <= 0.0) + return std::nullopt; + + double qty_closed = 0.0; + size_t prefix_size = 0; + for (const auto& lot : lots) { + // Match the source's original accumulation and endpoint ordering. + // Once at the endpoint, even a tiny next sibling stays unselected. + if (qty_closed >= qty_limit - kSourceFifoEndpointEpsilon) break; + if (!std::isfinite(lot.qty) || lot.qty <= 0.0) return std::nullopt; + const double close_qty = std::min(lot.qty, qty_limit - qty_closed); + const double keep_qty = lot.qty - close_qty; + if (keep_qty > kSourceFifoEndpointEpsilon) return std::nullopt; + ++prefix_size; + qty_closed += close_qty; + } + if (prefix_size == 0 || prefix_size == lots.size()) return std::nullopt; + + execution::SelectedOpeningSet selection{cycle, {}}; + std::unordered_set included; + double selected_qty = 0.0; + for (size_t index = 0; index < prefix_size; ++index) { + const auto& lot = lots[index]; + if (lot.entry_incarnation == 0) return std::nullopt; + if (included.insert(lot.entry_incarnation).second) + selection.incarnations.push_back(lot.entry_incarnation); + selected_qty += lot.qty; + if (!std::isfinite(selected_qty)) return std::nullopt; + } + // An opening identity may have multiple physical fragments, but all of + // its live fragments must belong to this prefix. Otherwise use Reduce. + for (size_t index = prefix_size; index < lots.size(); ++index) { + if (included.count(lots[index].entry_incarnation) != 0) + return std::nullopt; + } + return selection; +} + // Source predicates are resolved here, never retained by native settlement. // Every fragment of an opening must agree with the selected source predicate. template @@ -412,9 +457,14 @@ void BacktestEngine::execute_partial_exit_qty( fill_price = apply_fill_slippage(fill_price, is_buy); const int pre_count = position_entry_count_; execution::Action action = order_action::Reduce{qty_to_close}; + std::optional prefix; if (std::abs(held) - qty_to_close <= kQtyEpsilon) action = execution::Flatten{}; - const auto result = settle_resolved_execution( - action, execution::Fill{fill_price, {}, {}, 0}); + else prefix = source_fifo_prefix_membership( + pyramid_entries_, qty_to_close, position_cycle_seq_); + const execution::Fill fill{fill_price, {}, {}, 0}; + const auto result = prefix + ? settle_execution_selected_with_lifecycle(execution::Flatten{}, fill, {}, *prefix) + : settle_resolved_execution(action, fill); if (result.status != execution::Status::Applied && result.status != execution::Status::NoEffect) throw std::runtime_error("invalid resolved partial-close settlement"); @@ -1385,23 +1435,18 @@ void BacktestEngine::flip_market_position_to(const std::string& id, bool is_long uint64_t entry_incarnation) { // The incoming direction is also the closing direction. The caller has // already resolved this one price; no un-slip/re-slip or second ticket. - execution::Action action = execution::Flatten{}; + const execution::Fill fill{fill_price, id, {}, entry_incarnation}; + double incoming = 0.0; if (!close_only) { - double held = 0.0; - for (const auto& lot : pyramid_entries_) held += lot.qty; - const double incoming = source_reversal_qty( + incoming = source_reversal_qty( fill_price, explicit_qty, explicit_qty_type, explicit_qty_prequantized); if (!std::isfinite(incoming) || incoming < 0.0) throw std::runtime_error("invalid resolved flip quantity"); - if (incoming > 0.0) { - const double total = held + incoming; - if (!std::isfinite(total) || total == held || (held > 0.0 && total == incoming)) - throw std::runtime_error("unrepresentable resolved flip quantity"); - action = order_action::Transact{is_long ? total : -total}; - } } - const auto result = settle_resolved_execution( - action, execution::Fill{fill_price, id, {}, entry_incarnation}); + const auto result = incoming > 0.0 + ? settle_reversal_with_lifecycle_v1( + execution::ReverseTo{is_long ? incoming : -incoming}, fill, {}) + : settle_resolved_execution(execution::Flatten{}, fill); if (result.status != execution::Status::Applied && result.status != execution::Status::NoEffect) throw std::runtime_error("invalid resolved flip settlement"); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 379b2ccaa..5c18c009f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -6,6 +6,9 @@ set(TEST_SOURCES test_native_settlement_projection test_native_settlement_callers test_native_reversal_contract + test_native_reverse_to + test_source_exact_reversal + test_source_fifo_endpoints test_order_action test_native_order_core test_native_order_terminal @@ -352,6 +355,9 @@ set_tests_properties(test_native_cpp_abi PROPERTIES TIMEOUT 240) set(PINEFORGE_SETTLEMENT_ABI_BASE_RECEIPT "${PROJECT_BINARY_DIR}/settlement-abi-base/receipt.json" CACHE FILEPATH "Prepared actual R2 ABI provider receipt") +set(PINEFORGE_SETTLEMENT_ABI_PRIOR_RECEIPT + "${PROJECT_BINARY_DIR}/settlement-abi-prior/receipt.json" + CACHE FILEPATH "Prepared actual 0e18690 ABI provider receipt") add_test(NAME test_settlement_cpp_abi COMMAND ${Python3_EXECUTABLE} ${PROJECT_SOURCE_DIR}/scripts/check_settlement_cpp_abi.py --compiler ${CMAKE_CXX_COMPILER} @@ -359,6 +365,7 @@ add_test(NAME test_settlement_cpp_abi --include ${PROJECT_SOURCE_DIR}/include --generated-include ${PROJECT_BINARY_DIR}/include --base-receipt ${PINEFORGE_SETTLEMENT_ABI_BASE_RECEIPT} + --prior-receipt ${PINEFORGE_SETTLEMENT_ABI_PRIOR_RECEIPT} --receipt ${PROJECT_BINARY_DIR}/settlement-abi-receipt.json ${_pf_script_cpp_abi_flags}) set_tests_properties(test_settlement_cpp_abi PROPERTIES TIMEOUT 600) diff --git a/tests/fixtures/settlement_cpp_abi/0e18690/manifest.json b/tests/fixtures/settlement_cpp_abi/0e18690/manifest.json new file mode 100644 index 000000000..b38140744 --- /dev/null +++ b/tests/fixtures/settlement_cpp_abi/0e18690/manifest.json @@ -0,0 +1,277 @@ +{ + "commit": "0e18690db3fb6bb4705841c2a86936dfa206380a", + "files": { + "include/pineforge/bar.hpp": { + "bytes": 655, + "git_blob": "836f73daab382e9c8d5a6d03a942a921e39b756f", + "sha256": "665c2528dca64ac9474de622dfd7ffdb540ff91316f70317e39aada158d1e0de" + }, + "include/pineforge/broker_events.hpp": { + "bytes": 4343, + "git_blob": "2972d53c7b039af266f4da161301c01eec54c16d", + "sha256": "4f3d9948a4d1604750901fa3982d6ea49bd50dd15a86c4e5d690b83d1f5f23c0" + }, + "include/pineforge/color.hpp": { + "bytes": 1253, + "git_blob": "a4d54f37a1c9d8d10ff5423aca4d42814a666446", + "sha256": "29eb9d4473e4dd488f959fb9f53999656e3feb6aa25255287e80855ecabe96d6" + }, + "include/pineforge/compat/pine/exit_activation.hpp": { + "bytes": 3017, + "git_blob": "a62927dd936e839160671cee665de908cb7f5dc3", + "sha256": "9190382359099021870d260718ad593c567024603b016294d1b113c278ddd73d" + }, + "include/pineforge/compat/pine/exit_lifecycle.hpp": { + "bytes": 882, + "git_blob": "c78f64b82adaf18e013e017a633b9b9b6b48f22d", + "sha256": "2fcfce8c914424404767bdd7b2fc6ea30decbc3f5f5a6123cac9080e2b9bd8d2" + }, + "include/pineforge/compat/pine/frozen_market_instruction.hpp": { + "bytes": 3201, + "git_blob": "8890ad48c5e4485b9d5a65a5bdd7f7f876e39c24", + "sha256": "e67a4b470e82086a089d3bf0b8996f88ab0fd5d86046682d6cb19b2ed64922c8" + }, + "include/pineforge/compat/pine/intraday_cap.hpp": { + "bytes": 10469, + "git_blob": "0aba76b6a78251df33e92f19290473adecf5a61e", + "sha256": "0ac0eab34625ab9a974037ff8c0217603a01deb81599350a4b24f7bf48ac0cbd" + }, + "include/pineforge/compat/pine/intraday_order_budget.hpp": { + "bytes": 3878, + "git_blob": "730d941fffa3939e375489f40a50901ccea9ec87", + "sha256": "43f7700f9c66c630ab8a9d8fd899eecb03c2fd9aabff11d2dc2b7fa85f92981b" + }, + "include/pineforge/compat/pine/market_admission.hpp": { + "bytes": 1057, + "git_blob": "33cf82e5e53ae870c3665ede69d7cde2e9695752", + "sha256": "83ce322f8864a569e197493cd2bcca23aa226997d3508caaa6a783ff1bb97b03" + }, + "include/pineforge/compat/pine/order_birth.hpp": { + "bytes": 841, + "git_blob": "aa48c054f613b65ee5ded14fde41643f807347bd", + "sha256": "e819f5c4202911e5e592f4472925a15c07f7300fabcfecf4f214348e5ee605f8" + }, + "include/pineforge/compat/pine/order_priority.hpp": { + "bytes": 1577, + "git_blob": "86135165fd3d83fc4638a2295dda769024836517", + "sha256": "e88ffcf28bf3fac95bc439b3d99a3944fc9c36ce0d3892fd365d25f9f3f920e4" + }, + "include/pineforge/compat/pine/reservation_expansion.hpp": { + "bytes": 759, + "git_blob": "1115a6593f5fabadf722b14305a213b07afb6c85", + "sha256": "7ad4195af574df40a600b748399f00c41cafa9441bfdcb5b90898ec8a4b577dd" + }, + "include/pineforge/drawing.hpp": { + "bytes": 17150, + "git_blob": "d04f12bb214b82c5d0b174824c25ff9f148ab090", + "sha256": "1635cfc9e65a91369cc1491a64dc564944b6828333af62e3ecd80edb5ba74183" + }, + "include/pineforge/engine.hpp": { + "bytes": 312399, + "git_blob": "68cc254148529095b909b98a0764b3bbbbbd23ab", + "sha256": "7b9b0e5f4152e688ae80574cc99c770e26940e8885bbb19787d5b085d2fe21bd" + }, + "include/pineforge/execution.hpp": { + "bytes": 4294, + "git_blob": "f270b7c64544ee9188c22752d5e41eaa78bf01d1", + "sha256": "e6e2440118f38a0eba5b75a0fb0cf03675f92939c154aaf8dc0837653a113895" + }, + "include/pineforge/execution_close_scope.hpp": { + "bytes": 714, + "git_blob": "d4633524495566d68e7fba8e3244fa32f8bcc01e", + "sha256": "b319a176f7d3d0c668f2a1b5c607e3b2077088221e9196a1d490aa6cd548f0b6" + }, + "include/pineforge/execution_close_selection.hpp": { + "bytes": 517, + "git_blob": "1a6e9e89444e5730d923356b290beddb121df2d8", + "sha256": "efb4c90fd058f71ef896e32c9e08db4400b79e7d138c1c5b5f6eda3275646c6e" + }, + "include/pineforge/execution_consumer.hpp": { + "bytes": 3010, + "git_blob": "19e99314462c9c3729953b0f343abd4aa0bf1496", + "sha256": "ffd09417d1f1e8aab32794673d6802616c5055b7976101e49416c283b64856e0" + }, + "include/pineforge/execution_projection.hpp": { + "bytes": 1835, + "git_blob": "68abfa45dfc387243b66cb72a270b191ed7c894b", + "sha256": "464f5f4a98cc83d78cbf769c2cc2c7e80e6934cf72ff24e0dd0496f4790d85f1" + }, + "include/pineforge/exit_leg_lifecycle.hpp": { + "bytes": 22622, + "git_blob": "91492ec22e05d7b6d9327f60ff5e239ae6906fcd", + "sha256": "9d785271511a350114adc7cf5bcb1600cf569f0a10559a8671b7e4b9cb9dc89a" + }, + "include/pineforge/generic_matrix.hpp": { + "bytes": 25438, + "git_blob": "8694e5577d56221498cca844aa14cddb9a15e84f", + "sha256": "9c94a877b6f9e5965604b03f572ceea2ea3680de586d69ef0627c78208624150" + }, + "include/pineforge/leg_activation.hpp": { + "bytes": 1353, + "git_blob": "4160c841782800853bb574219b3dad2ca8aa0cc6", + "sha256": "859d75d0ae90ec3cc109765da066b37c3bbbff9c7c840368fbdd05dc7e178f3f" + }, + "include/pineforge/live_parser.h": { + "bytes": 3870, + "git_blob": "0d00afe0586477c69dea35a1bfef72f3acb491a3", + "sha256": "f1308086a66fe6a0636b8b2fc44c3fcb6b0c21ed0311654297ecc8ea5e959e1e" + }, + "include/pineforge/log.hpp": { + "bytes": 496, + "git_blob": "6cc4c35ad1a902f05022fb11357dff86281b5301", + "sha256": "df5db36e01ddaa66c44c1b378e6b0c3b05e0c732c59fbcb5401e3606585d25a3" + }, + "include/pineforge/magnifier.hpp": { + "bytes": 3241, + "git_blob": "1bbc910a869d162900d7610fa0bd9a88776b7b0f", + "sha256": "c0ed220b6df054b44596a434d4f48b1c3e960ca329bb8644a405a608175e06ff" + }, + "include/pineforge/map.hpp": { + "bytes": 14476, + "git_blob": "5dd59cbaad9b93138738d3f92b8c457c0ac92a7d", + "sha256": "772a4f6c79ea01af6ad057e361e821d79a7d9884e4aff8bac5a2f77565e55b9f" + }, + "include/pineforge/market_admission.hpp": { + "bytes": 10221, + "git_blob": "70be6af6b556bf77cab6c9f823bb3492cf7c8294", + "sha256": "22914434a2c82f761b98c1b5a2ff9365c89bb802b584cdda2a3c30c5986ec6fe" + }, + "include/pineforge/market_driver.hpp": { + "bytes": 5374, + "git_blob": "3e6c68fb491156520640b1095dfce057395375a3", + "sha256": "f89b8b49127dff19f88199da50db3defbcd891c2a25cc777f595d7f277a7cd3c" + }, + "include/pineforge/math.hpp": { + "bytes": 1316, + "git_blob": "9ec09d539dd0ba732449424e6a8c80c3f5454888", + "sha256": "be95fa75d1e8d288ed49d62fa56257efe99435be7ba357fbb93cc36c0dc9eda3" + }, + "include/pineforge/matrix.hpp": { + "bytes": 4868, + "git_blob": "22b29448fb60dded446543b1940c1a1466986133", + "sha256": "f463ab76db92436fd818839c390e69462a794e07413d99fb49af4ed1b67bca6f" + }, + "include/pineforge/metrics.hpp": { + "bytes": 1353, + "git_blob": "f468f485bbdd0512536e96390d2b411228c2f517", + "sha256": "9e35101f16c2717dbfde428ecc515802dd758056e080fd45ee6411e9d0ead5be" + }, + "include/pineforge/na.hpp": { + "bytes": 1064, + "git_blob": "e98505aadc68a09cbe5a14aacf591a799a15ecd2", + "sha256": "0e8c256fcdeda94a0855458fa3b8c4ad4f9864dc2aecce969934f6bea82fd0a5" + }, + "include/pineforge/native_calendar.hpp": { + "bytes": 19589, + "git_blob": "742ecfabc3764d626617c2762d4b574d46d8ff46", + "sha256": "5495cb007509937616255a9615dcc92c49e5115692c8375ef93b594a36e55860" + }, + "include/pineforge/native_host.hpp": { + "bytes": 10304, + "git_blob": "58573006ad056ecaa773dc79d22ca1375d51fac8", + "sha256": "9b7236e5946d1e5d2355cd6a72a02d987ab6b215acf2155ebce89771c8015808" + }, + "include/pineforge/native_order.hpp": { + "bytes": 36944, + "git_blob": "238475d372cc87b91d163c6c1a378f10751d444e", + "sha256": "3881f6b1ebecb95801e5ea6f4acb5a5a8203d94322ffdc36663d69b4966b7c08" + }, + "include/pineforge/native_order_identity.hpp": { + "bytes": 1948, + "git_blob": "c0a6835c8fdb5c3b331ad519439af12946e34cf2", + "sha256": "6f6d63c1a1f54499f04a137d0b13ea343aaf94ffe090058a62c854311345ac69" + }, + "include/pineforge/native_run_spec.hpp": { + "bytes": 5518, + "git_blob": "1b5ed883a71ed772e9f115415ab5c7acb1f920f0", + "sha256": "e7be6d930c4eed13d9fa2358f529c98e5ad37b75ad72013f1ba65513cabb8da1" + }, + "include/pineforge/order_action.hpp": { + "bytes": 3470, + "git_blob": "d992472cbefefdf8e29704a21f04614d68d39a58", + "sha256": "7d76ddc3f6ba76216d5feb34cdf28908e33a92a56757a707ac6bd94dca315001" + }, + "include/pineforge/order_birth.hpp": { + "bytes": 4833, + "git_blob": "ed4a4038e9d079c92b4ebd777a5f22f1f1256b0c", + "sha256": "0eadb3e2a92248fbc8e6ba8f94285b15b0e83d7a0396f5904b40c216619604e4" + }, + "include/pineforge/order_cancellation.hpp": { + "bytes": 7172, + "git_blob": "a4f11409f19c7a85347879ecfd942bd7bae5c2eb", + "sha256": "389eb3659f121351ad4b53b936d3a2c79f343ecdb6c0c4bb0bc13777596b90ee" + }, + "include/pineforge/order_priority.hpp": { + "bytes": 694, + "git_blob": "3e9fa5ee4121971a7962db8a60859aba6fb9ec95", + "sha256": "25bb7898972153b044675245db230c1171644856ef904f4dd02353b8968d433b" + }, + "include/pineforge/pending_order_mirror.hpp": { + "bytes": 21978, + "git_blob": "fc08bfd7328718de1c658bdcea1826c2fbc9eebe", + "sha256": "7921179f690dbb6f00ee8ab6ca71b3174fcc42a61ebc6786c3727c7f85a576a4" + }, + "include/pineforge/pine_float_compare.hpp": { + "bytes": 2893, + "git_blob": "a3de0e9a2ef3517098be500880fda71bdcfe2260", + "sha256": "3d389142b8e4fbd899c6923c11a27447920a1c9e9db0338ff9abc8a9f4a06d1c" + }, + "include/pineforge/pineforge.h": { + "bytes": 74362, + "git_blob": "31db9f0b9f8ac5f33041917f7d599daff91013c8", + "sha256": "e4c735a1e39411db4156ac13a50cee10c554039b20f33e3c2299cd73be6a6c3e" + }, + "include/pineforge/position_close_obligation.hpp": { + "bytes": 1060, + "git_blob": "09576ee0f2a1d49becc46f5701f4369cf8f87bf6", + "sha256": "6f6ad0aaca952dda7a9386cb78e57705b3494364dbd8970b0a051fe99d05b777" + }, + "include/pineforge/quantity_intent.hpp": { + "bytes": 3054, + "git_blob": "41262fee5aa2db4032fc008542c573032563629b", + "sha256": "c4cc0f1cd5d6ed1f248825712a5ba4b4f5a808b787c05c58971934b89d770b90" + }, + "include/pineforge/reservation_expansion.hpp": { + "bytes": 2434, + "git_blob": "b4302c9eed97c3409d9911fd6cecdc64fa0aad86", + "sha256": "27e514249f62aa7353ac084a6e59e25e338a867fe4d67ce4646446a1cf9f9dab" + }, + "include/pineforge/series.hpp": { + "bytes": 3439, + "git_blob": "61baaef2c351d37b0d6a5eb179e91ce670eb83c3", + "sha256": "e7709f67df9046c9eba163c2dbe91f8161084a955265ccf6b163355856c0afde" + }, + "include/pineforge/session_time.hpp": { + "bytes": 10337, + "git_blob": "af3a4472d852e0965947c006d0b6ad85304539e7", + "sha256": "f58c94bfae948e08fa21e52f20eae3940b31e398d83b800baf3a31f3a5fde669" + }, + "include/pineforge/str_utils.hpp": { + "bytes": 820, + "git_blob": "d44059f10b41872ea57a1abd9c246fadb3b7b3eb", + "sha256": "58e052a040da94f3824ec36db05b37dfe9e2219a32e843b56998fb1f0efe1393" + }, + "include/pineforge/ta.hpp": { + "bytes": 35301, + "git_blob": "3baeef801f44b938c16137fca5881784646760f9", + "sha256": "7221324be971ef6de425253656327395d8a1ffd081b3c1eef0c8c5b96f51e6b6" + }, + "include/pineforge/timeframe.hpp": { + "bytes": 29534, + "git_blob": "3d1bc9d495ac7e5b39ea5bae57c9d9afe580192b", + "sha256": "5e4c312da463fdcff55e0887188523c8bbf85a2ae62f3317a2b23f22fe0161e2" + }, + "include/pineforge/version.h.in": { + "bytes": 893, + "git_blob": "27118bd81741597e187e3a49d04ac201f9508011", + "sha256": "9b9e81d5e0998441a82dbe15a4495182fdfe126baea1cf26bdaaf51e491ad99f" + }, + "include/pineforge/window_sum.hpp": { + "bytes": 2828, + "git_blob": "1ea86860f6cb901fe79ff89ef46f1c77ce7e2b8c", + "sha256": "b5519f95d57d46e2a8045d380c1e1e1427ce744f4d53ee3a04ebfdaf942d3f68" + } + }, + "schemaVersion": "pineforge-settlement-abi-base-headers/v1", + "tree": "c29545d262bf6dcc22e46aa3382422f2e9c68378" +} diff --git a/tests/fixtures/settlement_cpp_abi/README.md b/tests/fixtures/settlement_cpp_abi/README.md index 77be59171..f9352c416 100644 --- a/tests/fixtures/settlement_cpp_abi/README.md +++ b/tests/fixtures/settlement_cpp_abi/README.md @@ -5,29 +5,42 @@ R2 commit `e60e57156a54bb1c74c0c0a4f5a338fd924c046d`, tree `e209c89035ec884774f8f3e68e1dc118f26aaf51`. Each entry records bytes, SHA-256 and Git blob identity. Binary libraries are **not** checked into this fixture. +`0e18690/manifest.json` similarly pins the complete 54-file header closure of +R3 commit `0e18690db3fb6bb4705841c2a86936dfa206380a`, tree +`c29545d262bf6dcc22e46aa3382422f2e9c68378`. This provider has selected settlement +and account projection, and predates the four `ReverseTo` methods. Both real +historical providers remain required; the selected-method rejection uses R2. + The new settlement checker supplements `check_native_cpp_abi.py` and `check_script_cpp_abi.py`; retain those existing checks and their old epoch and UBSan RTTI controls. No generated pairing executable is run. -## Prepare the real old provider explicitly +## Prepare the real historical providers explicitly The shared `ci_verify.py` profiles use `PINEFORGE_VERSION_SOURCE=FILE`, so verification version identity is independent of tags or checkout depth. They -fetch the pinned old commit without tags only when it is missing and reuse a -matching prepared archive on subsequent runs. For manual preparation, configure -the current build first, then fetch the old object if needed, outside CTest: +fetch each pinned old commit without tags only when it is missing and reuse +matching prepared archives on subsequent runs. For manual preparation, configure +the current build first, then fetch each old object if needed, outside CTest: ```sh git fetch --no-tags --depth=1 origin e60e57156a54bb1c74c0c0a4f5a338fd924c046d python3 scripts/prepare_settlement_cpp_abi_base.py \ --source-repo . --current-build build \ --output build/settlement-abi-base --jobs 4 +git fetch --no-tags --depth=1 origin 0e18690db3fb6bb4705841c2a86936dfa206380a +python3 scripts/prepare_settlement_cpp_abi_base.py \ + --source-repo . --current-build build \ + --output build/settlement-abi-prior --jobs 4 \ + --commit 0e18690db3fb6bb4705841c2a86936dfa206380a \ + --tree c29545d262bf6dcc22e46aa3382422f2e9c68378 \ + --header-manifest tests/fixtures/settlement_cpp_abi/0e18690/manifest.json ``` -For the sanitizer lane, replace both build paths with `build-asan`; for the +For the sanitizer lane, replace the build paths with `build-asan`; for the native-runner lane use `build-live`. The preparation script copies the actual current CMake compiler/build-type/flags/sanitizer settings, Eigen package or -already populated FetchContent source, and platform options. It builds the full pinned R2 static library +already populated FetchContent source, and platform options. It builds each full pinned static library with tests, tutorial, runner and corpus disabled. Source comes only from immutable local Git objects. Dependencies must already be available in the current build; its downloaded Eigen source is reused when needed. FetchContent stays disconnected. The script itself performs no Git/dependency @@ -47,32 +60,42 @@ python3 scripts/check_settlement_cpp_abi.py \ --compiler /path/to/current/c++ --library build/lib/libpineforge.a \ --include include --generated-include build/include \ --base-receipt build/settlement-abi-base/receipt.json \ + --prior-receipt build/settlement-abi-prior/receipt.json \ --receipt build/settlement-abi-receipt.json ``` Pass `--extra-flag=-fsanitize=address,undefined` for an instrumented archive, as the existing CMake ABI guards do. The compiler must match the current -build and the prepared base. Missing artifacts fail with the preparation +build and both prepared providers. A preserved Mac Release archive cannot stand +in for a Linux, Debug or sanitizer provider. Missing artifacts fail with the preparation command; no skip, stub or implicit network fallback is available. The checker: -* authenticates old headers and both archives, checks old defined symbols, +* authenticates both historical header inventories and all three archives, checks old defined symbols, and requires current archive freshness against all `src`, `include`, `cmake` files and `CMakeLists.txt`; -* freezes execution aggregate field/order/type/status encodings, CloseScope +* freezes execution aggregate field/order/type/status encodings, Action and CloseScope alternatives, native header contracts, engine storage declarations and virtual methods; compares actual compiler-emitted offsets/alignment/type - sizes for 251 engine data members plus returned structures; + sizes for 251 engine data members plus returned structures; separately freezes + selected/projection headers and compares their layouts against 0e18690; * compiles each old/new caller before any link result is interpreted; * links old Book/singleton/lifecycle, old private F8/F11 wrappers and old return-only host event callers against both full providers; * links the six new selected/projection methods and new private provenance helpers against the current provider, then rejects those same objects against the real R2 provider for every required method; +* links selected/projection callers built with 0e18690 headers against both + 0e18690 and current archives, plus current selected callers against 0e18690; +* links one caller for all four reversal methods against the current archive, + then rejects that same object against real 0e18690 with each exact method + name and `reverse_to_v1::ReverseTo` parameter namespace; * compiles wrong projection `_v2`/return-namespace and wrong selection-domain callers, then requires rejection at those exact symbols. Return namespace is not demanded in ordinary projection method mangling; +* rejects synthetic reversal `_v2` methods and `reverse_to_v2::ReverseTo` + parameters against current, supplementing the real historical rejection; * refuses unrelated undefined symbols, retains full compiler/link logs and records `executedBinaries: 0`. @@ -86,9 +109,10 @@ Three development modes return labeled partial receipts: `--base-only` checks old-provider controls; `--old-rejections-only` checks new public declarations against the real old provider before the new archive exists; `--public-only` runs all public pairings before new private provenance -helpers are integrated. None returns `status: passed` or -`newArchivePairingComplete: true`. Do not use either mode in CI acceptance. +helpers are integrated. These partial modes do not include the reversal matrix +and may omit `--prior-receipt`. None returns `status: passed` or +`newArchivePairingComplete: true`. Do not use these modes in CI acceptance. -The six offline Python refusal tests are run by +The offline Python refusal tests are run by `python3 scripts/test_settlement_cpp_abi.py -v`; they are tooling checks, not substitutes for the actual compiler/archive matrix. diff --git a/tests/test_native_reverse_to.cpp b/tests/test_native_reverse_to.cpp new file mode 100644 index 000000000..5ff60becd --- /dev/null +++ b/tests/test_native_reverse_to.cpp @@ -0,0 +1,524 @@ +// Literal tests of the actual resolved ReverseTo seam; no strategy/tape loop. +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge; +namespace x = pineforge::execution; +using ReverseTo = x::reverse_to_v1::ReverseTo; + +namespace { +int checks = 0, failures = 0; +const char* scenario = "setup"; +struct Abort {}; +#define CHECK(value) do { ++checks; if (!(value)) { ++failures; \ + std::printf("FAIL %s:%d %s\n", scenario, __LINE__, #value); } } while (0) +#define REQUIRE(value) do { const bool ok_ = bool(value); CHECK(ok_); \ + if (!ok_) throw Abort{}; } while (0) + +uint64_t bits(double value) { + uint64_t result; + static_assert(sizeof(result) == sizeof(value)); + std::memcpy(&result, &value, sizeof(result)); + return result; +} +void exact(double actual, double expected) { + if (bits(actual) != bits(expected)) + std::printf(" actual=%.17g (%016llx) expected=%.17g (%016llx)\n", + actual, static_cast(bits(actual)), + expected, static_cast(bits(expected))); + CHECK(bits(actual) == bits(expected)); +} +void near(double actual, double expected) { + const bool ok = std::isfinite(actual) && std::isfinite(expected) + && std::abs(actual - expected) <= 1e-12 * std::max(1.0, std::abs(expected)); + if (!ok) std::printf(" actual=%.17g expected=%.17g\n", actual, expected); + CHECK(ok); +} + +struct Book final : BacktestEngine { + Book() { + initial_capital_ = 1000; + commission_type_ = CommissionType::CASH_PER_ORDER; + commission_value_ = 0; + syminfo_.pointvalue = 1; + account_currency_fx_ = 1; + stream_observe_actions_ = true; + current_bar_ = {100, 130, 70, 100, 1, 1736121600000LL}; + bar_index_ = 3; + } + void on_bar(const Bar&) override {} + x::PhysicalExecutionContext context() const { + // Native execution coordinates intentionally differ from chart state. + return {1736121660000LL, 7, {}, {}}; + } + x::SettlementInspection inspect(double target, const x::Fill& fill) const { + return inspect_native_reversal_v1(ReverseTo{target}, fill); + } + x::AccountEffectProjection project(double target, const x::Fill& fill) const { + return project_native_reversal_v1(ReverseTo{target}, fill); + } + x::Result reverse(double target, const x::Fill& fill) { + return settle_native_reversal_at_v1(ReverseTo{target}, fill, context()); + } + x::Result reverse_with(double target, const x::Fill& fill, + const x::LifecycleEffects& effects) { + return settle_reversal_with_lifecycle_v1(ReverseTo{target}, fill, effects); + } + x::Result flow(const x::Action& action, const x::Fill& fill) { + return settle_native_execution_at(action, fill, context()); + } + void open(double quantity, double price, uint64_t incarnation, + std::optional paid = 0.0) { + const auto result = flow(order_action::Transact{quantity}, + {price, "seed-" + std::to_string(incarnation), "historical", incarnation, paid}); + REQUIRE(result.status == x::Status::Applied); + } + void schedule(CommissionType kind, double value, double pointvalue = 1, + double fx = 1) { + commission_type_ = kind; + commission_value_ = value; + syminfo_.pointvalue = pointvalue; + account_currency_fx_ = fx; + } + const auto& lots() const { return pyramid_entries_; } + auto& lots() { return pyramid_entries_; } + const auto& rows() const { return trades_; } + size_t actions() const { return stream_order_actions_.size(); } + double balance() const { return initial_capital_ + net_profit_sum_; } + double net() const { return net_profit_sum_; } + double marked(double price) const { return marked_equity(price); } + double quantity() const { + return position_side_ == PositionSide::SHORT ? -position_qty_ : position_qty_; + } + int64_t cycle() const { return position_cycle_seq_; } + int64_t next_cycle() const { return next_position_cycle_seq_; } + uint64_t next_order() const { return next_order_incarnation_; } + uint64_t event_sequence() const { return exit_leg_event_seq_; } + void next_cycle(int64_t value) { next_position_cycle_seq_ = value; } + void corrupt_side(PositionSide value) { position_side_ = value; } + void initial(double value) { initial_capital_ = value; } + void exhaust_stream() { stream_action_sequence_ = UINT64_MAX; } + void exhaust_wins() { win_trades_count_ = std::numeric_limits::max(); } + void exhaust_lifecycle() { exit_leg_event_seq_ = UINT64_MAX; } + void pending_exit() { + PendingOrder order; + order.type = OrderType::EXIT; + order.id = "retained"; + order.from_entry = "seed-11"; + order.incarnation = 700; + order.created_seq = 700; + order.legs.attach(order.incarnation, position_cycle_seq_); + pending_orders_.push_back(std::move(order)); + } + size_t pending_count() const { return pending_orders_.size(); } + const PendingOrder* pending_data() const { return pending_orders_.data(); } +}; + +x::Fill fill(double price = 100, std::optional ticket = {}) { + return {price, "reverse", "resolved once", 90, ticket}; +} +struct Snapshot { + uint64_t broker, stream, next_order, event_sequence; + int64_t cycle, next_cycle; + size_t lots, rows, actions, pending; + double balance, quantity; + const PendingOrder* pending_data; + explicit Snapshot(const Book& book) + : broker(book.broker_state_hash()), stream(book.stream_state_hash()), + next_order(book.next_order()), event_sequence(book.event_sequence()), + cycle(book.cycle()), next_cycle(book.next_cycle()), lots(book.lots().size()), + rows(book.rows().size()), actions(book.actions()), pending(book.pending_count()), + balance(book.balance()), quantity(book.quantity()), pending_data(book.pending_data()) {} + void unchanged(const Book& book) const { + CHECK(book.broker_state_hash() == broker); + CHECK(book.stream_state_hash() == stream); + CHECK(book.next_order() == next_order && book.event_sequence() == event_sequence); + CHECK(book.cycle() == cycle && book.next_cycle() == next_cycle); + CHECK(book.lots().size() == lots && book.rows().size() == rows); + CHECK(book.actions() == actions && book.pending_count() == pending); + CHECK(book.pending_data() == pending_data); + exact(book.balance(), balance); + exact(book.quantity(), quantity); + } +}; + +x::AccountEffectProjection agree_and_apply(Book& book, double target, const x::Fill& f) { + const Snapshot before(book); + const auto inspection = book.inspect(target, f); + before.unchanged(book); + const auto projection = book.project(target, f); + before.unchanged(book); + const auto repeated = book.project(target, f); + before.unchanged(book); + REQUIRE(inspection.status == x::Status::Applied); + REQUIRE(projection.status == x::Status::Applied); + REQUIRE(repeated.status == x::Status::Applied); + exact(inspection.opened_units, target); + exact(projection.opened_units, target); + exact(projection.signed_units_after, target); + exact(inspection.resulting_abs_units, std::abs(target)); + exact(projection.resulting_abs_units, std::abs(target)); + CHECK(inspection.resulting_lot_count == 1 && projection.resulting_lot_count == 1); + CHECK(inspection.would_open && projection.would_open); + CHECK(inspection.incoming_short == (target < 0)); + CHECK(projection.incoming_short == (target < 0)); + exact(inspection.closed_units, projection.closed_units); + exact(inspection.current_ticket, projection.current_ticket); + exact(inspection.resulting_abs_notional, projection.resulting_abs_notional); + exact(repeated.realized_balance, projection.realized_balance); + CHECK(projection.cycle_after == before.next_cycle); + CHECK(repeated.cycle_after == before.next_cycle); + + const auto result = book.reverse(target, f); + REQUIRE(result.status == x::Status::Applied); + REQUIRE(book.lots().size() == 1); + exact(result.closed_units, projection.closed_units); + exact(result.opened_units, target); + exact(result.current_ticket, projection.current_ticket); + exact(book.lots()[0].qty, std::abs(target)); + exact(book.quantity(), target); + exact(book.balance(), projection.realized_balance); + exact(book.marked(f.price), projection.marked_equity); + exact(book.lots()[0].entry_commission_account, projection.remaining_entry_cost); + CHECK(book.cycle() == projection.cycle_after && book.next_cycle() == before.next_cycle + 1); + CHECK(result.first_trade_index == before.rows); + CHECK(result.closed_trade_count == before.lots); + CHECK(result.opened_lot_incarnation == f.incarnation); + CHECK(book.rows().size() == before.rows + before.lots); + CHECK(book.actions() == before.actions + before.lots + 1); + CHECK(book.lots()[0].entry_incarnation == f.incarnation); + CHECK(book.lots()[0].entry_id == f.id && book.lots()[0].entry_comment == f.comment); + exact(book.lots()[0].price, f.price); + CHECK(book.lots()[0].time == book.context().effective_time_ms); + CHECK(book.lots()[0].entry_bar_index == book.context().interval_index); + for (size_t index = before.rows; index < book.rows().size(); ++index) { + CHECK(book.rows()[index].exit_time == book.context().effective_time_ms); + CHECK(book.rows()[index].exit_bar_index == book.context().interval_index); + exact(book.rows()[index].exit_price, f.price); + CHECK(book.rows()[index].exit_id == f.id && book.rows()[index].exit_comment == f.comment); + } + return projection; +} + +void exact_target(double sign) { + scenario = "ReverseTo exact non-dyadic target and one cycle per reversal"; + Book book; + book.open(sign, 100, 11); + const auto projection = agree_and_apply(book, -sign * .1, fill(100 + sign * 10)); + REQUIRE(book.rows().size() == 1); + exact(book.rows()[0].qty, 1); + near(book.rows()[0].pnl, 10); + near(projection.realized_balance, 1010); + CHECK(bits(book.lots()[0].qty) == UINT64_C(0x3fb999999999999a)); + agree_and_apply(book, sign * .2, fill(100)); + CHECK(book.cycle() == 3 && book.next_cycle() == 4); +} + +void non_dyadic_roster(double sign) { + scenario = "ReverseTo closes each non-dyadic roster member in order"; + Book book; + book.open(sign * .1, 100, 11); + book.open(sign * .2, 100, 12); + book.open(sign * .3, 100, 13); + agree_and_apply(book, -sign * .1, fill()); + REQUIRE(book.rows().size() == 3); + const double quantities[] = {.1, .2, .3}; + for (size_t index = 0; index < 3; ++index) { + CHECK(book.rows()[index].entry_incarnation == 11 + index); + exact(book.rows()[index].qty, quantities[index]); + } +} + +void absorbed_quantity(double sign, bool tiny_target) { + scenario = tiny_target ? "large held and absorbed tiny target remains representable" + : "tiny held and absorbed large target remains representable"; + const double held = tiny_target ? 1e16 : .1; + const double target = tiny_target ? .1 : 1e16; + Book book; + book.open(sign * held, 1, 11); + agree_and_apply(book, -sign * target, fill(1)); + REQUIRE(book.rows().size() == 1); + exact(book.rows()[0].qty, held); +} + +void tiny_roster_member(double sign) { + scenario = "ReverseTo whole-book close retains a tiny physical roster member"; + Book book; + book.open(sign * 1e16, 1, 11); + // Construct the valid physical roster directly: a Transact add intentionally + // cannot express this absorbed member. ReverseTo must still close both lots. + PyramidEntry tiny{1, 1736121600000LL, .1, "tiny", 3}; + tiny.entry_incarnation = 12; + tiny.entry_commission_account = 0; + book.lots().push_back(tiny); + agree_and_apply(book, -sign * .2, fill(1)); + REQUIRE(book.rows().size() == 2); + CHECK(book.rows()[0].entry_incarnation == 11 && book.rows()[1].entry_incarnation == 12); + exact(book.rows()[0].qty, 1e16); + exact(book.rows()[1].qty, .1); +} + +void one_ticket(double sign, CommissionType type, double value, + std::optional explicit_ticket, double ticket, + double first_commission, double second_commission, + double opening_paid, double balance, double equity) { + scenario = "ReverseTo one current ticket conserves historical paid entry costs"; + Book book; + book.open(sign * 2, 120 - sign * 20, 11, 4); + book.open(sign * 3, 120 - sign * 10, 12, 6); + book.schedule(type, value); + const auto projection = agree_and_apply(book, -sign * 5, fill(120, explicit_ticket)); + REQUIRE(book.rows().size() == 2); + near(projection.current_ticket, ticket); + near(book.rows()[0].commission, first_commission); + near(book.rows()[1].commission, second_commission); + near(book.lots()[0].entry_commission_account, opening_paid); + near(book.balance(), balance); + near(book.marked(120), equity); + // Existing paid costs are 10; one current ticket is shared by all effects. + near(book.rows()[0].commission + book.rows()[1].commission + + book.lots()[0].entry_commission_account, 10 + ticket); +} + +void historical_fx(double sign) { + scenario = "ReverseTo percentage fee uses current FX without repricing paid entry costs"; + Book book; + book.schedule(CommissionType::PERCENT, 1, 2, 2); + book.open(sign * 2, 120 - sign * 20, 11, std::nullopt); + book.open(sign * 3, 120 - sign * 10, 12, std::nullopt); + const double first_paid = sign > 0 ? 8 : 11.2; + const double second_paid = sign > 0 ? 13.2 : 15.6; + near(book.lots()[0].entry_commission_account, first_paid); + near(book.lots()[1].entry_commission_account, second_paid); + book.schedule(CommissionType::PERCENT, 1, 2, 3); + const auto projection = agree_and_apply(book, -sign * 5, fill(120)); + REQUIRE(book.rows().size() == 2); + near(projection.current_ticket, 72); + near(book.rows()[0].commission, first_paid + 14.4); + near(book.rows()[1].commission, second_paid + 21.6); + near(book.rows()[0].pnl, sign > 0 ? 217.6 : 214.4); + near(book.rows()[1].pnl, sign > 0 ? 145.2 : 142.8); + near(projection.remaining_entry_cost, 36); + near(projection.realized_balance, sign > 0 ? 1362.8 : 1357.2); + near(projection.marked_equity, sign > 0 ? 1326.8 : 1321.2); +} + +void invalid_inspection(const x::SettlementInspection& value, x::Status status) { + CHECK(value.status == status); + CHECK(value.closed_units == 0 && value.opened_units == 0 && value.resulting_abs_units == 0); + CHECK(value.resulting_lot_count == 0 && value.resulting_abs_notional == 0 && value.current_ticket == 0); + CHECK(!value.would_open && !value.incoming_short); +} +void refused(Book& book, double target, const x::Fill& f, x::Status status) { + const Snapshot before(book); + invalid_inspection(book.inspect(target, f), status); + before.unchanged(book); + const auto projection = book.project(target, f); + CHECK(projection.status == status); + CHECK(projection.closed_units == 0 && projection.opened_units == 0 && projection.resulting_abs_units == 0); + CHECK(projection.resulting_lot_count == 0 && projection.resulting_abs_notional == 0 && projection.current_ticket == 0); + CHECK(!projection.would_open && !projection.incoming_short); + CHECK(projection.realized_balance == 0 && projection.remaining_entry_cost == 0 && projection.marked_equity == 0); + CHECK(projection.cycle_after == 0 && projection.signed_units_after == 0); + before.unchanged(book); + const auto result = book.reverse(target, f); + CHECK(result.status == status); + CHECK(result.closed_units == 0 && result.opened_units == 0 && result.current_ticket == 0); + CHECK(result.first_trade_index == 0 && result.closed_trade_count == 0 && result.opened_lot_incarnation == 0); + before.unchanged(book); +} + +void quantity_and_book_refusals(double sign) { + scenario = "ReverseTo has no NoEffect quantity or side case"; + Book book; + book.open(sign, 100, 11); + for (double target : {0.0, -0.0, std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity(), -std::numeric_limits::infinity()}) + refused(book, target, fill(), x::Status::InvalidQuantity); + refused(book, sign * .1, fill(), x::Status::InvalidCloseTarget); + Book flat; + refused(flat, -sign * .1, fill(), x::Status::InvalidCloseTarget); + for (double invalid : {std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity()}) { + refused(book, -sign * .1, fill(invalid), x::Status::InvalidPrice); + refused(book, -sign * .1, fill(100, invalid), x::Status::InvalidAccounting); + } + scenario = "ReverseTo invalid physical book refuses before mutation"; + for (double quantity : {-1.0, 0.0, std::numeric_limits::quiet_NaN(), + std::numeric_limits::infinity()}) { + Book invalid; + invalid.open(sign, 100, 11); + invalid.lots()[0].qty = quantity; + refused(invalid, -sign * .1, fill(), x::Status::InvalidBook); + } + Book invalid_price; + invalid_price.open(sign, 100, 11); + invalid_price.lots()[0].price = std::numeric_limits::infinity(); + refused(invalid_price, -sign * .1, fill(), x::Status::InvalidBook); + Book invalid_side; + invalid_side.open(sign, 100, 11); + invalid_side.corrupt_side(static_cast(99)); + refused(invalid_side, -sign * .1, fill(), x::Status::InvalidBook); + Book absent_roster; + absent_roster.open(sign, 100, 11); + absent_roster.lots().clear(); + refused(absent_roster, -sign * .1, fill(), x::Status::InvalidBook); +} + +void overflow_refusals(double sign) { + scenario = "ReverseTo finite gross overflow precedes commission quote"; + Book gross; + const double large = std::numeric_limits::max() * .75; + gross.open(sign * large, 0, 11); + // A fee quote would itself be invalid. Quantity overflow must win first. + gross.schedule(CommissionType::CASH_PER_ORDER, std::numeric_limits::quiet_NaN()); + refused(gross, -sign * large, fill(0), x::Status::UnrepresentableQuantity); + + scenario = "ReverseTo finite component fees cannot overflow the aggregate ticket"; + Book ticket; + ticket.open(sign, 0, 11); + ticket.open(sign, 0, 12); + ticket.schedule(CommissionType::CASH_PER_CONTRACT, std::numeric_limits::max() / 2); + refused(ticket, -sign, fill(0), x::Status::InvalidAccounting); + + scenario = "ReverseTo overflowing held roster is invalid book"; + Book held; + held.open(sign * large, 0, 11); + auto copy = held.lots()[0]; + copy.entry_incarnation = 12; + held.lots().push_back(copy); + refused(held, -sign, fill(0), x::Status::InvalidBook); +} + +template void throws_atomically(Book& book, F call) { + const Snapshot before(book); + bool threw = false; + try { call(); } catch (const std::overflow_error&) { threw = true; } + CHECK(threw); + before.unchanged(book); +} +void exception_preflight() { + scenario = "ReverseTo cycle projection peeks without consuming exhausted sequence"; + for (int64_t next : {int64_t{0}, int64_t{-1}, std::numeric_limits::max()}) { + Book book; + book.open(1, 100, 11); + book.next_cycle(next); + const Snapshot before(book); + CHECK(book.inspect(-.1, fill()).status == x::Status::Applied); + before.unchanged(book); + throws_atomically(book, [&] { (void)book.project(-.1, fill()); }); + throws_atomically(book, [&] { (void)book.reverse(-.1, fill()); }); + } + scenario = "ReverseTo exhausted close counter refuses before close or opening"; + Book counter; + counter.open(1, 100, 11); + counter.exhaust_wins(); + throws_atomically(counter, [&] { (void)counter.reverse(-.1, fill(110)); }); + scenario = "ReverseTo exhausted stream refuses before close or opening"; + Book stream; + stream.open(1, 100, 11); + stream.exhaust_stream(); + throws_atomically(stream, [&] { (void)stream.reverse(-.1, fill()); }); + scenario = "ReverseTo exhausted pending lifecycle refuses atomically"; + Book lifecycle; + lifecycle.open(1, 100, 11); + lifecycle.pending_exit(); + lifecycle.exhaust_lifecycle(); + throws_atomically(lifecycle, [&] { (void)lifecycle.reverse(-.1, fill()); }); +} + +void invalid_lifecycle() { + scenario = "ReverseTo lifecycle wrapper validates batch before financial effects"; + Book book; + book.open(1, 100, 11); + book.pending_exit(); + x::LifecycleEffects effects; + effects.removals.push_back({999, 999, {}, 0}); + const Snapshot before(book); + const auto result = book.reverse_with(-.1, fill(), effects); + CHECK(result.status == x::Status::InvalidLifecycle); + CHECK(result.closed_units == 0 && result.opened_units == 0 && result.current_ticket == 0); + before.unchanged(book); +} + +void fresh_commit() { + scenario = "ReverseTo projection is read-only and does not authorize a stale close"; + Book book; + book.open(1, 100, 11); + const auto earlier = book.project(-.1, fill(110)); + REQUIRE(earlier.status == x::Status::Applied); + exact(earlier.closed_units, 1); + REQUIRE(book.flow(order_action::Reduce{.5}, fill(110, 0)).status == x::Status::Applied); + const auto fresh = agree_and_apply(book, -.1, fill(110)); + exact(fresh.closed_units, .5); + CHECK(fresh.closed_units != earlier.closed_units); + const Snapshot after(book); + CHECK(book.reverse(-.1, fill(110)).status == x::Status::InvalidCloseTarget); + after.unchanged(book); +} + +void sequential_balance() { + scenario = "ReverseTo projection preserves sequential realized PnL association"; + Book book; + book.initial(1); + const double high = 10000000000000100.0; + book.open(1, high, 11); + REQUIRE(book.flow(x::Flatten{}, fill(100, 0)).status == x::Status::Applied); + exact(book.net(), -1e16); + book.open(1, 100, 12); + book.open(.5, high - 2, 13); + const auto projection = agree_and_apply(book, -.1, fill(high, 0)); + REQUIRE(book.rows().size() == 3); + exact(book.rows()[1].pnl, 1e16); + exact(book.rows()[2].pnl, 1); + exact(book.net(), 1); + exact(projection.realized_balance, 2); + exact(projection.marked_equity, 2); +} + +template void run(F call) { + try { call(); } + catch (const Abort&) {} + catch (const std::exception& error) { + ++failures; + std::printf("FAIL %s exception: %s\n", scenario, error.what()); + } +} +} // namespace + +int main() { + static_assert(std::variant_size_v == 3); + for (double sign : {1.0, -1.0}) { + run([&] { exact_target(sign); }); + run([&] { non_dyadic_roster(sign); }); + run([&] { absorbed_quantity(sign, true); }); + run([&] { absorbed_quantity(sign, false); }); + run([&] { tiny_roster_member(sign); }); + run([&] { one_ticket(sign, CommissionType::CASH_PER_ORDER, 10, {}, 10, 6, 9, 5, 1055, 1050); }); + run([&] { one_ticket(sign, CommissionType::CASH_PER_CONTRACT, 2, {}, 20, 8, 12, 10, 1050, 1040); }); + run([&] { one_ticket(sign, CommissionType::CASH_PER_ORDER, 99, 0, 0, 4, 6, 0, 1060, 1060); }); + run([&] { one_ticket(sign, CommissionType::CASH_PER_ORDER, 99, -10, -10, 2, 3, -5, 1065, 1070); }); + run([&] { one_ticket(sign, CommissionType::CASH_PER_ORDER, -10, {}, -10, 2, 3, -5, 1065, 1070); }); + run([&] { historical_fx(sign); }); + run([&] { quantity_and_book_refusals(sign); }); + run([&] { overflow_refusals(sign); }); + } + run(exception_preflight); + run(invalid_lifecycle); + run(fresh_commit); + run(sequential_balance); + std::printf("%s native ReverseTo: %d checks, %d failures\n", + failures ? "FAIL" : "PASS", checks, failures); + return failures ? 1 : 0; +} diff --git a/tests/test_source_exact_reversal.cpp b/tests/test_source_exact_reversal.cpp new file mode 100644 index 000000000..a28863dd4 --- /dev/null +++ b/tests/test_source_exact_reversal.cpp @@ -0,0 +1,275 @@ +// Literal calls to the real F7 and F8 adapters. No run(), tape or strategy loop. +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge; +namespace x = pineforge::execution; +namespace { +int checks = 0, failures = 0; +const char* scenario = "setup"; +struct Abort {}; +#define CHECK(value) do { ++checks; if (!(value)) { ++failures; \ + std::printf("FAIL %s:%d %s\n", scenario, __LINE__, #value); } } while (0) +#define REQUIRE(value) do { const bool ok_ = bool(value); CHECK(ok_); \ + if (!ok_) throw Abort{}; } while (0) + +template struct Access { + friend auto access(Tag) { return Member; } +}; +struct FlipTag { friend auto access(FlipTag); }; +struct SequentialTag { friend auto access(SequentialTag); }; +template struct Access; +template struct Access; + +uint64_t bits(double value) { + uint64_t result; + static_assert(sizeof(result) == sizeof(value)); + std::memcpy(&result, &value, sizeof(result)); + return result; +} +void exact(double actual, double expected) { + if (bits(actual) != bits(expected)) + std::printf(" actual=%.17g (%016llx) expected=%.17g (%016llx)\n", + actual, static_cast(bits(actual)), + expected, static_cast(bits(expected))); + CHECK(bits(actual) == bits(expected)); +} +void near(double actual, double expected) { + const bool ok = std::isfinite(actual) && std::isfinite(expected) + && std::abs(actual - expected) <= 1e-12 * std::max(1.0, std::abs(expected)); + if (!ok) std::printf(" actual=%.17g expected=%.17g\n", actual, expected); + CHECK(ok); +} + +struct Book final : BacktestEngine { + Book() { + initial_capital_ = 1000; + commission_type_ = CommissionType::CASH_PER_ORDER; + commission_value_ = 0; + slippage_ = 0; + qty_step_ = 0; + pyramiding_ = 100; + syminfo_.pointvalue = 1; + syminfo_.mintick = .01; + syminfo_mintick_ = .01; + account_currency_fx_ = 1; + stream_observe_actions_ = true; + current_bar_ = {100, 130, 70, 110, 1, 1736121660000LL}; + bar_index_ = 7; + } + void on_bar(const Bar&) override {} + void open(double quantity, double price, uint64_t incarnation) { + const x::PhysicalExecutionContext context{1736121600000LL, 6, {}, {}}; + REQUIRE(settle_native_execution_at(order_action::Transact{quantity}, + x::Fill{price, "old", "historical", incarnation, 0}, context).status == x::Status::Applied); + } + void flip(bool buy, double price, double quantity, int type = -1, + bool frozen = true, bool close_only = false) { + (this->*access(FlipTag{}))(std::string("flip"), buy, price, quantity, + type, frozen, close_only, 90); + } + void sequential(bool buy, double price, double transaction) { + (this->*access(SequentialTag{}))(std::string("sequential"), buy, price, + transaction, -1, 91); + } + void fee(double value) { commission_value_ = value; } + void step(double value) { qty_step_ = value; } + void scale(double pointvalue, double fx) { + syminfo_.pointvalue = pointvalue; + account_currency_fx_ = fx; + } + void default_percent(double value) { + default_qty_type_ = QtyType::PERCENT_OF_EQUITY; + default_qty_value_ = value; + } + void already_resolved_slippage() { slippage_ = 99; } + void retain_exit() { + PendingOrder order; + order.id = "retained"; + order.from_entry = "old"; + order.type = OrderType::EXIT; + order.incarnation = 700; + order.created_seq = 700; + order.legs.attach(order.incarnation, position_cycle_seq_); + pending_orders_.push_back(std::move(order)); + } + const auto& lots() const { return pyramid_entries_; } + const auto& rows() const { return trades_; } + const auto& pending() const { return pending_orders_; } + double quantity() const { + return position_side_ == PositionSide::SHORT ? -position_qty_ : position_qty_; + } + double balance() const { return initial_capital_ + net_profit_sum_; } + double marked(double price) const { return marked_equity(price); } + int64_t cycle() const { return position_cycle_seq_; } + int64_t next_cycle() const { return next_position_cycle_seq_; } + size_t actions() const { return stream_order_actions_.size(); } +}; + +void exact_f7_and_unchanged_f8(double held_sign) { + scenario = "F7 preserves exact requested target; F8 preserves transaction remainder"; + const bool buy = held_sign < 0; + const double price = 100 + held_sign * 10; + Book flip; + flip.open(held_sign, 100, 11); + flip.retain_exit(); + const auto* pending = flip.pending().data(); + const auto next_cycle = flip.next_cycle(); + const auto actions = flip.actions(); + flip.already_resolved_slippage(); + flip.step(1); // Frozen source quantity must not be floored a second time. + flip.flip(buy, price, .1); + REQUIRE(flip.lots().size() == 1 && flip.rows().size() == 1); + CHECK(bits(flip.lots()[0].qty) == UINT64_C(0x3fb999999999999a)); + exact(flip.quantity(), -held_sign * .1); + exact(flip.rows()[0].qty, 1); + near(flip.rows()[0].pnl, 10); + exact(flip.rows()[0].exit_price, price); + exact(flip.lots()[0].price, price); + CHECK(flip.rows()[0].entry_incarnation == 11); + CHECK(flip.lots()[0].entry_incarnation == 90 && flip.lots()[0].entry_id == "flip"); + CHECK(flip.rows()[0].exit_id == "flip"); + CHECK(flip.rows()[0].exit_time == 1736121660000LL && flip.rows()[0].exit_bar_index == 7); + CHECK(flip.lots()[0].time == 1736121660000LL && flip.lots()[0].entry_bar_index == 7); + CHECK(flip.cycle() == next_cycle && flip.next_cycle() == next_cycle + 1); + CHECK(flip.actions() == actions + 2); + REQUIRE(flip.pending().size() == 1); + CHECK(flip.pending().data() == pending && flip.pending()[0].incarnation == 700); + CHECK(flip.pending()[0].legs.target().owner == flip.cycle()); + + Book class_c; + class_c.open(held_sign, 100, 11); + class_c.sequential(buy, price, 1.1); + REQUIRE(class_c.lots().size() == 1 && class_c.rows().size() == 1); + // These are the old and current F8 witness bits, not exact source Q bits. + CHECK(bits(class_c.lots()[0].qty) == UINT64_C(0x3fb99999999999a0)); + CHECK(bits(class_c.lots()[0].qty) != bits(.1)); + exact(class_c.rows()[0].qty, 1); + near(class_c.rows()[0].pnl, 10); + + Book class_b; + class_b.open(held_sign, 100, 11); + class_b.sequential(buy, price, .1); + REQUIRE(class_b.lots().empty() && class_b.rows().size() == 1); + exact(class_b.rows()[0].qty, 1); + CHECK(class_b.cycle() == 0); +} + +void f7_non_dyadic_roster(double sign) { + scenario = "F7 closes non-dyadic roster and opens exact target with one ticket"; + Book book; + book.open(sign * .1, 100, 11); + book.open(sign * .2, 100, 12); + book.open(sign * .3, 100, 13); + book.fee(6); + book.flip(sign < 0, 100, .1); + REQUIRE(book.rows().size() == 3 && book.lots().size() == 1); + exact(book.lots()[0].qty, .1); + const double quantities[] = {.1, .2, .3}; + double paid = book.lots()[0].entry_commission_account; + for (size_t index = 0; index < 3; ++index) { + CHECK(book.rows()[index].entry_incarnation == 11 + index); + exact(book.rows()[index].qty, quantities[index]); + paid += book.rows()[index].commission; + } + near(paid, 6); + near(book.marked(100), 994); +} + +void f7_absorbed_quantities(double sign, bool tiny_target) { + scenario = tiny_target ? "F7 accepts tiny target absorbed by old held quantity" + : "F7 accepts large target that absorbs old held quantity"; + const double held = tiny_target ? 1e16 : .1; + const double quantity = tiny_target ? .1 : 1e16; + Book book; + book.open(sign * held, 1, 11); + book.flip(sign < 0, 1, quantity); + REQUIRE(book.lots().size() == 1 && book.rows().size() == 1); + exact(book.lots()[0].qty, quantity); + exact(book.rows()[0].qty, held); +} + +void f7_zero_and_close_only(double sign) { + for (bool close_only : {false, true}) { + scenario = close_only ? "F7 close-only stays Flatten" : "F7 resolved zero stays Flatten"; + Book book; + book.open(sign, 100, 11); + book.retain_exit(); + const auto* pending = book.pending().data(); + const auto next_cycle = book.next_cycle(); + book.fee(6); + book.flip(sign < 0, 100 + sign * 10, close_only ? 99 : 0, -1, true, close_only); + REQUIRE(book.rows().size() == 1 && book.lots().empty()); + exact(book.rows()[0].qty, 1); + near(book.rows()[0].commission, 6); + CHECK(book.cycle() == 0 && book.next_cycle() == next_cycle); + REQUIRE(book.pending().size() == 1); + CHECK(book.pending().data() == pending && book.pending()[0].legs.target().owner == 0); + } +} + +void f7_cash_size(double sign) { + scenario = "F7 cash sizing preserves pointvalue and current FX conversion"; + Book book; + book.scale(2, 2); + book.open(sign * 3, 100, 11); + book.fee(6); + book.flip(sign < 0, 100, 1000, static_cast(QtyType::CASH), false); + REQUIRE(book.lots().size() == 1 && book.rows().size() == 1); + exact(book.quantity(), -sign * 2.5); + near(book.rows()[0].commission + book.lots()[0].entry_commission_account, 6); + near(book.marked(100), 994); +} + +void f7_projected_percent(bool use_default) { + scenario = use_default ? "F7 default percent keeps Flatten-projected sizing" + : "F7 explicit percent keeps Flatten-projected sizing"; + Book book; + book.open(1, 100, 11); + book.open(3, 100, 12); + book.fee(6); + book.default_percent(50); + book.flip(false, 110, use_default ? std::numeric_limits::quiet_NaN() : 50, + use_default ? -1 : static_cast(QtyType::PERCENT_OF_EQUITY), false); + REQUIRE(book.lots().size() == 1 && book.rows().size() == 2); + // Closing 4 @110 first quotes a balance of 1034 for sizing. Half at110 is4.7. + exact(book.lots()[0].qty, 4.7000000000000002); + near(book.rows()[0].commission, .68965517241379315); + near(book.rows()[1].commission, 2.0689655172413794); + near(book.lots()[0].entry_commission_account, 3.2413793103448274); + near(book.balance(), 1037.2413793103448); + near(book.marked(110), 1034); +} + +template void run(F call) { + try { call(); } + catch (const Abort&) {} + catch (const std::exception& error) { + ++failures; + std::printf("FAIL %s exception: %s\n", scenario, error.what()); + } +} +} // namespace + +int main() { + for (double sign : {1.0, -1.0}) { + run([&] { exact_f7_and_unchanged_f8(sign); }); + run([&] { f7_non_dyadic_roster(sign); }); + run([&] { f7_absorbed_quantities(sign, true); }); + run([&] { f7_absorbed_quantities(sign, false); }); + run([&] { f7_zero_and_close_only(sign); }); + run([&] { f7_cash_size(sign); }); + } + run([] { f7_projected_percent(false); }); + run([] { f7_projected_percent(true); }); + std::printf("%s source exact reversal: %d checks, %d failures\n", + failures ? "FAIL" : "PASS", checks, failures); + return failures ? 1 : 0; +} diff --git a/tests/test_source_fifo_endpoints.cpp b/tests/test_source_fifo_endpoints.cpp new file mode 100644 index 000000000..1a096394f --- /dev/null +++ b/tests/test_source_fifo_endpoints.cpp @@ -0,0 +1,509 @@ +// Literal source FIFO endpoint calls paired with the actual native Reduce owner. +// No BacktestEngine::run(), generated strategy, tape, corpus or grading loop. +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace pineforge; +namespace x = pineforge::execution; + +namespace { +int checks = 0, failures = 0; +const char* scenario = "setup"; +struct Abort {}; +#define CHECK(value) do { ++checks; if (!(value)) { ++failures; \ + std::printf("FAIL %s:%d %s\n", scenario, __LINE__, #value); } } while (0) +#define REQUIRE(value) do { ++checks; if (!(value)) { ++failures; \ + std::printf("FAIL %s:%d %s\n", scenario, __LINE__, #value); throw Abort{}; } } while (0) + +template struct Access { friend auto access(Tag) { return Member; } }; +#define ACCESS(Tag, Member) struct Tag { friend auto access(Tag); }; \ + template struct Access +ACCESS(Partial, execute_partial_exit_qty); +ACCESS(ByQuantity, execute_partial_exit_by_entry_qty); +ACCESS(Drain, fifo_drain); +ACCESS(ComputeClose, compute_close_target_qty); +ACCESS(ImmediateClose, execute_immediate_close); +ACCESS(ExitFill, apply_exit_order_fill); +#undef ACCESS +template struct Args; +template struct Args { using tuple = std::tuple; }; +using Cause = std::tuple_element_t<2, typename Args::tuple>; +constexpr Cause Script = static_cast(0), Bracket = static_cast(1), Margin = static_cast(2); + +uint64_t bits(double value) { + uint64_t result; + static_assert(sizeof(result) == sizeof(value)); + std::memcpy(&result, &value, sizeof(result)); + return result; +} +void exact(double actual, double expected) { + if (bits(actual) != bits(expected)) + std::printf(" actual=%.17g (%016llx) expected=%.17g (%016llx)\n", + actual, static_cast(bits(actual)), + expected, static_cast(bits(expected))); + CHECK(bits(actual) == bits(expected)); +} +void near(double actual, double expected) { + const bool ok = std::isfinite(actual) && std::isfinite(expected) + && std::abs(actual - expected) <= 1e-12 * std::max(1.0, std::abs(expected)); + if (!ok) std::printf(" actual=%.17g expected=%.17g\n", actual, expected); + CHECK(ok); +} + +struct Book final : BacktestEngine { + Book() { + initial_capital_ = 1000; + commission_type_ = CommissionType::CASH_PER_ORDER; + commission_value_ = 0; + syminfo_.pointvalue = 1; + syminfo_.mintick = .01; + syminfo_mintick_ = .01; + account_currency_fx_ = 1; + qty_step_ = 0; + slippage_ = 0; + pyramiding_ = 100; + stream_observe_actions_ = true; + bar(1, 100); + } + void on_bar(const Bar&) override {} + void bar(int index, double price) { + current_bar_ = {price, price + 20, price - 20, price, 1, 1736121600000LL + index * 60000}; + bar_index_ = index; + } + x::PhysicalExecutionContext context() const { return {current_bar_.timestamp, bar_index_, {}, {}}; } + void open(double quantity, double price, uint64_t incarnation, const char* label, + std::optional paid = 0.0) { + REQUIRE(settle_native_execution_at(order_action::Transact{quantity}, + x::Fill{price, label, "historical", incarnation, paid}, context()).status == x::Status::Applied); + } + void source(double quantity, double price = 100, Cause cause = Script) { + (this->*access(Partial{}))(price, quantity, cause); + } + void native(double quantity, double price = 100) { + REQUIRE(settle_native_execution_at(order_action::Reduce{quantity}, + x::Fill{price, {}, {}, 0}, context()).status == x::Status::Applied); + } + void by_entry(double quantity, double price = 100) { + (this->*access(ByQuantity{}))(price, std::string("prefix"), quantity, Script); + } + void compatibility_drain(double quantity, double price = 100) { + (void)(this->*access(Drain{}))(nullptr, quantity, price, position_side_ == PositionSide::LONG); + } + void close_logical(const char* id) { + const double na = std::numeric_limits::quiet_NaN(); + double matching = 0, quantity = 0, retired = 0; + bool all = false; + REQUIRE((this->*access(ComputeClose{}))(std::string(id), na, na, false, + matching, quantity, all, retired)); + exact(matching, .8); + exact(quantity, .8); + CHECK(!all && retired == 0); + (this->*access(ImmediateClose{}))(std::string(id), std::string("funded close"), + quantity, matching, false, true, false, false, false); + } + void exit(PendingOrder& order, double price) { + int closed_bar = -1; + uint64_t closed_incarnation = 0; + bool closed_long = false; + (this->*access(ExitFill{}))(order, price, closed_bar, closed_incarnation, closed_long); + CHECK(closed_bar == -1 && closed_incarnation == 0 && !closed_long); + } + void schedule(CommissionType kind, double value, double pointvalue = 1, double fx = 1) { + commission_type_ = kind; + commission_value_ = value; + syminfo_.pointvalue = pointvalue; + account_currency_fx_ = fx; + } + void funded(const char* id, double quantity) { id_unclosed_qty_[id] = quantity; } + bool funded(const char* id) const { return id_unclosed_qty_.count(id) != 0; } + void slots(int count) { position_entry_count_ = count; } + int slots() const { return position_entry_count_; } + void slippage(int ticks, double tick) { slippage_ = ticks; syminfo_mintick_ = tick; syminfo_.mintick = tick; } + const auto& lots() const { return pyramid_entries_; } + auto& lots() { return pyramid_entries_; } + const auto& rows() const { return trades_; } + size_t actions() const { return stream_order_actions_.size(); } + int64_t cycle() const { return position_cycle_seq_; } + int64_t next_cycle() const { return next_position_cycle_seq_; } + void cycle_for_fixture(int64_t value) { position_cycle_seq_ = value; } + double quantity() const { return position_qty_; } + double balance() const { return initial_capital_ + net_profit_sum_; } + int index() const { return bar_index_; } + int64_t timestamp() const { return current_bar_.timestamp; } + uint64_t broker_fills() const { return broker_fill_event_seq_; } +}; + +void same_lot(const PyramidEntry& actual, const PyramidEntry& expected) { + exact(actual.qty, expected.qty); + exact(actual.price, expected.price); + exact(actual.entry_commission_account, expected.entry_commission_account); + exact(actual.max_runup, expected.max_runup); + exact(actual.max_drawdown, expected.max_drawdown); + exact(actual.entry_path_position, expected.entry_path_position); + CHECK(actual.time == expected.time && actual.entry_bar_index == expected.entry_bar_index); + CHECK(actual.entry_id == expected.entry_id && actual.entry_comment == expected.entry_comment); + CHECK(actual.entry_incarnation == expected.entry_incarnation); + CHECK(actual.skip_entry_bar_high == expected.skip_entry_bar_high); + CHECK(actual.skip_entry_bar_low == expected.skip_entry_bar_low); + CHECK(actual.market_pyramid_add == expected.market_pyramid_add); + CHECK(actual.bracket_slot_shadowed == expected.bracket_slot_shadowed); + CHECK(actual.ordinary_market_open == expected.ordinary_market_open); + CHECK(actual.pooc_terminal_market_entry == expected.pooc_terminal_market_entry); + CHECK(actual.ordinary_stop_open == expected.ordinary_stop_open); +} +void same_row(const Trade& actual, const Trade& expected) { + exact(actual.qty, expected.qty); + exact(actual.entry_price, expected.entry_price); + exact(actual.exit_price, expected.exit_price); + exact(actual.pnl, expected.pnl); + exact(actual.pnl_pct, expected.pnl_pct); + exact(actual.commission, expected.commission); + exact(actual.max_runup, expected.max_runup); + exact(actual.max_drawdown, expected.max_drawdown); + CHECK(actual.entry_time == expected.entry_time && actual.exit_time == expected.exit_time); + CHECK(actual.entry_bar_index == expected.entry_bar_index && actual.exit_bar_index == expected.exit_bar_index); + CHECK(actual.entry_incarnation == expected.entry_incarnation && actual.is_long == expected.is_long); + CHECK(actual.entry_id == expected.entry_id && actual.exit_id == expected.exit_id); + CHECK(actual.entry_comment == expected.entry_comment && actual.exit_comment == expected.exit_comment); + CHECK(actual.exit_from_bracket == expected.exit_from_bracket && actual.open_at_end == expected.open_at_end); +} +void same_native_effects(const Book& source, const Book& native) { + REQUIRE(source.rows().size() == native.rows().size()); + REQUIRE(source.lots().size() == native.lots().size()); + for (size_t index = 0; index < source.rows().size(); ++index) same_row(source.rows()[index], native.rows()[index]); + for (size_t index = 0; index < source.lots().size(); ++index) same_lot(source.lots()[index], native.lots()[index]); + exact(source.quantity(), native.quantity()); + exact(source.balance(), native.balance()); + CHECK(source.cycle() == native.cycle() && source.next_cycle() == native.next_cycle()); + CHECK(source.actions() == native.actions()); +} +void seed(Book& book, double sign, const std::vector& identities = {11, 12, 13}, + const char* sibling_label = "sibling") { + REQUIRE(identities.size() == 3); + book.open(sign * .7, 100, identities[0], "prefix", 7); + book.bar(2, 100); + book.open(sign * .1, 100, identities[1], "prefix", 1); + book.bar(3, 100); + book.open(sign, 100, identities[2], sibling_label, 17); + book.lots().back().max_runup = .123; + book.lots().back().max_drawdown = .456; + book.lots().back().entry_path_position = .75; + book.lots().back().skip_entry_bar_low = true; + book.bar(7, 100); +} + +void endpoint(double sign, double request) { + scenario = "source complete FIFO endpoint closes exact prefix and preserves sibling"; + Book book; + seed(book, sign); + book.schedule(CommissionType::CASH_PER_ORDER, 6); + const auto sibling = book.lots()[2]; + const auto cycle = book.cycle(), next_cycle = book.next_cycle(); + const auto actions = book.actions(); + const auto time = book.timestamp(); + book.source(request); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 1); + exact(book.rows()[0].qty, .7); + exact(book.rows()[1].qty, .1); + CHECK(book.rows()[0].entry_incarnation == 11 && book.rows()[1].entry_incarnation == 12); + same_lot(book.lots()[0], sibling); + exact(book.quantity(), 1); + CHECK(book.cycle() == cycle && book.next_cycle() == next_cycle); + CHECK(book.actions() == actions + 2); + CHECK(book.timestamp() == time && book.index() == 7); + CHECK(book.slots() == 1); + for (const auto& row : book.rows()) { + CHECK(row.exit_time == time && row.exit_bar_index == 7); + exact(row.exit_price, 100); + } + near(book.rows()[0].commission + book.rows()[1].commission, 14); // paid8 + one current6 + near(book.balance(), 986); +} + +void reduce_control(double sign, double request) { + scenario = "real interior source quantity retains original native Reduce effects"; + Book source, native; + seed(source, sign); + seed(native, sign); + source.schedule(CommissionType::CASH_PER_ORDER, 6); + native.schedule(CommissionType::CASH_PER_ORDER, 6); + source.source(request); + native.native(request); + same_native_effects(source, native); + if (request < .8) { + REQUIRE(source.lots().size() == 2 && source.rows().size() == 2); + CHECK(source.lots()[0].qty > 1e-10 && source.lots()[0].entry_incarnation == 12); + exact(source.lots()[1].qty, 1); + } else { + REQUIRE(source.lots().size() == 1 && source.rows().size() == 3); + CHECK(source.rows()[2].qty > 0 && source.rows()[2].entry_incarnation == 13); + CHECK(source.lots()[0].qty < 1); + } +} + +void native_spill_and_scope_walls(double sign) { + scenario = "native Reduce and retained compatibility drain preserve exact spill"; + Book native, drain; + seed(native, sign); + seed(drain, sign); + native.native(.8); + drain.compatibility_drain(.8); + same_native_effects(drain, native); + REQUIRE(native.rows().size() == 3 && native.lots().size() == 1); + exact(native.rows()[2].qty, 1.1102230246251565e-16); + exact(native.lots()[0].qty, .99999999999999989); + CHECK(native.rows()[2].entry_incarnation == 13); + + scenario = "entry-scoped interior dust stays outside source FIFO translation"; + Book scoped; + scoped.open(sign, 100, 11, "prefix", 6); + scoped.open(sign * 3, 100, 12, "sibling", 17); + const auto sibling = scoped.lots()[1]; + const double request = 1 - 5e-11; + scoped.by_entry(request); + REQUIRE(scoped.rows().size() == 1 && scoped.lots().size() == 2); + exact(scoped.rows()[0].qty, request); + CHECK(scoped.lots()[0].qty > 0 && scoped.lots()[0].qty < 1e-10); + same_lot(scoped.lots()[1], sibling); +} + +void identity_fallback(double sign, const std::vector& identities) { + scenario = "unowned or split physical identities fall back without scope expansion"; + Book source, native; + seed(source, sign, identities); + seed(native, sign, identities); + source.source(.8); + native.native(.8); + same_native_effects(source, native); + REQUIRE(source.rows().size() == 3 && source.lots().size() == 1); + exact(source.rows()[2].qty, 1.1102230246251565e-16); + CHECK(source.lots()[0].entry_incarnation == identities[2]); +} + +void complete_fragments(double sign) { + scenario = "complete repeated identity inside prefix is selected once and closes all fragments"; + Book book; + seed(book, sign, {11, 11, 13}); + const auto sibling = book.lots()[2]; + book.source(.8); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 1); + CHECK(book.rows()[0].entry_incarnation == 11 && book.rows()[1].entry_incarnation == 11); + exact(book.rows()[0].qty, .7); + exact(book.rows()[1].qty, .1); + same_lot(book.lots()[0], sibling); +} + +void unavailable_selection_cycle(double sign) { + scenario = "unavailable selected opening cycle keeps scalar source behavior"; + Book source, native; + seed(source, sign); + seed(native, sign); + source.cycle_for_fixture(0); + native.cycle_for_fixture(0); + source.source(.8); + native.native(.8); + same_native_effects(source, native); + REQUIRE(source.rows().size() == 3 && source.lots().size() == 1); + exact(source.rows()[2].qty, 1.1102230246251565e-16); +} + +void stop_before_tiny_sibling(double sign) { + scenario = "source stops before next sibling without using its tiny size"; + Book book; + book.open(sign * .7, 100, 11, "prefix"); + book.open(sign * .1, 100, 12, "prefix"); + book.open(sign * 5e-11, 100, 13, "tiny-unselected", .25); + book.open(sign, 100, 14, "later", 17); + const auto tiny = book.lots()[2], later = book.lots()[3]; + book.source(.8); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 2); + same_lot(book.lots()[0], tiny); + same_lot(book.lots()[1], later); +} + +void logical_funding_physical_fifo(double sign) { + scenario = "logical close funding selects oldest physical FIFO identities"; + Book book; + seed(book, sign, {11, 12, 13}, "L5"); + // The logical credit has diverged from L5's live physical1 after prior + // default-FIFO attribution. It funds .8 while the old prefix bears other IDs. + book.funded("L5", .8); + const auto sibling = book.lots()[2]; + const auto fills = book.broker_fills(); + book.close_logical("L5"); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 1); + CHECK(book.rows()[0].entry_incarnation == 11 && book.rows()[1].entry_incarnation == 12); + CHECK(book.rows()[0].entry_id == "prefix" && book.rows()[1].entry_id == "prefix"); + CHECK(book.rows()[0].exit_id == "__close__L5" && book.rows()[1].exit_id == "__close__L5"); + CHECK(book.rows()[0].exit_comment == "funded close" && book.rows()[1].exit_comment == "funded close"); + CHECK(!book.funded("L5")); + CHECK(book.broker_fills() == fills + 1); + same_lot(book.lots()[0], sibling); +} + +void frozen_reservation(double sign) { + scenario = "priced frozen reservation closes old prefix and preserves newer sibling"; + Book book; + book.open(sign * .7, 100, 11, "old-first", 7); + book.bar(2, 100); + book.open(sign * .1, 100, 12, "old-second", 1); + PendingOrder order; + order.type = OrderType::EXIT; + order.id = "frozen-basket"; + order.from_entry = ""; + order.qty = .8; + order.qty_percent = 100; + order.incarnation = 90; + order.created_seq = 90; + order.created_bar = 3; + order.created_position_side = sign > 0 ? PositionSide::LONG : PositionSide::SHORT; + order.created_position_cycle_seq = book.cycle(); + order.quantity_request.request(QuantityIntent::units(.8)); + order.quantity_request.reserve(.8, .8); + order.legs.set_limit_price(100); + book.bar(4, 100); + book.open(sign, 100, 13, "newer", 17); + const auto newer = book.lots()[2]; + const auto cycle = book.cycle(), next_cycle = book.next_cycle(); + book.bar(7, 100); + book.schedule(CommissionType::CASH_PER_ORDER, 6); + book.slots(9); + book.exit(order, 100); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 1); + exact(book.rows()[0].qty, .7); + exact(book.rows()[1].qty, .1); + same_lot(book.lots()[0], newer); + exact(order.qty, .8); + CHECK(book.cycle() == cycle && book.next_cycle() == next_cycle && book.slots() == 9); + CHECK(book.rows()[0].exit_time == book.timestamp() && book.rows()[1].exit_time == book.timestamp()); + near(book.rows()[0].commission + book.rows()[1].commission, 14); +} + +void fee_schedule(double sign, CommissionType type, double fee, + double expected_first, double expected_second, double expected_balance) { + scenario = "source prefix uses one existing commission quote and full paid historical costs"; + Book book; + seed(book, sign); + const auto sibling = book.lots()[2]; + book.schedule(type, fee); + // Wipe-side endpoint must realize full .1 and all historical cost1. + book.source(.8 - 5e-11); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 1); + exact(book.rows()[0].qty, .7); + exact(book.rows()[1].qty, .1); + near(book.rows()[0].commission, expected_first); + near(book.rows()[1].commission, expected_second); + near(book.balance(), expected_balance); + same_lot(book.lots()[0], sibling); +} + +void historical_fx(double sign) { + scenario = "prefix Flatten realizes historical percentage costs at original FX"; + Book book; + book.schedule(CommissionType::PERCENT, 1, 2, 2); + book.open(sign * .7, 100, 11, "prefix", std::nullopt); + book.open(sign * .1, 100, 12, "prefix", std::nullopt); + book.open(sign, 100, 13, "sibling", std::nullopt); + near(book.lots()[0].entry_commission_account, 2.8); + near(book.lots()[1].entry_commission_account, .4); + const auto sibling = book.lots()[2]; + book.schedule(CommissionType::PERCENT, 1, 2, 3); + book.source(.8 - 5e-11); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 1); + near(book.rows()[0].commission, 7); + near(book.rows()[1].commission, 1); + near(book.balance(), 992); + same_lot(book.lots()[0], sibling); +} + +void source_slots_price_and_clock(double sign, Cause cause) { + scenario = "prefix settlement retains source slot policy and applies slippage once"; + Book book; + seed(book, sign); + const auto sibling = book.lots()[2]; + const auto cycle = book.cycle(), next_cycle = book.next_cycle(); + const auto timestamp = book.timestamp(); + book.slots(9); + book.slippage(2, .25); + book.source(.8, 110, cause); + REQUIRE(book.rows().size() == 2 && book.lots().size() == 1); + CHECK(book.slots() == (cause == Bracket ? 9 : 1)); + CHECK(book.cycle() == cycle && book.next_cycle() == next_cycle); + CHECK(book.timestamp() == timestamp && book.index() == 7); + same_lot(book.lots()[0], sibling); + for (const auto& row : book.rows()) { + exact(row.exit_price, 110 - sign * .5); + CHECK(row.exit_time == timestamp && row.exit_bar_index == 7); + } +} + +void whole_book_and_noop(double sign) { + scenario = "existing whole-book source endpoint promotion remains unchanged"; + Book book; + seed(book, sign); + const auto next = book.next_cycle(); + book.source(book.quantity() - 5e-11); + REQUIRE(book.rows().size() == 3 && book.lots().empty()); + exact(book.rows()[0].qty, .7); + exact(book.rows()[1].qty, .1); + exact(book.rows()[2].qty, 1); + CHECK(book.cycle() == 0 && book.next_cycle() == next); + + scenario = "empty sub-epsilon source prefix remains a no-op"; + Book noop; + seed(noop, sign); + const auto broker = noop.broker_state_hash(), stream = noop.stream_state_hash(); + const auto actions = noop.actions(); + noop.source(5e-11); + CHECK(noop.broker_state_hash() == broker && noop.stream_state_hash() == stream); + CHECK(noop.rows().empty() && noop.lots().size() == 3 && noop.actions() == actions); +} + +template void run(F call) { + try { call(); } + catch (const Abort&) {} + catch (const std::exception& error) { + ++failures; + std::printf("FAIL %s exception: %s\n", scenario, error.what()); + } +} +} // namespace + +int main() { + static_assert(std::variant_size_v == 3); + for (double sign : {1.0, -1.0}) { + for (double quantity : {.8, .8 - 5e-11, .8 + 5e-11}) + run([&] { endpoint(sign, quantity); }); + for (double quantity : {.8 - 2e-10, .8 + 1e-10, .75, .85}) + run([&] { reduce_control(sign, quantity); }); + run([&] { native_spill_and_scope_walls(sign); }); + for (const auto& identities : {std::vector{0, 0, 13}, {0, 12, 13}, {11, 0, 13}, {11, 12, 11}}) + run([&] { identity_fallback(sign, identities); }); + run([&] { complete_fragments(sign); }); + run([&] { unavailable_selection_cycle(sign); }); + run([&] { stop_before_tiny_sibling(sign); }); + run([&] { logical_funding_physical_fifo(sign); }); + run([&] { frozen_reservation(sign); }); + run([&] { fee_schedule(sign, CommissionType::CASH_PER_ORDER, 6, 12.25, 1.75, 986); }); + run([&] { fee_schedule(sign, CommissionType::CASH_PER_ORDER, -6, 1.75, .25, 998); }); + run([&] { fee_schedule(sign, CommissionType::CASH_PER_CONTRACT, 2, 8.4, 1.2, 990.4); }); + run([&] { fee_schedule(sign, CommissionType::PERCENT, 1, 7.7, 1.1, 991.2); }); + run([&] { historical_fx(sign); }); + for (Cause cause : {Script, Bracket, Margin}) + run([&] { source_slots_price_and_clock(sign, cause); }); + run([&] { whole_book_and_noop(sign); }); + } + std::printf("%s source FIFO endpoints: %d checks, %d failures\n", + failures ? "FAIL" : "PASS", checks, failures); + return failures ? 1 : 0; +}