From f56da3dd345e43a32c02448719dcb85eff9be483 Mon Sep 17 00:00:00 2001 From: Andriy Date: Mon, 14 Sep 2026 11:59:22 +0100 Subject: [PATCH 1/3] fixed-rate graphs: timer, Versioned history, yield points, default priority A subsystem on its own clock becomes a second compiled graph beside the frame graph, bounded by a Deferred for inputs and a Versioned for outputs. The library supplies the mechanisms; the composition stays in samples. - ts/timer.h, src/timer.cpp: ts::sleep, ts::sleep_until and ts::Periodic. One lazily created timer thread with a min-heap delivers each wakeup as a task at the sleep's priority, never inline. Cancellation settles promptly. An armed sleep holds an External_wait. On Windows the thread waits on a high-resolution waitable timer. Worker-less mode is fatal, and destroy_scheduler stops the thread first (a live sleep is fatal under TS_SAFETY_CHECKS). - Versioned: a third replica rotated at the swap, publish stamps, copy resync by default (replay is a construction fatal). read_last_versions() returns a Version_view, the read grant, lent when the context already grants the front; co_await ts::read_last_versions(v) is the awaitable form. Access_awaiter's resume bookkeeping is factored into finish_acquire(). - ts::yield(): when a high task is queued, runs it inline on the yielding worker's stack under a scope that resets the thread's task state, then returns. One relaxed load otherwise. parallel_for yields at every chunk claim, and the trace excludes the nested span from the yielding body's time. - Static_task_graph::set_default_priority. - sample/fixed_rate.cpp (--fixed-rate [n]) and game_frame's fixed_rate variant: physics on a 60 Hz graph and networking on a 30 Hz graph beside the frame, with a third --trace SVG and --bench rows. Tests: a timer group, plus versioned, yield, parallel_for, graph and integration additions, and a death scenario for each new fatal. Negative checks were done for the yield path, the rotation and the External_wait. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QTtvN1mH7K6MWzpYiyJ88w --- CMakeLists.txt | 3 + benchmarks/game_frame_bench.cpp | 9 + include/ts/coroutine_support.h | 9 +- include/ts/detail/journal.h | 3 +- include/ts/detail/task_block.h | 10 + include/ts/detail/trace_owner.h | 9 + include/ts/parallel_for.h | 14 + include/ts/recorder.h | 4 +- include/ts/scheduler.h | 5 + include/ts/static_task_graph.h | 11 +- include/ts/task.h | 14 + include/ts/timer.h | 88 ++++++ include/ts/ts.h | 1 + include/ts/versioned.h | 347 +++++++++++++++++++++-- macrame.vcxproj | 2 + macrame.vcxproj.filters | 6 + macrame_playground.vcxproj | 3 + macrame_playground.vcxproj.filters | 9 + sample/fixed_rate.cpp | 425 +++++++++++++++++++++++++++++ sample/game_frame.cpp | 301 ++++++++++++++++++-- src/guarded.cpp | 11 + src/main.cpp | 15 +- src/scheduler.cpp | 91 ++++++ src/static_task_graph.cpp | 14 + src/timer.cpp | 370 +++++++++++++++++++++++++ tests/graph_tests.cpp | 21 ++ tests/integration_tests.cpp | 43 +++ tests/parallel_tests.cpp | 46 ++++ tests/scheduler_tests.cpp | 98 +++++++ tests/tests.cpp | 67 +++++ tests/timer_tests.cpp | 154 +++++++++++ tests/timer_tests.h | 3 + tests/versioned_tests.cpp | 166 +++++++++++ tsan/tsan_main.cpp | 26 ++ 34 files changed, 2347 insertions(+), 51 deletions(-) create mode 100644 include/ts/timer.h create mode 100644 sample/fixed_rate.cpp create mode 100644 src/timer.cpp create mode 100644 tests/timer_tests.cpp create mode 100644 tests/timer_tests.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 52724c8..073c26a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,7 @@ set(TS_CORE_SOURCES src/guarded.cpp src/scheduler.cpp src/static_task_graph.cpp + src/timer.cpp src/worker_thread.cpp ) @@ -48,6 +49,7 @@ set(TS_SAMPLE_SOURCES sample/blackboard.cpp sample/coloring.cpp sample/events.cpp + sample/fixed_rate.cpp sample/game_frame.cpp sample/lazy_bvh.cpp sample/physics.cpp @@ -80,6 +82,7 @@ set(TS_DRIVER_SOURCES tests/scheduler_tests.cpp tests/task_tests.cpp tests/tests.cpp + tests/timer_tests.cpp tests/versioned_tests.cpp ) diff --git a/benchmarks/game_frame_bench.cpp b/benchmarks/game_frame_bench.cpp index df94600..0f438c1 100644 --- a/benchmarks/game_frame_bench.cpp +++ b/benchmarks/game_frame_bench.cpp @@ -12,6 +12,7 @@ namespace sample { void game_frame_stats(int frames, float time_scale, double& avg_ms, double& serial_ms, float& transform0); void game_frame_free_stats(int frames, float time_scale, double& avg_ms, double& serial_ms, float& transform0); +void game_frame_fixed_stats(int frames, float time_scale, double& avg_ms, double& serial_ms, float& transform0); } namespace @@ -70,4 +71,12 @@ void run_game_frame_bench() report_frame("free 1.0", heavy_free, heavy_graph); report_frame("graph .05", light_graph, 0.0); report_frame("free .05", light_free, light_graph); + + // The optimised frame with physics and networking on their own clocks: a different + // composition, not a different schedule of the same one, so the delta mixes the optimised + // levers with the physics chain leaving the frame. The trace separates the two. + double heavy_fixed = frame_us(&sample::game_frame_fixed_stats, 20, 1.0f); + double light_fixed = frame_us(&sample::game_frame_fixed_stats, 200, 0.05f); + report_frame("fixed 1.0", heavy_fixed, heavy_graph); + report_frame("fixed .05", light_fixed, light_graph); } diff --git a/include/ts/coroutine_support.h b/include/ts/coroutine_support.h index f2ea63f..ef89c03 100644 --- a/include/ts/coroutine_support.h +++ b/include/ts/coroutine_support.h @@ -876,7 +876,9 @@ struct Access_awaiter return true; // suspended; on_acquired will resume when the pipe grants } - Access_guard await_resume() noexcept + // The acquire's bookkeeping, undone once the grant is ours: the wait edge and the + // suspension record. Shared with the awaiters that resume into another guard type. + void finish_acquire() noexcept { #if TS_RULE_ON(TS_RULE_CIRCULAR_WAIT) if (recorded_) @@ -889,6 +891,11 @@ struct Access_awaiter registered_ = false; } #endif + } + + Access_guard await_resume() noexcept + { + finish_acquire(); return Access_guard(scheduler_, pipe_, obj_); // prvalue -> elided into the local } diff --git a/include/ts/detail/journal.h b/include/ts/detail/journal.h index c81a4f0..78bc160 100644 --- a/include/ts/detail/journal.h +++ b/include/ts/detail/journal.h @@ -26,7 +26,8 @@ namespace ts // dynamic stage-vs-cut race). template class Deferred; -template class Versioned; +enum class History; // versioned.h +template class Versioned; namespace detail { diff --git a/include/ts/detail/task_block.h b/include/ts/detail/task_block.h index d04d444..05a1a0f 100644 --- a/include/ts/detail/task_block.h +++ b/include/ts/detail/task_block.h @@ -82,6 +82,16 @@ struct Task_control_block; // other diagnostics. [[noreturn]] void escaped_exception_diagnose(const char* what) noexcept; +// `Priority::high` entries currently queued, maintained by the scheduler: incremented before +// the push and decremented after a successful pop, so it never under-counts a queued entry. +// `ts::yield()` reads it relaxed - the whole cost of a yield point with nothing pending. +inline std::atomic high_queued{ 0 }; + +// The slow half of a yield point (defined in scheduler.cpp): on a worker, when the caller runs +// below `high` (`own`), pop one queued `high` entry and run it on this thread, then return. +// A no-op otherwise. +void yield_to_high(Priority own) noexcept; + #if TS_RULE_ON(TS_RULE_DEADLOCK_NET) // Work that only a non-worker thread can complete, currently outstanding (see // `ts::External_wait`). The deadlock net's second predicate: quiescence with a nonzero count diff --git a/include/ts/detail/trace_owner.h b/include/ts/detail/trace_owner.h index fe6150a..0926e3b 100644 --- a/include/ts/detail/trace_owner.h +++ b/include/ts/detail/trace_owner.h @@ -67,6 +67,11 @@ struct Trace_owner_state : Tls_scalar {}; // reads it to book orchestration only for a top-level `execute()` (not one nested inside a body). struct In_functor_state : Tls_scalar {}; +// Ticks this thread has spent in tasks run inside yield points (`ts::yield`), a running sum. A +// `Trace_busy_scope` subtracts its growth over the scope, so a yielding body is credited only +// its own time; the nested task's own scope credits the rest. +struct Nested_span_state : Tls_scalar {}; + inline int trace_owner() noexcept { return Trace_owner_state::load(); } // Set the owning node for a scope (save/restore, so inline-nested runs and inherited @@ -103,6 +108,7 @@ class Trace_busy_scope active_ = true; owner_ = Trace_owner_state::load(); in_functor_prev_ = In_functor_state::exchange(true); + nested0_ = Nested_span_state::load(); t0_ = std::chrono::steady_clock::now().time_since_epoch().count(); } } @@ -111,6 +117,8 @@ class Trace_busy_scope if (active_) { long long dt = std::chrono::steady_clock::now().time_since_epoch().count() - t0_; + dt -= Nested_span_state::load() - nested0_; // tasks run inside this body's yield points + In_functor_state::store(in_functor_prev_); if (trace_body_add) trace_body_add(dt); // B += dt (machinery = busy - B is derived) @@ -126,6 +134,7 @@ class Trace_busy_scope bool in_functor_prev_ = false; int owner_ = -1; long long t0_ = 0; + long long nested0_ = 0; // `Nested_span_state` at the scope's start }; // Brackets a graph run's per-run setup + initial dispatch (link binding, node re-arm, diff --git a/include/ts/parallel_for.h b/include/ts/parallel_for.h index 5ef684e..fc62636 100644 --- a/include/ts/parallel_for.h +++ b/include/ts/parallel_for.h @@ -81,6 +81,7 @@ struct Parallel_base : Task_control_block Balance balance; std::optional inherited_ctx; int inherited_owner; // trace owner (graph node index), snapshotted like inherited_ctx + Priority priority = Priority::normal; // the helpers' dispatch priority, and the loop's own at its yield points #if TS_RULES_ANY unsigned inherited_relaxed; // the caller's `Relaxed_scope` opt-outs, snapshotted alike #endif @@ -124,6 +125,10 @@ void run_loop(Parallel_state* st) { for (;;) { + // A yield point between chunks (`ts::yield`): a queued `high` task runs here before the + // next claim. One relaxed load when nothing is pending. + if (high_queued.load(std::memory_order_relaxed) != 0) + yield_to_high(st->priority); int start, stop; if (st->token.is_cancel_requested()) { @@ -192,6 +197,13 @@ void run_loop(Colored_state* st) std::uint64_t cur = st->phase_next.load(std::memory_order_acquire); for (;;) { + // A yield point between chunks, as in the flat loop; the phase is re-read afterwards, + // since the band may have moved on while the nested task ran. + if (high_queued.load(std::memory_order_relaxed) != 0) + { + yield_to_high(st->priority); + cur = st->phase_next.load(std::memory_order_acquire); + } int ph = static_cast(cur >> 32); if (ph >= st->total_phases) break; @@ -315,6 +327,7 @@ void run_participants(State* st, std::optional priority) st->refcount.fetch_add(workers - 1, std::memory_order_relaxed); // one ref per helper Priority prio = resolved_priority(priority); // resolved once, on the calling thread + st->priority = prio; Scheduler& sched = global_scheduler(); for (int t = 0; t < workers - 1; ++t) sched.submit(&helper_entry, st, prio); @@ -376,6 +389,7 @@ Task parallel_for_async(int n, Body&& body, Parallel_options opts = {}) st->refcount.fetch_add(workers, std::memory_order_relaxed); // one ref per helper Priority prio = detail::resolved_priority(opts.priority); // resolved once, on the calling thread + st->priority = prio; Scheduler& sched = global_scheduler(); for (int t = 0; t < workers; ++t) sched.submit(&detail::helper_entry, st, prio); diff --git a/include/ts/recorder.h b/include/ts/recorder.h index 2039ab2..9acc531 100644 --- a/include/ts/recorder.h +++ b/include/ts/recorder.h @@ -77,7 +77,7 @@ class Recorder private: template friend class Deferred; - template friend class Versioned; + template friend class Versioned; Recorder(detail::Journal& journal, typename detail::Journal::Slot& slot) noexcept : journal_(&journal) @@ -163,7 +163,7 @@ class Parallel_recorder private: template friend class Deferred; - template friend class Versioned; + template friend class Versioned; Parallel_recorder(detail::Journal& journal, Scheduler& scheduler) : journal_(&journal) diff --git a/include/ts/scheduler.h b/include/ts/scheduler.h index 8a96867..3f26232 100644 --- a/include/ts/scheduler.h +++ b/include/ts/scheduler.h @@ -138,6 +138,10 @@ namespace detail // `create_scheduler`. Returns a `unique_ptr` because `Scheduler` is non-movable. std::unique_ptr make_scheduler(Scheduler_config config = {}); +// A yield point's slow half (declared again beside `high_queued` in task_block.h, defined in +// scheduler.cpp); a friend of `Scheduler` so it can pop the `high` queue directly. +void yield_to_high(Priority own) noexcept; + // The slot behind `ts::current_worker_index()`, written only by a worker thread's entry and // exit. Behind the thread-local barrier (ts/detail/thread_local.h) like every other // thread-local here: the readers are header code (`Parallel_recorder::lane`, the trace @@ -156,6 +160,7 @@ class Scheduler { friend class detail::Worker_thread; friend std::unique_ptr detail::make_scheduler(Scheduler_config); + friend void detail::yield_to_high(Priority own) noexcept; public: ~Scheduler(); diff --git a/include/ts/static_task_graph.h b/include/ts/static_task_graph.h index bbcb3ae..088ba8e 100644 --- a/include/ts/static_task_graph.h +++ b/include/ts/static_task_graph.h @@ -63,7 +63,8 @@ class Graph_node requires (sizeof...(Nodes) > 0) && (std::is_same_v && ...) Graph_node& before(const Graph_node& successor, const Nodes&... more); - // Queue priority for this node when it is dispatched each run. + // Queue priority for this node when it is dispatched each run. Overrides the graph's + // default (`Static_task_graph::set_default_priority`). Graph_node& set_priority(Priority p); // Dispatch this node inline: when it becomes ready, run it on the thread that settled @@ -220,6 +221,11 @@ class Static_task_graph // records nothing (stamps and fold compile out). void set_trace(tools::Graph_trace* trace); + // Queue priority for every node that has not set its own (`Graph_node::set_priority`), + // including nodes added later - the spelling for a graph whose whole run is urgent, such as + // a fixed-rate graph with a deadline. Applied at each run's re-arm, like a node's own. + void set_default_priority(Priority p); + int node_count() const { return static_cast(nodes_.size()); } private: @@ -242,6 +248,7 @@ class Static_task_graph // is build-once, so it is never needed again). std::vector pipes; Priority priority = Priority::normal; // applied to `block` at each run's re-arm + bool priority_set = false; // set by `Graph_node::set_priority`; else follows the graph default bool inline_dispatch = false; // run on the settling thread if its acquires all succeed synchronously // --- derived by `compile()` / used by the run machinery -------------------------- @@ -342,6 +349,7 @@ class Static_task_graph bool links_lent_ = false; std::unique_ptr run_; // reused across execute() runs (one run at a time) bool compiled_ = false; + Priority default_priority_ = Priority::normal; // see `set_default_priority` // Attached via set_trace; not owned. Unconditional (one pointer) so the run logic // needs no `TS_PROFILING` blocks; without profiling it is stored but never read. tools::Graph_trace* trace_ = nullptr; @@ -397,6 +405,7 @@ Graph_node Static_task_graph::add_node(Named name, Fn&& fn, Objs&&... objs) } node.name = name; + node.priority = default_priority_; int index = static_cast(nodes_.size()); nodes_.push_back(std::move(node)); diff --git a/include/ts/task.h b/include/ts/task.h index b133026..94fb542 100644 --- a/include/ts/task.h +++ b/include/ts/task.h @@ -305,6 +305,20 @@ auto launch(Fn&& fn, Dispatch_options opts = {}, return detail::build_bare_task(std::forward(fn), std::move(opts), site); } +// A yield point for long-running work: if a `Priority::high` task is queued, run it now, on +// this thread, and return; otherwise return at once. With nothing pending the cost is one +// relaxed load, so it can sit in an inner loop. It never suspends, so it is legal in any body +// (a functor node, a `parallel_for` body, a coroutine segment), and grants held across it are +// safe: the task it runs was queued with its own turns already taken, so it cannot wait on +// them. The yielding work continues on the same stack afterwards. A no-op off a worker, in +// worker-less mode (nothing queues there), and in a task already running at `high`. Only +// queued `high` work is run; `normal` and `low` work never preempts through a yield point. +inline void yield() +{ + if (detail::high_queued.load(std::memory_order_relaxed) != 0) + detail::yield_to_high(detail::resolved_priority(std::nullopt)); +} + // Declares that something the task system is waiting on will be completed by a thread the // scheduler does not own - an OS I/O completion, a GPU fence, a `Signal` triggered from a // dedicated engine thread, a `Frame_gate`'s next `open()`. Hold one for as long as that diff --git a/include/ts/timer.h b/include/ts/timer.h new file mode 100644 index 0000000..6dd8dff --- /dev/null +++ b/include/ts/timer.h @@ -0,0 +1,88 @@ +#pragma once + +// Time-driven waits. `ts::sleep` / `ts::sleep_until` return a task that settles at a deadline, +// and `ts::Periodic` is a fixed-rate tick source built on them - the clock a fixed-rate graph +// runs on (docs/guide.md, "Fixed-rate subsystems"). Design of record: +// docs/internals/timer-primitive-design.md. + +#include "ts/cancellation.h" +#include "ts/priority.h" +#include "ts/task.h" + +#include +#include +#include + +namespace ts +{ + +// Options for the time-driven waits. +struct Sleep_options +{ + // Cancels the wait: the returned task settles cancelled promptly instead of at the deadline. + Cancellation_token token = {}; + // Priority of the task that delivers the wakeup, which is where an awaiting coroutine + // resumes. Unset = the calling task's priority, else `normal` (`detail::resolved_priority`). + std::optional priority{}; +}; + +// A task that settles completed at `deadline`, or earlier and cancelled once `opts.token` is +// requested. Deadlines are kept by one timer thread, created on first use and stopped by +// `destroy_scheduler`. It runs no user code: the wakeup is delivered as a task at +// `opts.priority`, so a coroutine awaiting the sleep resumes on a worker. A deadline already +// passed returns a settled task. Worker-less mode has no worker to deliver on and is fatal. +// Await or cancel every sleep before `destroy_scheduler` (fatal under `TS_SAFETY_CHECKS`). +[[nodiscard("the returned task is the wait: co_await or sync it")]] +Task sleep_until(std::chrono::steady_clock::time_point deadline, Sleep_options opts = {}, + std::source_location site = std::source_location::current()); + +// `sleep_until(now + duration, opts)`. +[[nodiscard("the returned task is the wait: co_await or sync it")]] +Task sleep(std::chrono::steady_clock::duration duration, Sleep_options opts = {}, + std::source_location site = std::source_location::current()); + +// A fixed-rate tick source. Deadlines sit on the grid `origin + k * period` (origin = the +// construction instant), so late delivery never drifts the grid: +// +// ts::Periodic tick{ 16'667us, { .priority = ts::Priority::high } }; +// for (;;) +// { +// int due = co_await tick.next(); // grid points passed since the previous tick +// ... +// } +// +// `next()` settles with the number of grid points passed since the previous `next()` settled: +// 1 in steady state, more when the consumer fell behind (what to do about it is the +// consumer's policy), and 0 once `opts.token` is requested. One consumer awaits `next()` at a +// time, and the `Periodic` must outlive the task `next()` returns. +class Periodic +{ +public: + explicit Periodic(std::chrono::steady_clock::duration period, Sleep_options opts = {}, + std::source_location site = std::source_location::current()); + + [[nodiscard("the returned task is the tick: co_await or sync it")]] + Task next(); + + // Re-anchor the grid at now: the next deadline is one period away. For resuming after a + // deliberate pause, which would otherwise report every missed period at once. + void reset(); + + std::chrono::steady_clock::duration period() const noexcept { return period_; } + +private: + std::chrono::steady_clock::duration period_; + std::chrono::steady_clock::time_point next_deadline_; + Sleep_options opts_; + std::source_location site_; +}; + +namespace detail +{ +// Stop the timer thread and join it (the next sleep restarts it). With `check_armed`, a sleep +// still armed is fatal under `TS_SAFETY_CHECKS` - `destroy_scheduler` passes true, program exit +// false. Armed sleeps dropped here never settle. +void timer_shutdown(bool check_armed) noexcept; +} + +} // namespace ts diff --git a/include/ts/ts.h b/include/ts/ts.h index 3676498..00b4b7d 100644 --- a/include/ts/ts.h +++ b/include/ts/ts.h @@ -18,5 +18,6 @@ #include "ts/access.h" // TS_CHECK_ACCESS, Access_context (for instrumenting guarded types) #include "ts/rules.h" // ts::Rule, Relaxed_scope - the waiting-rule check policy #include "ts/frame_gate.h" // ts::Frame_gate - re-enter at the next frame boundary +#include "ts/timer.h" // ts::sleep, ts::sleep_until, ts::Periodic - time-driven waits #include "ts/coroutine_support.h" // co_await a Task, ts::read_only/ts::read_write access guards diff --git a/include/ts/versioned.h b/include/ts/versioned.h index 01a6c47..ea16d61 100644 --- a/include/ts/versioned.h +++ b/include/ts/versioned.h @@ -1,16 +1,20 @@ #pragma once #include "ts/access.h" +#include "ts/coroutine_support.h" // the held-grant awaiter `read_last_versions` resumes through #include "ts/fatal.h" #include "ts/guarded.h" #include "ts/recorder.h" #include "ts/task.h" +#include #include #include +#include #include #include #include +#include #include #include #include @@ -41,6 +45,106 @@ enum class Resync overwrite, }; +// Which published versions a `Versioned` keeps readable. `current` is the double buffer. +// `current_and_previous` also keeps the version published before it, readable together with +// the current one through `read_last_versions` - what a consumer interpolating between a +// fixed-rate producer's outputs needs. The cost is a third replica, rotated at every publish, +// and a resync by copy: after the rotation the shadow holds the version before last, which one +// replayed batch cannot bring forward, so `Resync::replay` is rejected (`copy` is the default for +// this history, `overwrite` is allowed). +enum class History { current, current_and_previous }; + +// The publish instants of the two versions `read_last_versions` returns. +struct Version_stamps +{ + std::chrono::steady_clock::time_point previous_published{}; + std::chrono::steady_clock::time_point current_published{}; + std::uint64_t current_serial = 0; // publishes so far - 0 before the first + + // Where `at` falls between the two publishes: 0 at `previous_published`, 1 at + // `current_published`, clamped to [0, 1]. 1 when the two coincide (before the second publish). + double fraction_at(std::chrono::steady_clock::time_point at) const noexcept; +}; + +template +class Versioned; + +namespace detail +{ + +template struct Version_awaiter; +struct Versioned_access; + +// The storage `History::current_and_previous` adds; empty otherwise (a base, so it costs nothing). +template +struct Versioned_history +{ +}; + +template +struct Versioned_history +{ + T previous_{}; // the version published before the front's, rotated in at every swap + Version_stamps stamps_{}; // written under the front's write grant, read under a read grant +}; + +// `read_last_versions()` from a task that holds no grant on the front would park a worker on +// the front's pipe - the in-task blocking rule, with the awaitable form as the fix. +inline void check_version_read_may_block() +{ +#if TS_RULE_ON(TS_RULE_IN_TASK_SYNC) + if (Current_task::get() != nullptr && rule_enforced(Rule::in_task_sync)) + { + ts::fatal("Versioned::read_last_versions() inside a task that holds no grant on the front would block " + "a worker - declare state() on the node, or co_await ts::read_last_versions(v)"); + } +#endif +} + +} // namespace detail + +// The last two published versions of a `Versioned`, held +// under one read grant on its front for the view's lifetime: +// +// auto [previous, current, stamps] = poses.read_last_versions(); +// render(lerp(previous, current, stamps.fraction_at(now))); +// +// Non-copyable and non-movable: the view is the grant, and it installs its own access context +// (like `Access_guard`), so both versions pass the harness while it lives and neither does +// after. In a coroutine it counts as a live guard: `co_await` while one is alive is fatal. +template +class Version_view +{ +public: + const T& previous; + const T& current; + const Version_stamps stamps; + + ~Version_view(); + + Version_view(const Version_view&) = delete; + Version_view& operator=(const Version_view&) = delete; + + // Tuple protocol (`std::tuple_size`/`tuple_element` below): 0 = previous, 1 = current, + // 2 = stamps. + template + decltype(auto) get() const noexcept; + +private: + template friend class Versioned; + friend struct detail::Version_awaiter; + + // `held_pipe` is the front's pipe when the view took its own read turn (released at + // destruction), null when the calling context already granted the front (the view is lent). + Version_view(const T& previous_version, const T& current_version, const Version_stamps& stamps_now, + detail::Pipe* held_pipe) noexcept; + + detail::Pipe* held_pipe_; + Access_context ctx_; + const Access_context* prev_ = nullptr; + bool counted_ = false; // counted as a live guard of the running coroutine +}; + // `Versioned` - double-buffered state with an atomic publish step: a coarse, batched // cousin of RCU / MVCC snapshot isolation. It keeps two copies of `T` behind one // `Guarded` "front": readers always see the last published version - a stable @@ -57,6 +161,9 @@ enum class Resync // `Static_task_graph` treat it like a normal guarded object. (Swap/resync mechanics: // `docs/internals/deferred-versioned-state.md`.) // +// `Versioned` keeps the version before the current one as +// well, readable as a pair through `read_last_versions` (see `History`). +// // Use (dynamic tasks): // ts::Versioned tf{ ts::Named{"transforms"} }; // owns both replicas // ts::Guarded& front = tf.state(); // its front - a Guarded readers access @@ -87,29 +194,29 @@ enum class Resync // Sibling `Deferred` shares the same staging journal but applies to a single object // with no second replica or snapshot - reach for `Deferred` to batch writes and apply // them at a chosen point, `Versioned` when readers need a stable snapshot across a cycle. -template -class Versioned +template +class Versioned : private detail::Versioned_history { static_assert(std::default_initializable, "Versioned: T must be default-constructible (both replicas)"); static_assert(std::swappable, "Versioned: publish swaps the replicas' contents"); + friend struct detail::Versioned_access; + + // Replay cannot resync the rotated shadow of `History::current_and_previous` (see `History`). + static constexpr Resync default_resync = history == History::current ? Resync::replay : Resync::copy; + public: // Leading `ts::Named` (a literal, or `ts::Named{}` for the construction site) names the // front instance for the DOT dump, the trace and the diagnostics. Required, like // `Guarded`'s: the name is what every diagnostic about this object prints. template requires std::same_as, Named> - explicit Versioned(N&& name, Resync policy = Resync::replay) + explicit Versioned(N&& name, Resync policy = default_resync) : front_(name) , policy_(policy) , front_ptr_(detail::Guarded_access::instance(front_)) { - Signal ready; - ready.trigger(); - chain_ = ready; // the "previous publish" of the first publish -#if TS_SAFETY_CHECKS - last_publish_ = ready; // the "last" publish's returned gate - done for a fresh instance -#endif + init(); } // With a declared lock rank (`ts::Rank`, access.h), forwarded to the front `Guarded`: @@ -118,17 +225,12 @@ class Versioned // against and such an await cannot be satisfied - graph declarations and `read()` need none. template requires std::same_as, Named> - explicit Versioned(N&& name, Rank rank, Resync policy = Resync::replay) + explicit Versioned(N&& name, Rank rank, Resync policy = default_resync) : front_(name, rank) , policy_(policy) , front_ptr_(detail::Guarded_access::instance(front_)) { - Signal ready; - ready.trigger(); - chain_ = ready; -#if TS_SAFETY_CHECKS - last_publish_ = ready; -#endif + init(); } // Two destruction contracts, both fatal under TS_SAFETY_CHECKS (same severity as @@ -208,6 +310,16 @@ class Versioned return std::as_const(front_).access(std::forward(fn), opts, site); } + // The last two published versions under one read grant - see `Version_view`. Lent when + // the calling context already grants the front (a graph node that declared `state()`, a + // body under `read_only`), so no turn is taken. On a blue thread it takes a read turn, + // parking behind a writer. A task that holds no grant on the front uses the awaitable + // `co_await ts::read_last_versions(v)`; blocking a worker here is fatal under + // `Rule::in_task_sync`. + [[nodiscard("the view is the read grant: bind it - auto [previous, current, stamps] = ...")]] + Version_view read_last_versions() + requires (history == History::current_and_previous); + // The front's `Guarded` - for static-graph declarations. Declare read access // only; the one sanctioned writer is the publish node (`publish_fn`). // Writing it directly bypasses versioning and breaks the replica invariant. @@ -357,6 +469,30 @@ class Versioned private: using Batch = std::vector::Command>; + // The construction tail both constructors share: a resolved publish chain, and for + // `History::current_and_previous` the resync check and the initial stamps. + void init() + { + Signal ready; + ready.trigger(); + chain_ = ready; // the "previous publish" of the first publish +#if TS_SAFETY_CHECKS + last_publish_ = ready; // the "last" publish's returned gate - done for a fresh instance +#endif + if constexpr (history == History::current_and_previous) + { + if (policy_ == Resync::replay) + { + fatal("Versioned with Resync::replay - after the rotation the " + "shadow holds the version before last, which one replayed batch cannot bring forward; " + "use Resync::copy (the default for this history) or Resync::overwrite"); + } + const auto now = std::chrono::steady_clock::now(); + this->stamps_.previous_published = now; + this->stamps_.current_published = now; + } + } + // Phase 1 work: the shadow is unobservable, so this needs no grant on the // front - readers of the current version run concurrently. void apply_to_shadow(Batch& batch) @@ -368,16 +504,27 @@ class Versioned cmd(shadow_); } - // Phase 2 work: nanoseconds under the write grant. The scope names both - // replicas in case T's swap runs instrumented members. + // Phase 2 work: nanoseconds under the write grant. The scope names every + // replica it touches in case T's swap runs instrumented members. void swap_replicas(T& front) { Access_context ctx; ctx.add(&front, Access::read_write, detail::pipe_epoch(detail::Guarded_access::pipe(front_)), detail::pipe_rank(detail::Guarded_access::pipe(front_))); ctx.add(&shadow_, Access::read_write); // shadow: no pipe - grant-free by design + if constexpr (history == History::current_and_previous) + ctx.add(&this->previous_, Access::read_write); // no pipe either: the front's grant covers it Access_scope scope(ctx); using std::swap; swap(front, shadow_); + if constexpr (history == History::current_and_previous) + { + // Rotate: the old front becomes the previous version, and the old previous becomes + // the shadow the copy resync then brings to the new version. + swap(shadow_, this->previous_); + this->stamps_.previous_published = this->stamps_.current_published; + this->stamps_.current_published = std::chrono::steady_clock::now(); + ++this->stamps_.current_serial; + } } // Phase 3: bring the new shadow (old front contents) to the new version, as a @@ -455,21 +602,177 @@ class Versioned std::function hash_; }; +namespace detail +{ + +// Reaches the storage `History::current_and_previous` adds, for the free `read_last_versions`. +struct Versioned_access +{ + template + static T* front(Versioned& v) noexcept { return v.front_ptr_; } + + template + static const T& previous(const Versioned& v) noexcept { return v.previous_; } + + template + static const Version_stamps& stamps(const Versioned& v) noexcept { return v.stamps_; } +}; + +// The awaiter behind `co_await ts::read_last_versions(v)`: the read-guard awaiter's acquire, +// resumed into a `Version_view`. A context that already grants the front is lent - ready at +// once, no turn taken - which is also the await-under-guard rule's stated exemption, an access +// that cannot suspend. +template +struct Version_awaiter : Access_awaiter +{ + Version_awaiter(Scheduler& scheduler, Pipe& pipe, T* front, const T& previous, + const Version_stamps& stamps) noexcept + : Access_awaiter(scheduler, pipe, front) + , previous_(previous) + , stamps_(stamps) + { + } + + bool await_ready() noexcept + { + const Access_context* ctx = access_load(); + lent_ = ctx != nullptr && ctx->grants(this->obj_, Access::read_only); + return lent_ || Access_awaiter::await_ready(); + } + + Version_view await_resume() noexcept + { + if (lent_) + return Version_view(previous_, *this->obj_, stamps_, nullptr); + this->finish_acquire(); + return Version_view(previous_, *this->obj_, stamps_, &this->pipe_); + } + + const T& previous_; + const Version_stamps& stamps_; + bool lent_ = false; +}; + +} // namespace detail + +// The awaitable form of `Versioned::read_last_versions()`, for a coroutine: takes a read turn +// on the front without blocking a worker, or lends one the coroutine already holds. +// auto [previous, current, stamps] = co_await ts::read_last_versions(poses); +template + requires (history == History::current_and_previous) +[[nodiscard("co_await it - the view it resumes with is the read grant")]] +detail::Version_awaiter read_last_versions(Versioned& versioned) +{ + return detail::Version_awaiter(global_scheduler(), detail::Guarded_access::pipe(versioned.state()), + detail::Versioned_access::front(versioned), detail::Versioned_access::previous(versioned), + detail::Versioned_access::stamps(versioned)); +} + // The publish step as a graph-node body: declare it with write access on // `v.state()` - conflict derivation then orders it against every reader, and the // node's grant is exactly what `publish_into` needs. // auto flip = g.add_node("flip", ts::publish_fn(poses), poses.state()).after(sim); -template +template struct Publish_fn { - Versioned* versioned; + Versioned* versioned; void operator()(T& front) const { versioned->publish_into(front); } }; +template +Publish_fn publish_fn(Versioned& v) +{ + return Publish_fn{ &v }; +} + +// --- out-of-class definitions ---------------------------------------------------------------- + +inline double Version_stamps::fraction_at(std::chrono::steady_clock::time_point at) const noexcept +{ + if (current_published <= previous_published || at >= current_published) + return 1.0; + if (at <= previous_published) + return 0.0; + return std::chrono::duration(at - previous_published) + / std::chrono::duration(current_published - previous_published); +} + template -Publish_fn publish_fn(Versioned& v) +Version_view::Version_view(const T& previous_version, const T& current_version, const Version_stamps& stamps_now, + detail::Pipe* held_pipe) noexcept + : previous(previous_version) + , current(current_version) + , stamps(stamps_now) + , held_pipe_(held_pipe) { - return Publish_fn{ &v }; + if (const Access_context* running = detail::access_load()) + ctx_ = *running; // extend the running context: a lent view keeps the caller's grant on the front + if (held_pipe_ != nullptr) + ctx_.add(¤t, Access::read_only, detail::pipe_epoch(*held_pipe_), detail::pipe_rank(*held_pipe_)); + ctx_.add(&previous, Access::read_only); // no pipe of its own: the front's grant covers it + prev_ = detail::access_load(); + detail::access_store(&ctx_); +#if TS_RULE_ON(TS_RULE_AWAIT_UNDER_GUARD) + if (detail::current_coroutine_block() != nullptr) + { + detail::guard_depth_add(1); + counted_ = true; + } +#endif +} + +template +Version_view::~Version_view() +{ + detail::access_store(prev_); +#if TS_RULE_ON(TS_RULE_AWAIT_UNDER_GUARD) + if (counted_) + detail::guard_depth_add(-1); +#endif + if (held_pipe_ != nullptr) + detail::pipe_release(global_scheduler(), *held_pipe_, Access::read_only); +} + +template +template +decltype(auto) Version_view::get() const noexcept +{ + static_assert(I < 3, "a Version_view binds three names: previous, current, stamps"); + if constexpr (I == 0) + return (previous); + else if constexpr (I == 1) + return (current); + else + return (stamps); +} + +template +Version_view Versioned::read_last_versions() + requires (history == History::current_and_previous) +{ + const Access_context* ctx = detail::access_load(); + if (ctx != nullptr && ctx->grants(front_ptr_, Access::read_only)) + return Version_view(this->previous_, *front_ptr_, this->stamps_, nullptr); + + detail::check_version_read_may_block(); + detail::Pipe& pipe = detail::Guarded_access::pipe(front_); + Signal granted; + if (!detail::pipe_acquire(global_scheduler(), pipe, Access::read_only, [granted]() mutable { granted.trigger(); })) + granted.sync(); + return Version_view(this->previous_, *front_ptr_, this->stamps_, &pipe); } } // namespace ts + +// Tuple protocol for `Version_view`: `auto [previous, current, stamps] = ...` binds references +// to the two versions and the stamps through the view's member `get()`. +template +struct std::tuple_size> : std::integral_constant +{ +}; + +template +struct std::tuple_element> +{ + using type = std::conditional_t; +}; diff --git a/macrame.vcxproj b/macrame.vcxproj index f6cd7f9..ebd04a7 100644 --- a/macrame.vcxproj +++ b/macrame.vcxproj @@ -21,6 +21,7 @@ + @@ -61,6 +62,7 @@ + 18.0 diff --git a/macrame.vcxproj.filters b/macrame.vcxproj.filters index cfae2f7..1be5b67 100644 --- a/macrame.vcxproj.filters +++ b/macrame.vcxproj.filters @@ -36,6 +36,9 @@ src + + src + @@ -140,6 +143,9 @@ include\ts + + include\ts + include\ts\detail diff --git a/macrame_playground.vcxproj b/macrame_playground.vcxproj index a6aa46f..4daf67f 100644 --- a/macrame_playground.vcxproj +++ b/macrame_playground.vcxproj @@ -31,6 +31,7 @@ + @@ -54,6 +55,7 @@ + @@ -75,6 +77,7 @@ + diff --git a/macrame_playground.vcxproj.filters b/macrame_playground.vcxproj.filters index 5fd12c3..725e300 100644 --- a/macrame_playground.vcxproj.filters +++ b/macrame_playground.vcxproj.filters @@ -66,6 +66,9 @@ sample + + sample + tsan @@ -120,6 +123,9 @@ tests + + tests + @@ -179,5 +185,8 @@ tests + + tests + diff --git a/sample/fixed_rate.cpp b/sample/fixed_rate.cpp new file mode 100644 index 0000000..85de1fb --- /dev/null +++ b/sample/fixed_rate.cpp @@ -0,0 +1,425 @@ +// A fixed-rate graph beside a variable-rate frame graph. A physics world ticks at 60 Hz on +// its own clock (`ts::Periodic`), while the frame loop runs the frame graph as fast as it can. +// Neither loop knows the other's rate, and the frame loop has no accumulator and no catch-up +// path: how many ticks fall into a frame is the clock's business. +// +// The boundary between the two is two objects, the only state both graphs touch: +// - `intents`, a `Deferred`: gameplay stages impulses grant-free at any +// moment, and the tick's step node commits them at the start of the tick. Whatever was +// staged before that commit belongs to that tick; anything later waits for the next. +// - `poses`, a `Versioned`: the tick publishes +// an extract of the world at its end, and render reads the last two published versions +// together, interpolating by where the frame falls between their publish instants. +// The world itself has one accessor, the step node, so no frame code can reach it. +// +// The physics graph's nodes are `high` (`set_default_priority`), and gameplay's long loop and +// every `parallel_for` chunk boundary are yield points (`ts::yield`), so a tick that comes due +// while the workers are busy with frame work starts within one chunk of the frame's. +// +// Checked every run: ticks run one at a time in order, the interpolated pair is always two +// consecutive ticks, the interpolation fraction stays within [0, 1], and every staged intent is +// applied exactly once. Which tick picks up a given intent depends on wall time, as it would with +// a real input device, so the full run is not bit-reproducible; the physics graph alone, driven +// by a tick-indexed intent script, is (`fixed_rate_physics_hash`). + +#include "ts/coroutine_support.h" +#include "ts/deferred.h" +#include "ts/guarded.h" +#include "ts/parallel_for.h" +#include "ts/static_task_graph.h" +#include "ts/task.h" +#include "ts/timer.h" +#include "ts/versioned.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace sample +{ + +namespace +{ + +using Clock = std::chrono::steady_clock; + +constexpr int body_count = 256; +constexpr float tick_dt = 1.0f / 60.0f; +constexpr auto tick_period = std::chrono::microseconds(16'667); +// More ticks than this in one wake are dropped rather than run back to back: a clock that +// fell far behind slows simulated time down instead of stalling every frame behind a burst. +constexpr int max_ticks_per_wake = 2; + +// Stand-in cost for work the sample does not model, kept on the CPU the way real work is. +void spin_for(std::chrono::microseconds duration) +{ + const auto until = Clock::now() + duration; + while (Clock::now() < until) + { + } +} + +struct Vec2 +{ + float x = 0.0f; + float y = 0.0f; +}; + +// The simulation. Sealed: the step node's write grant is its only accessor. +class Physics_world +{ +public: + Physics_world() + : pos_(body_count) + , vel_(body_count) + { + for (int i = 0; i < body_count; ++i) + pos_[static_cast(i)] = { static_cast(i % 16), 10.0f + static_cast(i / 16) }; + } + + void add_impulse(int body, Vec2 dv) + { + TS_CHECK_ACCESS(); + Vec2& v = vel_[static_cast(body)]; + v.x += dv.x; + v.y += dv.y; + ++impulses_applied_; + } + + void step(float dt) + { + TS_CHECK_ACCESS(); + for (std::size_t i = 0; i < pos_.size(); ++i) + { + vel_[i].y -= 9.8f * dt; + pos_[i].x += vel_[i].x * dt; + pos_[i].y += vel_[i].y * dt; + if (pos_[i].y < 0.0f) + { + pos_[i].y = -pos_[i].y; + vel_[i].y = -vel_[i].y * 0.8f; + } + } + ++tick_; + } + + const std::vector& positions() const { TS_CHECK_ACCESS(); return pos_; } + std::uint64_t tick() const { TS_CHECK_ACCESS(); return tick_; } + long long impulses_applied() const { TS_CHECK_ACCESS(); return impulses_applied_; } + + std::size_t hash() const + { + TS_CHECK_ACCESS(); + std::size_t h = 1469598103934665603ull; + for (const Vec2& p : pos_) + { + h = (h ^ static_cast(p.x * 1000.0f)) * 1099511628211ull; + h = (h ^ static_cast(p.y * 1000.0f)) * 1099511628211ull; + } + return h; + } + +private: + std::vector pos_; + std::vector vel_; + std::uint64_t tick_ = 0; + long long impulses_applied_ = 0; +}; + +// The extract a tick publishes: positions and the tick that produced them. +class Pose_snapshot +{ +public: + void assign(const std::vector& positions, std::uint64_t tick) + { + TS_CHECK_ACCESS(); + positions_ = positions; + tick_ = tick; + } + + Vec2 at(int body) const + { + TS_CHECK_ACCESS(); + return body < static_cast(positions_.size()) ? positions_[static_cast(body)] : Vec2{}; + } + + std::uint64_t tick() const { TS_CHECK_ACCESS(); return tick_; } + +private: + std::vector positions_; + std::uint64_t tick_ = 0; +}; + +struct Input_state +{ + long long frame = 0; +}; + +struct Gameplay_state +{ + Vec2 focus; // what render centres on: body 0, as the frame last saw it +}; + +struct Render_output +{ + float body0_y = 0.0f; // interpolated between the last two ticks + float focus_y = 0.0f; // as gameplay last saw it +}; + +// The costs that stand in for real work, scaled together. +struct Costs +{ + std::chrono::microseconds physics; + std::chrono::microseconds gameplay; + std::chrono::microseconds render; +}; + +Costs costs_at(float scale) +{ + auto us = [scale](double value) { return std::chrono::microseconds(static_cast(value * scale)); }; + return { us(3000.0), us(4000.0), us(1500.0) }; +} + +// What the run observed, written by the nodes and read after both loops stop. +struct Run_stats +{ + std::atomic intents_staged{ 0 }; + std::atomic render_frames{ 0 }; + std::atomic pairs_not_consecutive{ 0 }; + std::atomic fraction_out_of_range{ 0 }; + long long ticks = 0; // written by the driver only + long long wakes = 0; + long long dropped = 0; + Clock::duration shortest_wake_interval = Clock::duration::max(); + Clock::duration longest_wake_interval = Clock::duration::zero(); +}; + +// Both domains' state. Declared before the graphs that hold recorders into it, so the graphs +// are destroyed first. +struct Domains +{ + ts::Guarded world{ ts::Named{ "world" } }; + ts::Deferred intents{ world }; + ts::Versioned poses{ ts::Named{ "poses" } }; + + ts::Guarded input{ ts::Named{ "input" } }; + ts::Guarded gameplay{ ts::Named{ "gameplay" } }; + ts::Guarded render{ ts::Named{ "render" } }; +}; + +// One tick: commit the intents staged since the last tick, step, publish the extract. +ts::Static_task_graph build_physics_graph(Domains& domains, Costs costs) +{ + ts::Static_task_graph graph; + auto step = graph.add_node("physics_step", + [&domains, costs, out = domains.poses.recorder()](Physics_world& world) mutable + { + (void)domains.intents.commit(); // inline: this node holds the write grant + world.step(tick_dt); + // The solver's cost, fanned out; the chunks inherit the node's `high` priority. + ts::parallel_for(4, [costs](int) { spin_for(costs.physics / 4); }); + out.stage([positions = world.positions(), tick = world.tick()](Pose_snapshot& snapshot) + { + snapshot.assign(positions, tick); + }); + }, + domains.world); + graph.add_node("physics_publish", ts::publish_fn(domains.poses), domains.poses.state()).after(step); + graph.set_default_priority(ts::Priority::high); + graph.compile(); + return graph; +} + +// The frame: input, gameplay (stages intents, reads the last snapshot), render (interpolates). +ts::Static_task_graph build_frame_graph(Domains& domains, Costs costs, Run_stats& stats) +{ + ts::Static_task_graph graph; + graph.add_node("input", [](Input_state& input) { ++input.frame; }, domains.input); + + graph.add_node("gameplay", + [costs, &stats, rec = domains.intents.recorder()](const Input_state& input, const Pose_snapshot& poses, + Gameplay_state& gameplay) mutable + { + gameplay.focus = poses.at(0); + if (input.frame % 20 == 0) + { + const int body = static_cast(input.frame / 20 % body_count); + rec.stage([body](Physics_world& world) { world.add_impulse(body, { 0.0f, 6.0f }); }); + stats.intents_staged.fetch_add(1, std::memory_order_relaxed); + } + // A long body with yield points: a tick that comes due mid-loop runs here, between + // two slices, instead of waiting for the whole body. + for (int slice = 0; slice < 8; ++slice) + { + spin_for(costs.gameplay / 8); + ts::yield(); + } + }, + domains.input, domains.poses.state(), domains.gameplay); + + graph.add_node("render", + [&domains, costs, &stats](const Gameplay_state& gameplay, const Pose_snapshot&, Render_output& render) + { + // Lent: this node declared a read on the front, so the pair takes no turn of its own. + auto [previous, current, stamps] = domains.poses.read_last_versions(); + const double alpha = stamps.fraction_at(Clock::now()); + if (current.tick() != previous.tick() + 1) + stats.pairs_not_consecutive.fetch_add(1, std::memory_order_relaxed); + if (alpha < 0.0 || alpha > 1.0) + stats.fraction_out_of_range.fetch_add(1, std::memory_order_relaxed); + const float y0 = previous.at(0).y; + const float y1 = current.at(0).y; + render.body0_y = y0 + static_cast(alpha) * (y1 - y0); + render.focus_y = gameplay.focus.y; + spin_for(costs.render); + stats.render_frames.fetch_add(1, std::memory_order_relaxed); + }, + domains.gameplay, domains.poses.state(), domains.render); + + graph.compile(); + return graph; +} + +// The fixed-rate driver: wait for the clock, run the ticks it reports, repeat until stopped. +// The overload policy lives here and nowhere else. +ts::Task run_ticks(ts::Static_task_graph& physics, ts::Periodic& clock, Run_stats& stats) +{ + Clock::time_point last_wake{}; + for (;;) + { + const int due = co_await clock.next(); + if (due == 0) + co_return; // stopped + const Clock::time_point now = Clock::now(); + if (stats.wakes > 0) + { + stats.shortest_wake_interval = std::min(stats.shortest_wake_interval, now - last_wake); + stats.longest_wake_interval = std::max(stats.longest_wake_interval, now - last_wake); + } + last_wake = now; + ++stats.wakes; + const int run = std::min(due, max_ticks_per_wake); + stats.dropped += due - run; + for (int i = 0; i < run; ++i) + co_await physics.execute(); + stats.ticks += run; + } +} + +struct Outcome +{ + bool ok = false; + double frame_ms = 0.0; + double ticks_per_second = 0.0; +}; + +Outcome run_domains(int frames, float scale, bool print) +{ + const Costs costs = costs_at(scale); + Run_stats stats; + Domains domains; + ts::Static_task_graph physics = build_physics_graph(domains, costs); + ts::Static_task_graph frame = build_frame_graph(domains, costs, stats); + + // Two ticks before the loops start, so the pair render reads holds two real versions. + physics.execute().sync(); + physics.execute().sync(); + + ts::Cancellation_source stop; + ts::Periodic clock{ tick_period, { .token = stop.token(), .priority = ts::Priority::high } }; + ts::Task driver = run_ticks(physics, clock, stats); + + const Clock::time_point t0 = Clock::now(); + for (int f = 0; f < frames; ++f) + frame.execute().sync(); + const double elapsed_s = std::chrono::duration(Clock::now() - t0).count(); + + stop.request_cancel(); + driver.sync(); + physics.execute().sync(); // one more tick commits whatever the last frames staged + + const long long ticks_total = stats.ticks + 3; // the two primers and the final tick + const auto [world_ticks, applied] = domains.world.access([](const Physics_world& world) + { + return std::pair{ static_cast(world.tick()), world.impulses_applied() }; + }).sync(); + + Outcome outcome; + outcome.frame_ms = 1000.0 * elapsed_s / frames; + outcome.ticks_per_second = elapsed_s > 0.0 ? static_cast(stats.ticks) / elapsed_s : 0.0; + outcome.ok = world_ticks == ticks_total + && applied == stats.intents_staged.load() + && stats.pairs_not_consecutive.load() == 0 + && stats.fraction_out_of_range.load() == 0 + && stats.render_frames.load() == frames; + + if (print) + { + auto ms = [](Clock::duration d) { return std::chrono::duration(d).count(); }; + std::printf("\n[fixed_rate] %d frames at %.2f ms/frame; %lld ticks in %.2f s = %.1f ticks/s (period %.2f ms)\n", + frames, outcome.frame_ms, stats.ticks, elapsed_s, outcome.ticks_per_second, + std::chrono::duration(tick_period).count()); + if (stats.wakes > 1) + { + std::printf(" wake intervals %.2f .. %.2f ms, %lld ticks dropped by the overload policy\n", + ms(stats.shortest_wake_interval), ms(stats.longest_wake_interval), stats.dropped); + } + std::printf(" %lld intents staged, %lld applied; pairs consecutive: %s; fraction in [0, 1]: %s -> %s\n", + stats.intents_staged.load(), applied, stats.pairs_not_consecutive.load() == 0 ? "yes" : "no", + stats.fraction_out_of_range.load() == 0 ? "yes" : "no", outcome.ok ? "ok" : "FAILED"); + } + return outcome; +} + +} // namespace + +// --- entry points ----------------------------------------------------------------- + +// Runs both loops for `frames` frames and prints what the fixed-rate side did. +void run_fixed_rate_sample(int frames) +{ + (void)run_domains(frames, 1.0f, true); +} + +// The structural checks of a run, for the integration test (true = every check held). +bool fixed_rate_self_check(int frames, float scale) +{ + return run_domains(frames, scale, false).ok; +} + +// The physics graph alone for `ticks` ticks, with intents staged on a tick-indexed script from +// the calling thread: the world's hash is a function of the script only, whatever the worker +// count - the determinism a fixed-rate graph keeps when its input cut sequence is fixed. +std::size_t fixed_rate_physics_hash(int ticks) +{ + Run_stats stats; + Domains domains; + ts::Static_task_graph physics = build_physics_graph(domains, costs_at(0.02f)); + { + auto rec = domains.intents.recorder(); + for (int tick = 0; tick < ticks; ++tick) + { + if (tick % 7 == 0) + { + const int body = tick % body_count; + rec.stage([body](Physics_world& world) { world.add_impulse(body, { 1.0f, 4.0f }); }); + } + physics.execute().sync(); + } + } + return domains.world.access([](const Physics_world& world) { return world.hash(); }).sync(); +} + +// Headless run at a fast scale, for the sanitizer driver: both graphs, the clock, the yield +// points and the `Versioned` pair read under concurrency. +void stress_fixed_rate(int frames) +{ + (void)run_domains(frames, 0.05f, false); +} + +} // namespace sample diff --git a/sample/game_frame.cpp b/sample/game_frame.cpp index 153488d..2f4185c 100644 --- a/sample/game_frame.cpp +++ b/sample/game_frame.cpp @@ -28,6 +28,14 @@ // "what does the static graph buy me" - the pipe still gives safety, the graph // gives the schedule. // +// Frame_variant::fixed_rate - the optimised frame with physics and networking on +// their own clocks: a 60 Hz physics graph (broadphase -> narrowphase -> solver -> +// finalize) and a 30 Hz network graph, each driven by a `ts::Periodic`, beside the +// variable-rate frame graph (`Fixed_rate_ticks`). The frame reaches them only +// through staged inputs (`Deferred`) and published snapshots (`Versioned`), and +// propagation interpolates the last two physics ticks. The physics chain leaves the +// frame's critical path; how many ticks fall into a frame is the clocks' business. +// // What the layers show: // - `Static_task_graph` - nodes over guarded stores; every edge derived from // parameter const-ness. A real render frame's worth of nodes: a gameplay @@ -71,15 +79,18 @@ #include "ts/coroutine_support.h" #include "ts/static_task_graph.h" #include "ts/task.h" +#include "ts/timer.h" #include "ts/versioned.h" #if TS_PROFILING #include "graph_trace.h" // tools/: the aggregating runtime trace (see trace_game_frame) #endif +#include #include #include #include +#include #include #include #include @@ -92,7 +103,8 @@ namespace sample // levers the trace makes obvious. `build_frame_graph` takes either. `graph_free` is // not a graph at all - the same baseline frame hand-composed with coroutines and the // access verbs (`run_frame_graph_free`), the comparison case for what `compile()` buys. -enum class Frame_variant { baseline, optimised, graph_free }; +// `fixed_rate` is the optimised frame with physics and networking on their own clocks. +enum class Frame_variant { baseline, optimised, graph_free, fixed_rate }; namespace { @@ -123,12 +135,17 @@ std::atomic hud_snapshots{ 0 }; class Float_store { public: + Float_store() = default; // empty: the fixed-rate variant's snapshots before their first publish explicit Float_store(int entities) : data_(entities, 0.0f) {} int size() const { TS_CHECK_ACCESS(); return static_cast(data_.size()); } float get(int i) const { TS_CHECK_ACCESS(); return data_[i]; } void set(int i, float v) { TS_CHECK_ACCESS(); data_[i] = v; } + // The whole store as a batch - how the fixed-rate variant's ticks publish their snapshots. + std::vector values() const { TS_CHECK_ACCESS(); return data_; } + void assign(const std::vector& values) { TS_CHECK_ACCESS(); data_ = values; } + private: std::vector data_; }; @@ -277,6 +294,17 @@ struct World ts::Guarded draw_lists{ ts::Named{"draw_lists"} }; ts::Deferred draw_staged{ draw_lists }; // references an earlier member - fine + // The fixed-rate variant's boundary objects, unused by the others. Physics publishes its + // bodies after every tick, the last two readable together for interpolation, and takes + // its inputs from the frame as staged commands. The network tick takes what the frame + // sends through an outbox and publishes its state for the frame's readers. + ts::Versioned body_snapshot{ ts::Named{"body_snapshot"} }; + ts::Guarded physics_inputs{ ts::Named{"physics_inputs"}, entity_count }; + ts::Deferred physics_intents{ physics_inputs }; + ts::Versioned net_snapshot{ ts::Named{"net_snapshot"} }; + ts::Guarded net_outbox{ ts::Named{"net_outbox"}, entity_count }; + ts::Deferred net_outbox_stream{ net_outbox }; + // Streaming loads run detached and can finish after the frame that launched them, so at // teardown some may still be staging into `assets_stream`. Flush on shutdown: wait for the // in-flight loaders to drain, then apply the final batch as one write - leaving no staged @@ -351,6 +379,9 @@ void tick_solver(const Contacts&, const Combat&, Velocities&); void tick_finalize(const Velocities&, Bodies&); // transform propagation (stages this frame's transforms; the flip publishes) void tick_propagation(const Local_xf&, const Bodies&, const Velocities&, ts::Recorder&); +// (fixed-rate variant: bodies interpolated between the physics tick's last two snapshots) +void tick_propagation_interpolated(const Local_xf&, const Bodies& previous_bodies, const Bodies& current_bodies, + double fraction, ts::Recorder&); // render pipeline (reads last frame's transforms) void tick_frustum_cull(const Transforms& prev_xf, const Camera&, const Renderables&, Visibility&); void tick_occlusion_cull(const Transforms& prev_xf, const Visibility&, Vis_final&); @@ -386,6 +417,10 @@ ts::Graph_node add_submit(ts::Static_task_graph&, World&, bool opt); ts::Graph_node add_navigation(ts::Static_task_graph&, World&); ts::Graph_node add_AI(ts::Static_task_graph&, World&, bool opt); +// the fixed-rate variant's two tick graphs (definitions below build_frame_graph) +ts::Static_task_graph build_physics_tick_graph(World&); +ts::Static_task_graph build_network_tick_graph(World&); + // --- the frame graph -------------------------------------------------------------- // Build the frame. Construction is shared; `opt` branches the few nodes whose access @@ -393,7 +428,12 @@ ts::Graph_node add_AI(ts::Static_task_graph&, World&, bool opt); // body-level splits (combat, ik_post). ts::Static_task_graph build_frame_graph(World& world, Frame_variant variant, const char* DOT_path = nullptr) { - const bool opt = variant == Frame_variant::optimised; + // The fixed-rate variant is the optimised frame with physics and networking moved onto + // their own clocks: their nodes leave this graph, and the frame reads their snapshots. + const bool fixed = variant == Frame_variant::fixed_rate; + const bool opt = variant == Frame_variant::optimised || fixed; + ts::Guarded& net = fixed ? world.net_snapshot.state() : world.net; + ts::Guarded& bodies = fixed ? world.body_snapshot.state() : world.bodies; ts::Static_task_graph graph; // Priorities model importance, not measured wins, and are identical in both @@ -402,10 +442,24 @@ ts::Static_task_graph build_frame_graph(World& world, Frame_variant variant, con // longest pole and the present deadline. // Frame head. - graph.add_node("input", &tick_input, world.input); + if (fixed) // the network tick reads what the frame sends it, never the live input store + { + graph.add_node("input", + [rec = world.net_outbox_stream.recorder()](Input& in) mutable + { + tick_input(in); + rec.stage([](Float_store& outbox) { outbox.set(0, 1.0f); }); + }, + world.input); + } + else + { + graph.add_node("input", &tick_input, world.input); + } graph.add_node("camera", &tick_camera, world.input, world.camera); - graph.add_node("networking", &tick_networking, world.input, world.net); - graph.add_node("scripting", &tick_scripting, world.input, world.net, world.script_events); + if (!fixed) + graph.add_node("networking", &tick_networking, world.input, world.net); + graph.add_node("scripting", &tick_scripting, world.input, net, world.script_events); // Streaming declares only `input`: it stages loaded assets into `assets_stream` (grant-free) // and reads `asset_source` through its own async loads - neither is a declared node access. graph.add_node("streaming", @@ -416,12 +470,26 @@ ts::Static_task_graph build_frame_graph(World& world, Frame_variant variant, con // Gameplay trio: shared inputs, disjoint outputs -> runs in parallel. Combat is // parallelised in the optimised variant (a critical bar); economy/quests stay serial. - graph.add_node("combat", // L2: per-entity split when opt - [opt](const Transforms& xf, const Input& in, const Net& net, const Script_events& ev, Combat& c) - { tick_combat(xf, in, net, ev, c, opt); }, - world.transforms.state(), world.input, world.net, world.script_events, world.combat); - graph.add_node("economy", &tick_economy, world.transforms.state(), world.input, world.net, world.script_events, world.economy); - graph.add_node("quests", &tick_quests, world.transforms.state(), world.input, world.net, world.script_events, world.quests); + if (fixed) + { + graph.add_node("combat", // L2, and its impulses reach the physics tick as staged inputs + [rec = world.physics_intents.recorder()](const Transforms& xf, const Input& in, const Net& n, + const Script_events& ev, Combat& c) mutable + { + tick_combat(xf, in, n, ev, c, true); + rec.stage([](Float_store& inputs) { inputs.set(0, 1.0f); }); + }, + world.transforms.state(), world.input, net, world.script_events, world.combat); + } + else + { + graph.add_node("combat", // L2: per-entity split when opt + [opt](const Transforms& xf, const Input& in, const Net& n, const Script_events& ev, Combat& c) + { tick_combat(xf, in, n, ev, c, opt); }, + world.transforms.state(), world.input, net, world.script_events, world.combat); + } + graph.add_node("economy", &tick_economy, world.transforms.state(), world.input, net, world.script_events, world.economy); + graph.add_node("quests", &tick_quests, world.transforms.state(), world.input, net, world.script_events, world.quests); // Navigation + AI. Baseline AI reads this frame's trio (the trio binds AI on the // critical path); optimised reads last frame's gameplay snapshot instead, deleting @@ -448,18 +516,39 @@ ts::Static_task_graph build_frame_graph(World& world, Frame_variant variant, con world.anim_pose, world.local_xf); graph.add_node("skinning", &tick_skinning, world.local_xf, world.skin_matrices); - // Physics pipeline. - graph.add_node("broadphase", &tick_broadphase, world.bodies, world.broad_pairs); - graph.add_node("narrowphase", &tick_narrowphase, world.broad_pairs, world.contacts); - graph.add_node("solver", &tick_solver, world.contacts, world.combat, world.velocities).set_priority(ts::Priority::high); - graph.add_node("finalize", &tick_finalize, world.velocities, world.bodies); + // Physics pipeline (the fixed-rate variant runs it in its own graph, see + // `build_physics_tick_graph`). + if (!fixed) + { + graph.add_node("broadphase", &tick_broadphase, world.bodies, world.broad_pairs); + graph.add_node("narrowphase", &tick_narrowphase, world.broad_pairs, world.contacts); + graph.add_node("solver", &tick_solver, world.contacts, world.combat, world.velocities).set_priority(ts::Priority::high); + graph.add_node("finalize", &tick_finalize, world.velocities, world.bodies); + } // Propagation: this frame's transforms from animation + physics, staged grant-free - // (the physics->propagation edge derives from two conflicts: bodies + velocities). - auto propagation = graph.add_node("propagation", - [rec = world.transforms.recorder()](const Local_xf& lx, const Bodies& b, const Velocities& v) mutable - { tick_propagation(lx, b, v, rec); }, - world.local_xf, world.bodies, world.velocities); + // (the physics->propagation edge derives from two conflicts: bodies + velocities). The + // fixed-rate variant has no such edge: it interpolates the physics tick's last two + // published snapshots, read lent under this node's read of the snapshot front. + ts::Graph_node propagation; + if (fixed) + { + propagation = graph.add_node("propagation", + [&world, rec = world.transforms.recorder()](const Local_xf& lx, const Bodies&) mutable + { + auto [previous, current, stamps] = world.body_snapshot.read_last_versions(); + tick_propagation_interpolated(lx, previous, current, + stamps.fraction_at(std::chrono::steady_clock::now()), rec); + }, + world.local_xf, world.body_snapshot.state()); + } + else + { + propagation = graph.add_node("propagation", + [rec = world.transforms.recorder()](const Local_xf& lx, const Bodies& b, const Velocities& v) mutable + { tick_propagation(lx, b, v, rec); }, + world.local_xf, world.bodies, world.velocities); + } // Render pipeline - reads last frame's transforms (so it overlaps this frame's // simulation), which means every node here must run before the flip. That is intent, so @@ -483,8 +572,22 @@ ts::Static_task_graph build_frame_graph(World& world, Frame_variant variant, con // Off-path leaves. auto audio = graph.add_node("audio", &tick_audio, world.transforms.state(), world.audio_out).set_priority(ts::Priority::low); auto vfx = graph.add_node("vfx", &tick_vfx, world.transforms.state(), world.particles, world.vfx); - graph.add_node("replication", &tick_replication, world.combat, world.economy, world.quests, world.intents, world.replication); - graph.add_node("stats", &tick_stats, world.combat, world.economy, world.bodies, world.visibility, world.stats); + if (fixed) // the packed snapshot leaves through the network tick's outbox + { + graph.add_node("replication", + [rec = world.net_outbox_stream.recorder()](const Combat& c, const Economy& e, const Quests& q, + const Intents& i, Replication& r) mutable + { + tick_replication(c, e, q, i, r); + rec.stage([](Float_store& outbox) { outbox.set(1, 1.0f); }); + }, + world.combat, world.economy, world.quests, world.intents, world.replication); + } + else + { + graph.add_node("replication", &tick_replication, world.combat, world.economy, world.quests, world.intents, world.replication); + } + graph.add_node("stats", &tick_stats, world.combat, world.economy, bodies, world.visibility, world.stats); // The streaming commit slot: applies the assets staged so far as one write. It writes // `assets`, so the conflict edge to gc (which reads `assets`) orders it before gc; it has no // edge to streaming (staging is grant-free), so it commits whatever has arrived - last @@ -524,6 +627,116 @@ ts::Static_task_graph build_frame_graph(World& world, Frame_variant variant, con return graph; } +// --- the fixed-rate graphs (Frame_variant::fixed_rate) ---------------------------- + +// One physics tick: the four-stage pipeline the other variants run inside the frame, then the +// extract. The solver commits the inputs the frame staged since the last tick (it holds the +// write grant, so the commit applies inline); the extract publishes the bodies the frame reads. +ts::Static_task_graph build_physics_tick_graph(World& world) +{ + ts::Static_task_graph graph; + graph.add_node("broadphase", &tick_broadphase, world.bodies, world.broad_pairs); + graph.add_node("narrowphase", &tick_narrowphase, world.broad_pairs, world.contacts); + graph.add_node("solver", + [&world](const Contacts& contacts, Float_store& inputs, Velocities& velocities) + { + (void)world.physics_intents.commit(); + tick_solver(contacts, inputs, velocities); + }, + world.contacts, world.physics_inputs, world.velocities); + auto finalize = graph.add_node("finalize", &tick_finalize, world.velocities, world.bodies); + auto extract = graph.add_node("physics_extract", + [rec = world.body_snapshot.recorder()](const Bodies& b) mutable + { + rec.stage([values = b.values()](Bodies& snapshot) { snapshot.assign(values); }); + }, + world.bodies); + extract.after(finalize); + graph.add_node("physics_publish", ts::publish_fn(world.body_snapshot), world.body_snapshot.state()).after(extract); + graph.set_default_priority(ts::Priority::high); + graph.compile(); + return graph; +} + +// One network tick: take what the frame sent (the outbox commit), run the network update, and +// publish its state for the frame's readers. +ts::Static_task_graph build_network_tick_graph(World& world) +{ + ts::Static_task_graph graph; + auto tick = graph.add_node("net_tick", + [&world, rec = world.net_snapshot.recorder()](Float_store& outbox, Net& n) mutable + { + (void)world.net_outbox_stream.commit(); + tick_networking(outbox, n); + rec.stage([values = n.values()](Net& snapshot) { snapshot.assign(values); }); + }, + world.net_outbox, world.net); + graph.add_node("net_publish", ts::publish_fn(world.net_snapshot), world.net_snapshot.state()).after(tick); + graph.set_default_priority(ts::Priority::high); + graph.compile(); + return graph; +} + +// The fixed-rate variant's two clocks, running beside whatever drives the frame graph. Builds +// both tick graphs, primes them (two physics ticks, so the interpolated pair holds two real +// versions, and one network tick), then drives each from its own `ts::Periodic`. Destruction +// stops both clocks and runs one more tick of each, committing whatever the last frames staged. +class Fixed_rate_ticks +{ +public: + explicit Fixed_rate_ticks(World& world) + : physics_(build_physics_tick_graph(world)) + , network_(build_network_tick_graph(world)) + , physics_clock_(std::chrono::microseconds(16'667), ts::Sleep_options{ .token = stop_.token(), .priority = ts::Priority::high }) + , network_clock_(std::chrono::microseconds(33'333), ts::Sleep_options{ .token = stop_.token(), .priority = ts::Priority::high }) + { + physics_.execute().sync(); + physics_.execute().sync(); + network_.execute().sync(); + physics_driver_ = drive(physics_, physics_clock_, physics_ticks_); + network_driver_ = drive(network_, network_clock_, network_ticks_); + } + + ~Fixed_rate_ticks() + { + stop_.request_cancel(); + physics_driver_.sync(); + network_driver_.sync(); + physics_.execute().sync(); + network_.execute().sync(); + } + + Fixed_rate_ticks(const Fixed_rate_ticks&) = delete; + Fixed_rate_ticks& operator=(const Fixed_rate_ticks&) = delete; + +private: + // Runs the ticks each wake reports, at most two: a clock that fell further behind slows + // simulated time down rather than stalling the frame behind a burst of ticks. + static ts::Task drive(ts::Static_task_graph& graph, ts::Periodic& clock, long long& ticks) + { + for (;;) + { + const int due = co_await clock.next(); + if (due == 0) + co_return; + const int run = std::min(due, 2); + for (int i = 0; i < run; ++i) + co_await graph.execute(); + ticks += run; + } + } + + ts::Cancellation_source stop_; // before the clocks, which take its token + ts::Static_task_graph physics_; + ts::Static_task_graph network_; + ts::Periodic physics_clock_; + ts::Periodic network_clock_; + long long physics_ticks_ = 0; + long long network_ticks_ = 0; + ts::Task physics_driver_; + ts::Task network_driver_; +}; + // --- the draw-producer node builders ---------------------------------------------- // The two shapes differ in access, so each branches on `opt`. Baseline writes // `draw_lists` directly (a write grant -> the producers serialise on the queue); @@ -1011,6 +1224,23 @@ void tick_propagation(const Local_xf& local_xf, const Bodies& bodies, const Velo rec.stage([batch = std::move(out)](Transforms& t) { t.apply(batch); }); } +// The fixed-rate variant's propagation: bodies interpolated between the physics tick's last two +// published snapshots, by where this frame falls between their publish instants. +void tick_propagation_interpolated(const Local_xf& local_xf, const Bodies& previous_bodies, const Bodies& current_bodies, + double fraction, ts::Recorder& rec) +{ + const float f = static_cast(fraction); + std::vector out(static_cast(local_xf.size())); + for (int i = 0, n = local_xf.size(); i < n; ++i) + { + const float b0 = previous_bodies.get(i); + const float b1 = current_bodies.get(i); + out[static_cast(i)] = local_xf.get(i) + b0 + f * (b1 - b0); + } + parallel_cost(budget::propagation); + rec.stage([batch = std::move(out)](Transforms& t) { t.apply(batch); }); +} + // Render pipeline: consumes last frame's transforms (declared before the flip), // so it overlaps this frame's simulation - the render thread with one frame of // latency. Its own working stores (visibility, shadows) are this-frame. @@ -1441,6 +1671,9 @@ void frame_stats(int frames, float scale, Frame_variant variant, double& avg_ms, ts::Static_task_graph graph; if (!graph_free) graph = build_frame_graph(world, variant); + std::optional ticks; // destroyed before the graph and the world + if (variant == Frame_variant::fixed_rate) + ticks.emplace(world); using clock = std::chrono::steady_clock; auto t0 = clock::now(); @@ -1484,6 +1717,14 @@ void game_frame_free_stats(int frames, float scale, double& avg_ms, double& seri frame_stats(frames, scale, Frame_variant::graph_free, avg_ms, serial_ms, transform0); } +// The optimised frame with physics and networking on their own clocks (see `Fixed_rate_ticks`). +// Must produce the same `transform0` and draw count as the baseline. `serial_ms` is the frame +// variants' serial budget; here the physics and network share of it runs per tick, not per frame. +void game_frame_fixed_stats(int frames, float scale, double& avg_ms, double& serial_ms, float& transform0) +{ + frame_stats(frames, scale, Frame_variant::fixed_rate, avg_ms, serial_ms, transform0); +} + // Draw commands submitted over the last stats run. `submit` clears the queue, so this counts // only what the three producers pushed before it ran - an observable the producer/submit // ordering decides. The transform invariant cannot see that ordering (every mock system @@ -1593,6 +1834,9 @@ void trace_variant(int frames, Frame_variant variant, const char* base_SVG_path, World world{ entities }; ts::Static_task_graph graph = build_frame_graph(world, variant, DOT_path); + std::optional ticks; // the fixed-rate variant's clocks, untraced + if (variant == Frame_variant::fixed_rate) + ticks.emplace(world); #if TS_PROFILING ts::tools::Graph_trace trace; @@ -1657,6 +1901,9 @@ void trace_game_frame(int frames, const char* DOT_path, const char* SVG_path) ts::Scheduler_scope pool{ { .num_workers = static_cast(variant_workers) } }; trace_variant(frames, Frame_variant::baseline, SVG_path, "baseline", DOT_path, gt_baseline); trace_variant(frames, Frame_variant::optimised, SVG_path, "optimised", nullptr, gt_optimised); + // The frame graph only: its tick graphs run on the same workers, untraced (one traced graph + // at a time), and have no worker-less floor since a timer needs workers to deliver on. + trace_variant(frames, Frame_variant::fixed_rate, SVG_path, "fixed_rate", nullptr); } // Headless run of the optimised variant on a dedicated `workers`-thread @@ -1708,6 +1955,14 @@ void run_game_frame_sample(int frames, float scale) std::printf(" graph-free composition of the same frame: %.2f ms/frame (%+.1f%%), " "transform0 %.1f, %lld draw commands\n", free_ms, 100.0 * (free_ms - avg_ms) / avg_ms, free_transform0, free_drawn); + + // The optimised frame with physics (60 Hz) and networking (30 Hz) on their own clocks. + double fixed_ms = 0.0, fixed_serial_ms = 0.0; + float fixed_transform0 = 0.0f; + game_frame_fixed_stats(frames, scale, fixed_ms, fixed_serial_ms, fixed_transform0); + std::printf(" optimised frame with physics at 60 Hz and networking at 30 Hz on their own clocks: " + "%.2f ms/frame, transform0 %.1f, %lld draw commands\n", + fixed_ms, fixed_transform0, game_frame_draw_count()); } } // namespace sample diff --git a/src/guarded.cpp b/src/guarded.cpp index f9b75ac..718e87f 100644 --- a/src/guarded.cpp +++ b/src/guarded.cpp @@ -1,4 +1,5 @@ #include "ts/guarded.h" +#include "ts/timer.h" // the timer thread stops before the workers it delivers to #include "ts/detail/suspension_registry.h" #include @@ -25,6 +26,14 @@ std::mutex g_sched_mutex; std::unique_ptr g_scheduler; Scheduler_config g_config; std::atomic g_fast{ nullptr }; + +// Program exit without `destroy_scheduler`: the timer thread delivers wakeups through the +// scheduler, so it is stopped first. Declared after `g_scheduler`, so it is destroyed before it. +struct Timer_exit_stop +{ + ~Timer_exit_stop() { detail::timer_shutdown(false); } +}; +Timer_exit_stop g_timer_exit_stop; } Scheduler& global_scheduler() @@ -53,6 +62,8 @@ void destroy_scheduler() std::lock_guard lock(g_sched_mutex); if (!g_scheduler) ts::fatal("destroy_scheduler(): no Scheduler is running"); + // The timer thread first: it delivers wakeups as tasks on this scheduler. + detail::timer_shutdown(true); // Join first, publish "none running" second. `~Scheduler` joins the workers in its own // body, before any member is torn down, so the object stays fully valid for exactly as // long as a worker can still be executing. Clearing `g_fast` first opened a window where diff --git a/src/main.cpp b/src/main.cpp index da78158..6a9bd99 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -22,6 +22,8 @@ void run_events_sample(); void run_coloring_sample(); void run_scope_access_sample(); void run_lazy_BVH_sample(); +void run_fixed_rate_sample(int frames); +void stress_fixed_rate(int frames); } #include @@ -75,6 +77,7 @@ int main(int argc, char** argv) " width, which on a wide machine hides the contended interleavings\n" " a 2-core CI runner produces every time\n" " --bench run the benchmarks only\n" + " --fixed-rate [n] run the fixed-rate sample n frames (default 600)\n" " --stress run the sample many frames at a fast scale (for sanitizers)\n" " --dot [path] write the game_frame graph structure as Graphviz DOT\n" " (default sample_game_frame.dot; render with show_graph.bat)\n" @@ -141,10 +144,19 @@ int main(int argc, char** argv) return 0; } - // Stress entry: just the sample, many frames, fast scale (for sanitizers). + // The fixed-rate sample alone: a 60 Hz physics graph on its own clock beside the frame loop. + if (argc >= 2 && std::strcmp(argv[1], "--fixed-rate") == 0) + { + int frames = argc >= 3 ? std::atoi(argv[2]) : 600; + sample::run_fixed_rate_sample(frames > 0 ? frames : 600); + return exit_checking_ensure_failures(0); + } + + // Stress entry: just the samples, many frames, fast scale (for sanitizers). if (argc >= 2 && std::strcmp(argv[1], "--stress") == 0) { sample::run_game_frame_sample(2000, 0.2f); + sample::stress_fixed_rate(400); return exit_checking_ensure_failures(0); } @@ -192,6 +204,7 @@ int main(int argc, char** argv) run_all_tests(); sample::run_game_frame_sample(); sample::run_physics_sample(); + sample::run_fixed_rate_sample(600); sample::run_blackboard_sample(); sample::run_events_sample(); sample::run_coloring_sample(); diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 465ba39..6473a75 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -1,4 +1,5 @@ #include "ts/scheduler.h" +#include "ts/detail/task_block.h" // the ambient task state a yield point's nested dispatch resets #include "ts/detail/worker_thread.h" #include @@ -169,6 +170,8 @@ void Scheduler::submit(Task_func_ptr func, void* data, Priority priority) return; } + if (priority == Priority::high) + detail::high_queued.fetch_add(1, std::memory_order_relaxed); // before the push: never under-counts queues_[static_cast(priority)].push({ func, data }); signal_submit(); @@ -289,6 +292,7 @@ bool Scheduler::find_work(int worker_index, detail::Task_entry& out) if (queues_[0].pop(out)) // global high (strict) { + detail::high_queued.fetch_sub(1, std::memory_order_relaxed); ++since_low; return true; } @@ -465,4 +469,91 @@ bool Scheduler::all_empty() const return true; } +namespace detail +{ + +namespace +{ + +// Clears the thread's ambient task state for a task run inside another task's yield point and +// restores it afterwards, so the nested task starts as it would at the top of the worker loop: +// no current task, no grants, no scope children, no rule relaxation, no trace owner. Its own +// dispatch installs what it needs. Under a traced run the nested span is added to +// `Nested_span_state`, which the yielding body's `Trace_busy_scope` subtracts, so the span is +// credited as body time once - by the nested task's own scope. +class Nested_dispatch_scope +{ +public: + Nested_dispatch_scope() noexcept + : task_(Current_task::exchange(Task_ptr{})) + , access_(access_load()) + , scope_children_(Scope_children::exchange(nullptr)) + { + access_store(nullptr); +#if TS_RULES_ANY + relaxed_ = relaxed_load(); + relaxed_store(0); +#endif +#if TS_PROFILING + owner_ = Trace_owner_state::exchange(-1); + in_functor_ = In_functor_state::exchange(false); + if (trace_owner_armed.load(std::memory_order_relaxed) != 0) + t0_ = std::chrono::steady_clock::now().time_since_epoch().count(); +#endif + } + + ~Nested_dispatch_scope() + { +#if TS_PROFILING + if (t0_ != 0) + Nested_span_state::add(std::chrono::steady_clock::now().time_since_epoch().count() - t0_); + In_functor_state::store(in_functor_); + Trace_owner_state::store(owner_); +#endif +#if TS_RULES_ANY + relaxed_store(relaxed_); +#endif + Scope_children::store(scope_children_); + access_store(access_); + (void)Current_task::exchange(std::move(task_)); + } + + Nested_dispatch_scope(const Nested_dispatch_scope&) = delete; + Nested_dispatch_scope& operator=(const Nested_dispatch_scope&) = delete; + +private: + Task_ptr task_; + const Access_context* access_; + std::vector* scope_children_; +#if TS_RULES_ANY + unsigned relaxed_ = 0; +#endif +#if TS_PROFILING + int owner_ = -1; + bool in_functor_ = false; + long long t0_ = 0; +#endif +}; + +} // namespace + +// The entry runs exactly as a worker would run it, minus the busy timing: the yielding task's +// `run_task` span already covers this thread. One entry per call, so a yield point's latency +// is bounded by one task. A task it runs that settles a coroutine hands the resume to this +// thread's resume trampoline; if the yield point is itself inside a resumed segment, that +// resume runs once the yielding segment returns to the trampoline. +void yield_to_high(Priority own) noexcept +{ + if (own == Priority::high || current_worker_index() < 0) + return; + Task_entry task; + if (!global_scheduler().queues_[static_cast(Priority::high)].pop(task)) + return; // taken by another worker since the counter was read + high_queued.fetch_sub(1, std::memory_order_relaxed); + Nested_dispatch_scope scope; + task.func_(task.data_); +} + +} // namespace detail + } // namespace ts diff --git a/src/static_task_graph.cpp b/src/static_task_graph.cpp index 3500956..04984e0 100644 --- a/src/static_task_graph.cpp +++ b/src/static_task_graph.cpp @@ -67,6 +67,7 @@ Static_task_graph::Static_task_graph(Static_task_graph&& other) noexcept , links_lent_(other.links_lent_) , run_(std::move(other.run_)) , compiled_(other.compiled_) + , default_priority_(other.default_priority_) , trace_(other.trace_) {} @@ -140,10 +141,23 @@ Graph_node& Graph_node::before(const Graph_node& successor) Graph_node& Graph_node::set_priority(Priority p) { if (graph_) + { graph_->nodes_[index_].priority = p; // applied to the block in execute() (see re-arm) + graph_->nodes_[index_].priority_set = true; + } return *this; } +void Static_task_graph::set_default_priority(Priority p) +{ + default_priority_ = p; + for (Node& node : nodes_) + { + if (!node.priority_set) + node.priority = p; + } +} + Graph_node& Graph_node::set_inline() { if (graph_) diff --git a/src/timer.cpp b/src/timer.cpp new file mode 100644 index 0000000..f274f21 --- /dev/null +++ b/src/timer.cpp @@ -0,0 +1,370 @@ +#include "ts/timer.h" +#include "ts/coroutine_support.h" +#include "ts/fatal.h" +#include "ts/guarded.h" // global_scheduler + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +// Windows 10 1803+. Without it a waitable timer, like a condition variable timeout, wakes on +// the system timer tick (~15.6 ms by default), which is a whole period of a 60 Hz tick. +#ifndef CREATE_WAITABLE_TIMER_HIGH_RESOLUTION +#define CREATE_WAITABLE_TIMER_HIGH_RESOLUTION 0x00000002 +#endif +#endif + +namespace ts +{ +namespace +{ + +using Clock = std::chrono::steady_clock; + +// One armed wait. Owned by its heap entry; the cancel callback it carries captures a raw +// pointer to it, so the callback's lifetime is nested inside the state's. +struct Sleep_state : detail::Ref_counted +{ + detail::Task_ptr block; // the returned task's block + Priority priority = Priority::normal; // the delivery task's priority + bool live = true; // neither fired nor cancelled; guarded by the timer mutex + // Held while live: the wakeup comes from a thread the scheduler does not run, so the + // deadlock net must not read a quiescent pool as a deadlock (`External_wait`). + std::unique_ptr outstanding; + std::optional on_cancel; +}; + +struct Entry +{ + Clock::time_point deadline; + std::uint64_t serial; // FIFO among equal deadlines + detail::Ref_ptr state; +}; + +// Heap order for `std::push_heap`/`pop_heap`: the earliest deadline at the front. +struct Later +{ + bool operator()(const Entry& a, const Entry& b) const noexcept + { + return a.deadline != b.deadline ? a.deadline > b.deadline : a.serial > b.serial; + } +}; + +// A wakeup to deliver once the timer mutex is released. +struct Wakeup +{ + detail::Task_ptr block; + Priority priority; + std::unique_ptr outstanding; // released only after the delivery is queued +}; + +// Settle `block` from a task at `priority`, never inline: settling resumes awaiting coroutines +// on the settling thread, and neither the timer thread nor a cancelling thread may run them. +void deliver(Wakeup wakeup, bool cancelled) +{ + (void)ts::launch([block = std::move(wakeup.block), cancelled]() mutable + { + if (cancelled) + block->cancel(); + else + block->complete(); + }, { .priority = wakeup.priority, .name = "ts::sleep wakeup" }); +} + +// The deadline keeper: a min-heap of armed sleeps and the one thread that waits for its head. +class Timer_service +{ +public: + Timer_service() + { +#if defined(_WIN32) + timer_ = CreateWaitableTimerExW(nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); + if (timer_ == nullptr) + timer_ = CreateWaitableTimerExW(nullptr, nullptr, 0, TIMER_ALL_ACCESS); // pre-1803 fallback + wake_ = CreateEventW(nullptr, FALSE, FALSE, nullptr); + if (timer_ == nullptr || wake_ == nullptr) + ts::fatal("ts::sleep: could not create the timer thread's wait objects"); +#endif + } + + // Arm `state` for `deadline`, starting the thread if it is not running. + void arm(Clock::time_point deadline, detail::Ref_ptr state) + { + bool earlier; + { + std::scoped_lock lock(mutex_); + if (!thread_.joinable()) + { + stopping_ = false; + thread_ = std::thread([this] { run(); }); + } + state->outstanding = std::make_unique(); + ++live_; + earlier = heap_.empty() || deadline < heap_.front().deadline; + heap_.push_back(Entry{ deadline, next_serial_++, std::move(state) }); + std::push_heap(heap_.begin(), heap_.end(), Later{}); + } + if (earlier) + wake(); + } + + // The cancel callback's body: settle the wait cancelled now. The entry stays in the heap + // until its deadline and is dropped there. + void cancel(Sleep_state* state) + { + Wakeup wakeup; + { + std::scoped_lock lock(mutex_); + if (!state->live) + return; + state->live = false; + --live_; + wakeup = Wakeup{ state->block, state->priority, std::move(state->outstanding) }; + } + deliver(std::move(wakeup), true); + } + + void shutdown(bool check_armed) noexcept + { + std::vector dropped; // destroyed after the lock: a state's cancel callback takes it + { + std::scoped_lock lock(mutex_); + if (!thread_.joinable()) + return; +#if TS_SAFETY_CHECKS + if (check_armed && live_ != 0) + { + ts::fatal("destroy_scheduler with a ts::sleep still armed - await or cancel every sleep " + "(and every Periodic::next()) before tearing the scheduler down"); + } +#else + (void)check_armed; +#endif + stopping_ = true; + for (Entry& e : heap_) + e.state->live = false; + dropped = std::move(heap_); + heap_.clear(); + live_ = 0; + } + wake(); + thread_.join(); + } + +private: + void run() + { + std::vector due; + std::vector fire; + std::unique_lock lock(mutex_); + while (!stopping_) + { + if (heap_.empty()) + { + park(lock, nullptr); + continue; + } + const Clock::time_point head = heap_.front().deadline; + if (Clock::now() < head) + { + park(lock, &head); + continue; + } + const Clock::time_point now = Clock::now(); + while (!heap_.empty() && heap_.front().deadline <= now) + { + std::pop_heap(heap_.begin(), heap_.end(), Later{}); + due.push_back(std::move(heap_.back())); + heap_.pop_back(); + } + for (Entry& e : due) + { + if (e.state->live) + { + e.state->live = false; + --live_; + fire.push_back(Wakeup{ e.state->block, e.state->priority, std::move(e.state->outstanding) }); + } + } + lock.unlock(); + for (Wakeup& wakeup : fire) + deliver(std::move(wakeup), false); + fire.clear(); + due.clear(); + lock.lock(); + } + } + + // Wait until `deadline` (null = indefinitely) or a `wake()`, whichever comes first. The + // caller re-checks the heap under the lock afterwards, so a spurious return is harmless. + void park(std::unique_lock& lock, const Clock::time_point* deadline) + { +#if defined(_WIN32) + if (deadline != nullptr) + { + using Hundred_ns = std::chrono::duration>; + const long long ticks = std::chrono::ceil(*deadline - Clock::now()).count(); + if (ticks <= 0) + return; + LARGE_INTEGER due; + due.QuadPart = -ticks; // negative = relative + SetWaitableTimerEx(timer_, &due, 0, nullptr, nullptr, nullptr, 0); + lock.unlock(); + HANDLE handles[2] = { wake_, timer_ }; + WaitForMultipleObjects(2, handles, FALSE, INFINITE); + lock.lock(); + } + else + { + lock.unlock(); + WaitForSingleObject(wake_, INFINITE); + lock.lock(); + } +#else + if (deadline != nullptr) + cv_.wait_until(lock, *deadline); + else + cv_.wait(lock); +#endif + } + + // Callable with or without the mutex held: the parked thread re-checks under it, and the + // auto-reset event (or the condition variable's predicate loop) keeps a wake that lands + // before the park from being lost. + void wake() + { +#if defined(_WIN32) + SetEvent(wake_); +#else + cv_.notify_one(); +#endif + } + + std::mutex mutex_; + std::vector heap_; + std::uint64_t next_serial_ = 0; + int live_ = 0; + bool stopping_ = false; + std::thread thread_; +#if defined(_WIN32) + HANDLE timer_ = nullptr; + HANDLE wake_ = nullptr; +#else + std::condition_variable cv_; +#endif +}; + +// Created on first use and never destroyed: program exit stops its thread through +// `timer_shutdown` (called from the scheduler holder's teardown), and leaving the object alive +// means that call can never reach a destroyed service, whatever the static destruction order. +std::atomic g_service{ nullptr }; + +Timer_service& timer_service() +{ + if (Timer_service* service = g_service.load(std::memory_order_acquire)) + return *service; + auto* fresh = new Timer_service(); + Timer_service* expected = nullptr; + if (!g_service.compare_exchange_strong(expected, fresh, std::memory_order_acq_rel)) + { + delete fresh; + return *expected; + } + return *fresh; +} + +} // namespace + +Task sleep_until(Clock::time_point deadline, Sleep_options opts, std::source_location site) +{ + if (global_scheduler().single_threaded()) + { + ts::fatal("ts::sleep in worker-less mode - there is no worker to deliver the wakeup on, and the " + "timer thread must not run the waiting task itself"); + } + detail::Task_ptr block = detail::make_bare_block(); + detail::set_task_name(block, Named(site)); + Task result(block); + if (opts.token.is_cancel_requested()) + { + block->cancel(); + return result; + } + if (deadline <= Clock::now()) + { + block->complete(); + return result; + } + + detail::Ref_ptr state = detail::make_ref(); + state->block = block; + state->priority = detail::resolved_priority(opts.priority); + Timer_service& service = timer_service(); + service.arm(deadline, state); + // Registered after arming, so a token requested in between settles through `cancel`. A + // token already requested runs the callback here, in the constructor. + state->on_cancel.emplace(opts.token, [&service, raw = state.get()] { service.cancel(raw); }); + return result; +} + +Task sleep(Clock::duration duration, Sleep_options opts, std::source_location site) +{ + return sleep_until(Clock::now() + duration, std::move(opts), site); +} + +Periodic::Periodic(Clock::duration period, Sleep_options opts, std::source_location site) + : period_(period) + , next_deadline_(Clock::now() + period) + , opts_(std::move(opts)) + , site_(site) +{ + if (period <= Clock::duration::zero()) + ts::fatal("ts::Periodic: the period must be positive"); +} + +Task Periodic::next() +{ + if (opts_.token.is_cancel_requested()) + co_return 0; + Clock::time_point now = Clock::now(); + if (now < next_deadline_) + { + co_await sleep_until(next_deadline_, opts_, site_); + if (opts_.token.is_cancel_requested()) + co_return 0; + now = Clock::now(); + } + const int due = 1 + static_cast((now - next_deadline_) / period_); + next_deadline_ += period_ * due; + co_return due; +} + +void Periodic::reset() +{ + next_deadline_ = Clock::now() + period_; +} + +namespace detail +{ +void timer_shutdown(bool check_armed) noexcept +{ + if (Timer_service* service = g_service.load(std::memory_order_acquire)) + service->shutdown(check_armed); +} +} + +} // namespace ts diff --git a/tests/graph_tests.cpp b/tests/graph_tests.cpp index 034b5d2..86c224c 100644 --- a/tests/graph_tests.cpp +++ b/tests/graph_tests.cpp @@ -1405,6 +1405,26 @@ void test_death_nested_run_mode_conflict() { TS_CHECK(ts::test::expect_death("gr void test_death_nested_run_unquiet_scope() { TS_CHECK(ts::test::expect_death("graph_lend_unquiet_scope")); } void test_death_execute_in_flight() { TS_CHECK(ts::test::expect_death("graph_execute_in_flight")); } +// The graph's default priority applies to every node without its own, including nodes added +// after it was set; a node's own `set_priority` wins. +void test_graph_default_priority() +{ + auto running_priority = [] { return static_cast(ts::detail::resolved_priority(std::nullopt)); }; + std::atomic a_priority{ -1 }; + std::atomic b_priority{ -1 }; + std::atomic c_priority{ -1 }; + ts::Static_task_graph graph; + graph.add_node("a", [&] { a_priority.store(running_priority()); }); + graph.add_node("b", [&] { b_priority.store(running_priority()); }).set_priority(ts::Priority::low); + graph.set_default_priority(ts::Priority::high); + graph.add_node("c", [&] { c_priority.store(running_priority()); }); + graph.compile(); + graph.execute().sync(); + TS_CHECK(a_priority.load() == static_cast(ts::Priority::high)); + TS_CHECK(b_priority.load() == static_cast(ts::Priority::low)); + TS_CHECK(c_priority.load() == static_cast(ts::Priority::high)); +} + } // namespace void run_graph_tests() @@ -1476,4 +1496,5 @@ void run_graph_tests() run_if(with_harness, "TS_SAFETY_CHECKS=0", "death: graph move-constructed mid-run", test_death_graph_moved_mid_run); run_if(with_rule_in_task_sync, "TS_RULE_IN_TASK_SYNC off", "death: sync own object (sharp diagnostic)", test_death_sync_own_object); run("lifetime registration balance", test_lifetime_registration_balance); + run("default priority", test_graph_default_priority); } diff --git a/tests/integration_tests.cpp b/tests/integration_tests.cpp index 5f51e99..12bd755 100644 --- a/tests/integration_tests.cpp +++ b/tests/integration_tests.cpp @@ -16,6 +16,10 @@ void game_frame_stats(int frames, float time_scale, void game_frame_free_stats(int frames, float time_scale, double& avg_ms, double& serial_ms, float& transform0); long long game_frame_draw_count(); +void game_frame_fixed_stats(int frames, float time_scale, + double& avg_ms, double& serial_ms, float& transform0); +bool fixed_rate_self_check(int frames, float scale); +std::size_t fixed_rate_physics_hash(int ticks); } #include @@ -775,6 +779,42 @@ void test_parallel_for_in_node_no_reports() } #endif +// A fixed-rate physics graph on its own clock beside the frame graph (sample/fixed_rate.cpp): +// ticks run one at a time in order, render always interpolates two consecutive ticks, and +// every intent the frame staged is applied exactly once. +void test_fixed_rate_beside_frame() +{ + TS_CHECK(sample::fixed_rate_self_check(120, 0.05f)); +} + +// The game frame with physics and networking on their own clocks publishes the same transforms +// and submits the same draw commands as the baseline frame. +void test_engine_fixed_rate() +{ + double avg_ms = 0.0, serial_ms = 0.0; + float graph_xf = 0.0f, fixed_xf = 0.0f; + sample::game_frame_stats(5, 0.3f, avg_ms, serial_ms, graph_xf); + long long graph_drawn = sample::game_frame_draw_count(); + sample::game_frame_fixed_stats(5, 0.3f, avg_ms, serial_ms, fixed_xf); + long long fixed_drawn = sample::game_frame_draw_count(); + TS_CHECK(fixed_xf == 5.0f); + TS_CHECK(fixed_drawn == graph_drawn); +} + +// Given the same intent sequence the fixed-rate world is the same, whatever the worker count. +void test_fixed_rate_physics_deterministic() +{ + const std::size_t reference = sample::fixed_rate_physics_hash(40); + { + ts::Scheduler_scope pool{ { .num_workers = 1 } }; + TS_CHECK(sample::fixed_rate_physics_hash(40) == reference); + } + { + ts::Scheduler_scope pool{ { .num_workers = 4 } }; + TS_CHECK(sample::fixed_rate_physics_hash(40) == reference); + } +} + void run_integration_tests() { std::printf("\n[integration] tests\n"); @@ -811,4 +851,7 @@ void run_integration_tests() run("engine frame without a graph", test_engine_graph_free); run("oversubscription no deadlock", test_oversubscription_no_deadlock); run("deep await chain no deadlock", test_deep_await_chain_no_deadlock); + run("fixed-rate graph beside a frame graph", test_fixed_rate_beside_frame); + run("fixed-rate physics is deterministic", test_fixed_rate_physics_deterministic); + run("engine frame with fixed-rate physics and networking", test_engine_fixed_rate); } diff --git a/tests/parallel_tests.cpp b/tests/parallel_tests.cpp index 0006661..fb61de2 100644 --- a/tests/parallel_tests.cpp +++ b/tests/parallel_tests.cpp @@ -6,9 +6,12 @@ #include #include +#include #include #include +#include #include +#include #include #include @@ -453,6 +456,48 @@ void test_colored_edges() TS_CHECK(calls.load() == 6); // 2 non-empty items x 3 rounds } +// A chunk boundary is a yield point: once both workers are inside a normal parallel_for, a high +// task queued behind it starts within about one chunk, not after the rest of the loop (~380 ms). +void test_parallel_for_yields_to_high() +{ + using Clock = std::chrono::steady_clock; + ts::Scheduler_scope pool{ { .num_workers = 2 } }; + std::mutex executors_mutex; + std::set executors; + std::atomic executor_count{ 0 }; + std::atomic high_started_at{ 0 }; + ts::Task loop = ts::launch([&] + { + ts::parallel_for(40, [&](int) + { + { + std::scoped_lock lock(executors_mutex); + if (executors.insert(std::this_thread::get_id()).second) + executor_count.fetch_add(1); + } + const auto until = Clock::now() + std::chrono::milliseconds(20); + while (Clock::now() < until) + { + } + }, { .balance = ts::Balance::unbalanced }); + }); + // Both workers must be inside the loop before the high task is queued: an idle worker would + // take it straight from the queue, and the test would measure nothing. + const auto give_up = Clock::now() + std::chrono::seconds(2); + while (executor_count.load() < 2 && Clock::now() < give_up) + std::this_thread::yield(); + TS_CHECK(executor_count.load() == 2); + const Clock::time_point queued_at = Clock::now(); + ts::Task high = ts::launch([&] + { + high_started_at.store(Clock::now().time_since_epoch().count()); + }, { .priority = ts::Priority::high }); + high.sync(); + loop.sync(); + const auto latency = Clock::time_point(Clock::duration(high_started_at.load())) - queued_at; + TS_CHECK(latency < std::chrono::milliseconds(150)); +} + } // namespace void run_parallel_tests() @@ -476,4 +521,5 @@ void run_parallel_tests() run("parallel_for_colored counts", test_colored_counts); run("parallel_for_colored determinism across concurrency", test_colored_determinism); run("parallel_for_colored edges", test_colored_edges); + run("parallel_for yields to a queued high task between chunks", test_parallel_for_yields_to_high); } diff --git a/tests/scheduler_tests.cpp b/tests/scheduler_tests.cpp index bc95abd..7029912 100644 --- a/tests/scheduler_tests.cpp +++ b/tests/scheduler_tests.cpp @@ -1,10 +1,12 @@ #include "scheduler_scope.h" #include "scheduler_tests.h" #include "ts/scheduler.h" +#include "ts/task.h" #include "harness.h" #include "test_util.h" #include +#include #include #include #include @@ -356,6 +358,98 @@ void test_global_normal_valve() std::this_thread::yield(); // drain before the scope tears the pool down } +// --- yield points ------------------------------------------------------------------------ + +// A normal task that reaches a yield point while a high task is queued runs the high task +// there, on its own thread. With one worker the high task could not run anywhere else before +// the normal task finished. +static void test_yield_runs_pending_high() +{ + ts::Scheduler_scope pool{ { .num_workers = 1 } }; + std::atomic started{ false }; + std::atomic high_ran{ false }; + std::atomic high_thread{}; + std::thread::id normal_thread{}; + bool high_ran_inside = false; + ts::Task normal = ts::launch([&] + { + normal_thread = std::this_thread::get_id(); + started.store(true); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (!high_ran.load() && std::chrono::steady_clock::now() < deadline) + ts::yield(); + high_ran_inside = high_ran.load(); + }); + while (!started.load()) + std::this_thread::yield(); + ts::Task high = ts::launch([&] + { + high_thread.store(std::this_thread::get_id()); + high_ran.store(true); + }, { .priority = ts::Priority::high }); + normal.sync(); + high.sync(); + TS_CHECK(high_ran_inside); + TS_CHECK(high_thread.load() == normal_thread); +} + +// With nothing pending a yield point is a load and a branch. +static void test_yield_without_pending_is_cheap() +{ + const auto t0 = std::chrono::steady_clock::now(); + for (int i = 0; i < 1'000'000; ++i) + ts::yield(); + TS_CHECK(std::chrono::steady_clock::now() - t0 < std::chrono::milliseconds(500)); +} + +// A high task's yield point does not run another high task: order among equals stays the +// queue's. +static void test_yield_high_does_not_yield() +{ + ts::Scheduler_scope pool{ { .num_workers = 1 } }; + std::atomic started{ false }; + std::atomic second_ran{ false }; + bool second_ran_inside = true; + ts::Task first = ts::launch([&] + { + started.store(true); + const auto until = std::chrono::steady_clock::now() + std::chrono::milliseconds(50); + while (std::chrono::steady_clock::now() < until) + ts::yield(); + second_ran_inside = second_ran.load(); + }, { .priority = ts::Priority::high }); + while (!started.load()) + std::this_thread::yield(); + ts::Task second = ts::launch([&] { second_ran.store(true); }, { .priority = ts::Priority::high }); + first.sync(); + second.sync(); + TS_CHECK(!second_ran_inside); +} + +// Off a worker a yield point is a no-op: the blue thread does not run queued work. +static void test_yield_off_worker_is_noop() +{ + ts::Scheduler_scope pool{ { .num_workers = 1 } }; + std::atomic started{ false }; + std::atomic release{ false }; + ts::Task occupier = ts::launch([&] + { + started.store(true); + while (!release.load()) + std::this_thread::yield(); + }); + while (!started.load()) + std::this_thread::yield(); + std::atomic high_ran{ false }; + ts::Task high = ts::launch([&] { high_ran.store(true); }, { .priority = ts::Priority::high }); + ts::yield(); + TS_CHECK(!high_ran.load()); + release.store(true); + occupier.sync(); + high.sync(); + TS_CHECK(high_ran.load()); +} + void run_scheduler_tests() { std::printf("\n[scheduler] tests\n"); @@ -381,4 +475,8 @@ void run_scheduler_tests() []{ TS_CHECK(ts::test::expect_death("scheduler_use_after_destroy")); }); run("death: destroy_scheduler with none running", []{ TS_CHECK(ts::test::expect_death("scheduler_destroy_twice")); }); + run("yield: a pending high task runs inside a normal task's yield point", test_yield_runs_pending_high); + run("yield: with nothing pending it is cheap", test_yield_without_pending_is_cheap); + run("yield: a high task does not yield to high", test_yield_high_does_not_yield); + run("yield: off a worker it is a no-op", test_yield_off_worker_is_noop); } diff --git a/tests/tests.cpp b/tests/tests.cpp index caab59e..a857f51 100644 --- a/tests/tests.cpp +++ b/tests/tests.cpp @@ -24,8 +24,10 @@ #include "deferred_tests.h" #include "versioned_tests.h" #include "event_bus_tests.h" +#include "timer_tests.h" #include "ts/deferred.h" +#include "ts/timer.h" #include "ts/versioned.h" #include @@ -52,12 +54,32 @@ void run_all_tests() run_deferred_tests(); run_versioned_tests(); run_event_bus_tests(); + run_timer_tests(); } // Death scenario body: acquire a `Guarded` write guard, then `co_await` other work while // still holding it - the pipe-held-across-suspension anti-pattern. Runs eagerly, so the fatal // fires during the call below, before `sync()`. `never` is never triggered, so `co_await never` // always reaches `await_suspend` (the detector) rather than escaping. +// A `Versioned` payload whose reads the harness checks, for the version-view death scenarios. +struct Checked_value +{ + int read() const + { + TS_CHECK_ACCESS(); + return value; + } + int value = 0; +}; + +// Death scenario body: `co_await` while a version view is live - the view is a held grant, so +// this is the same anti-pattern as awaiting under an `Access_guard`. +static ts::Task await_under_version_view(ts::Versioned& v) +{ + [[maybe_unused]] auto [previous, current, stamps] = co_await ts::read_last_versions(v); + co_await ts::launch([] {}); +} + static ts::Task coro_await_under_guard(ts::Guarded& w, ts::Signal& never) { auto g = co_await ts::read_write(w); @@ -695,6 +717,39 @@ void run_death_scenario(const char* name) ts::Access_scope scope(ctx); v.publish_into(other); // not this Versioned's front -> fatal } + else if (std::strcmp(name, "versioned_history_replay") == 0) + { + // The rotated shadow is two versions behind; one replayed batch cannot resync it -> fatal + ts::Versioned v{ ts::Named{}, ts::Resync::replay }; + } +#if TS_RULE_ON(TS_RULE_IN_TASK_SYNC) + else if (std::strcmp(name, "versioned_last_versions_in_task") == 0) + { + ts::Versioned v{ ts::Named{} }; + ts::launch([&v] + { + [[maybe_unused]] auto [previous, current, stamps] = v.read_last_versions(); // no grant -> fatal + }).sync(); + } +#endif + else if (std::strcmp(name, "versioned_previous_after_view") == 0) + { + ts::Versioned v{ ts::Named{} }; + const Checked_value* previous_ptr = nullptr; + { + [[maybe_unused]] auto [previous, current, stamps] = v.read_last_versions(); + previous_ptr = &previous; + (void)previous.read(); // granted while the view lives + } + (void)previous_ptr->read(); // no grant once the view is gone -> fatal + } +#if TS_RULE_ON(TS_RULE_AWAIT_UNDER_GUARD) + else if (std::strcmp(name, "versioned_await_under_view") == 0) + { + ts::Versioned v{ ts::Named{} }; + await_under_version_view(v).sync(); // co_await while the view is live -> fatal + } +#endif else if (std::strcmp(name, "coro_await_under_guard") == 0) { ts::Guarded w{ ts::Named{} }; @@ -888,6 +943,18 @@ void run_death_scenario(const char* name) ts::destroy_scheduler(); ts::destroy_scheduler(); // none running -> fatal } + else if (std::strcmp(name, "timer_worker_less") == 0) + { + ts::Scheduler_scope inline_scope{ { .single_threaded = true } }; + (void)ts::sleep(std::chrono::milliseconds(1)); // no worker to deliver on -> fatal + } +#if TS_SAFETY_CHECKS + else if (std::strcmp(name, "timer_destroy_armed") == 0) + { + ts::Task pending = ts::sleep(std::chrono::seconds(10)); + ts::destroy_scheduler(); // a sleep is still armed -> fatal + } +#endif #if defined(__cpp_exceptions) || defined(_CPPUNWIND) // The body boundary (`detail::invoke_user_body`): an exception must not leave a body, on // any of the paths that invoke one. Each of these dies in the seam, not by unwinding into diff --git a/tests/timer_tests.cpp b/tests/timer_tests.cpp new file mode 100644 index 0000000..e2e0b70 --- /dev/null +++ b/tests/timer_tests.cpp @@ -0,0 +1,154 @@ +#include "timer_tests.h" +#include "harness.h" +#include "ts/coroutine_support.h" +#include "ts/scheduler.h" +#include "ts/task.h" +#include "ts/timer.h" + +#include +#include + +using ts::test::run; +using namespace ts::test; +using namespace std::chrono_literals; + +namespace +{ + +using Clock = std::chrono::steady_clock; + +// A sleep never settles before its deadline. +void test_sleep_waits_for_deadline() +{ + auto t0 = Clock::now(); + ts::sleep(20ms).sync(); + TS_CHECK(Clock::now() - t0 >= 20ms); +} + +// A deadline already passed, and a token already requested, both settle inside the call. +void test_sleep_settled_in_call() +{ + ts::Task past = ts::sleep_until(Clock::now() - 1ms); + TS_CHECK(past.is_done() && !past.is_cancelled()); + + ts::Cancellation_source source; + source.request_cancel(); + ts::Task cancelled = ts::sleep(10s, { .token = source.token() }); + TS_CHECK(cancelled.is_done() && cancelled.is_cancelled()); +} + +// Cancellation settles the wait promptly and cancelled, not at the deadline. +void test_sleep_cancel_is_prompt() +{ + ts::Cancellation_source source; + auto t0 = Clock::now(); + ts::Task wait = ts::sleep(10s, { .token = source.token() }); + std::this_thread::sleep_for(5ms); + source.request_cancel(); + wait.sync(); + TS_CHECK(wait.is_cancelled()); + TS_CHECK(Clock::now() - t0 < 5s); +} + +// A deadline earlier than the one the timer thread is parked on wakes it: the earlier sleep +// is not held back until the later deadline. +void test_sleep_earlier_deadline_preempts() +{ + ts::Task late = ts::sleep(400ms); + ts::Task early = ts::sleep(50ms); + auto t0 = Clock::now(); + early.sync(); + TS_CHECK(Clock::now() - t0 < 300ms); + TS_CHECK(!late.is_done()); + late.sync(); +} + +ts::Task worker_after_sleep() +{ + co_await ts::sleep(5ms); + co_return ts::current_worker_index(); +} + +// The wakeup is delivered as a task, so an awaiting coroutine resumes on a worker, never on the +// timer thread. +void test_sleep_resumes_on_worker() +{ + TS_CHECK(worker_after_sleep().sync() >= 0); +} + +// Steady state: each `next()` reports at least one grid point, and ten of them take ten periods. +void test_periodic_steady() +{ + ts::Periodic tick{ 5ms }; + auto t0 = Clock::now(); + int total = 0; + for (int i = 0; i < 10; ++i) + { + int due = tick.next().sync(); + TS_CHECK(due >= 1); + total += due; + } + TS_CHECK(total >= 10); + TS_CHECK(Clock::now() - t0 >= 45ms); +} + +// A consumer that falls behind learns how many grid points it missed, in one call. +void test_periodic_reports_missed_ticks() +{ + ts::Periodic tick{ 10ms }; + (void)tick.next().sync(); + std::this_thread::sleep_for(45ms); + int due = tick.next().sync(); + TS_CHECK(due >= 4); +} + +// A requested token ends the tick stream: the pending `next()` settles with 0, promptly. +void test_periodic_cancel() +{ + ts::Cancellation_source source; + ts::Periodic tick{ 10s, { .token = source.token() } }; + ts::Task pending = tick.next(); + auto t0 = Clock::now(); + source.request_cancel(); + TS_CHECK(pending.sync() == 0); + TS_CHECK(Clock::now() - t0 < 5s); +} + +// A blue thread waiting on a sleep while every worker is idle is not a deadlock: the armed +// sleep registers its pending wakeup with the deadlock net (`External_wait`). +void test_sleep_is_an_external_wait() +{ + ts::set_deadlock_net_window(50ms); + ts::sleep(300ms).sync(); + ts::set_deadlock_net_window(2000ms); + TS_CHECK(true); +} + +void test_sleep_worker_less_is_fatal() +{ + TS_CHECK(ts::test::expect_death("timer_worker_less")); +} + +void test_destroy_with_armed_sleep_is_fatal() +{ + TS_CHECK(ts::test::expect_death("timer_destroy_armed")); +} + +} // namespace + +void run_timer_tests() +{ + run("timer: sleep waits for its deadline", test_sleep_waits_for_deadline); + run("timer: past deadline and requested token settle in the call", test_sleep_settled_in_call); + run("timer: cancellation is prompt", test_sleep_cancel_is_prompt); + run("timer: an earlier deadline wakes the timer thread", test_sleep_earlier_deadline_preempts); + run("timer: the awaiting coroutine resumes on a worker", test_sleep_resumes_on_worker); + run("timer: Periodic steady state", test_periodic_steady); + run("timer: Periodic reports missed grid points", test_periodic_reports_missed_ticks); + run("timer: Periodic ends on cancellation", test_periodic_cancel); + run_if(with_rule_deadlock_net, "TS_ENABLED_RULES without deadlock_net", "timer: an armed sleep is an external wait", + test_sleep_is_an_external_wait); + run("timer: sleep in worker-less mode is fatal", test_sleep_worker_less_is_fatal); + run_if(with_harness, "TS_SAFETY_CHECKS=0", "timer: destroy_scheduler with an armed sleep is fatal", + test_destroy_with_armed_sleep_is_fatal); +} diff --git a/tests/timer_tests.h b/tests/timer_tests.h new file mode 100644 index 0000000..446bbb4 --- /dev/null +++ b/tests/timer_tests.h @@ -0,0 +1,3 @@ +#pragma once + +void run_timer_tests(); diff --git a/tests/versioned_tests.cpp b/tests/versioned_tests.cpp index 775a9c1..03df3cf 100644 --- a/tests/versioned_tests.cpp +++ b/tests/versioned_tests.cpp @@ -6,8 +6,12 @@ #include "test_util.h" #include +#include +#include #include +#include #include +#include #include #include @@ -510,6 +514,156 @@ void test_read_queued_option() TS_CHECK(body_thread.load() != caller); } +// --- History::current_and_previous --------------------------------------------------- + +using Versioned_pair = ts::Versioned; + +template +concept Has_last_versions = requires(V& v) { v.read_last_versions(); }; + +static_assert(!Has_last_versions>, "the pair read exists only with the previous version kept"); +static_assert(Has_last_versions); +static_assert(std::tuple_size_v> == 3); + +// Each publish rotates: the view reads the last two versions, and the stamps advance with them. +void test_last_versions_rotate() +{ + Versioned_pair v{ ts::Named{} }; + auto rec = v.recorder(); + { + auto [previous, current, stamps] = v.read_last_versions(); + TS_CHECK(previous == 0 && current == 0 && stamps.current_serial == 0); + } + rec.stage([](int& x) { x = 1; }); + v.publish().sync(); + rec.stage([](int& x) { x = 2; }); + v.publish().sync(); + ts::Version_stamps after_two; + { + auto [previous, current, stamps] = v.read_last_versions(); + TS_CHECK(previous == 1 && current == 2); + TS_CHECK(stamps.current_serial == 2); + TS_CHECK(stamps.previous_published <= stamps.current_published); + after_two = stamps; + } + rec.stage([](int& x) { x = 3; }); + v.publish().sync(); + auto [previous, current, stamps] = v.read_last_versions(); + TS_CHECK(previous == 2 && current == 3); + TS_CHECK(stamps.previous_published == after_two.current_published); +} + +void test_version_stamps_fraction() +{ + using std::chrono::milliseconds; + const auto t = std::chrono::steady_clock::now(); + ts::Version_stamps stamps{ t, t + milliseconds(10), 2 }; + TS_CHECK(std::abs(stamps.fraction_at(t + milliseconds(5)) - 0.5) < 1e-9); + TS_CHECK(stamps.fraction_at(t - milliseconds(1)) == 0.0); + TS_CHECK(stamps.fraction_at(t + milliseconds(20)) == 1.0); + ts::Version_stamps coinciding{ t, t, 0 }; + TS_CHECK(coinciding.fraction_at(t) == 1.0); +} + +// A node that declared the front reads the pair lent: it takes no read turn of its own, so the +// front's pipe shows one reader - the node - while the view is alive. +void test_last_versions_lent_in_node() +{ + Versioned_pair v{ ts::Named{} }; + auto rec = v.recorder(); + rec.stage([](int& x) { x = 5; }); + v.publish().sync(); + ts::detail::Pipe& pipe = ts::detail::Guarded_access::pipe(v.state()); + pipe.wait_until_idle(); // the copy resync's read has released the front + + int seen_previous = -1; + int seen_current = -1; + int readers = -1; + ts::Static_task_graph graph; + graph.add_node("reader", [&v, &pipe, &seen_previous, &seen_current, &readers](const int&) + { + auto [previous, current, stamps] = v.read_last_versions(); + seen_previous = previous; + seen_current = current; + std::scoped_lock lock(pipe.mutex); + readers = pipe.active_readers; + }, v.state()); + graph.compile(); + graph.execute().sync(); + TS_CHECK(seen_previous == 0 && seen_current == 5); + TS_CHECK(readers == 1); +} + +ts::Task read_pair_awaited(Versioned_pair& v) +{ + [[maybe_unused]] auto [previous, current, stamps] = co_await ts::read_last_versions(v); + co_return previous * 100 + current; +} + +// A coroutine that holds nothing awaits the pair: it queues behind a writer on the front and +// resumes with the view once the writer releases. +void test_last_versions_awaited() +{ + Versioned_pair v{ ts::Named{} }; + auto rec = v.recorder(); + rec.stage([](int& x) { x = 7; }); + v.publish().sync(); + std::atomic release{ false }; + ts::Task blocker = v.state().async([&release](int&) + { + while (!release.load()) + std::this_thread::yield(); + }); + ts::Task reader = read_pair_awaited(v); + TS_CHECK(!reader.is_done()); + release.store(true); + blocker.sync(); + TS_CHECK(reader.sync() == 7); +} + +// The blocking form on a blue thread takes its own read turn, so it waits out a writer. +void test_last_versions_blue_thread_parks() +{ + Versioned_pair v{ ts::Named{} }; + std::atomic release{ false }; + ts::Task blocker = v.state().async([&release](int&) + { + while (!release.load()) + std::this_thread::yield(); + }); + std::thread releaser([&release] + { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + release.store(true); + }); + { + [[maybe_unused]] auto [previous, current, stamps] = v.read_last_versions(); + TS_CHECK(release.load()); // granted only after the writer let go + } + releaser.join(); + blocker.sync(); +} + +void test_history_with_replay_is_fatal() +{ + TS_CHECK(ts::test::expect_death("versioned_history_replay")); +} + +void test_last_versions_blocking_in_task_is_fatal() +{ + TS_CHECK(ts::test::expect_death("versioned_last_versions_in_task")); +} + +void test_previous_after_view_is_fatal() +{ + TS_CHECK(ts::test::expect_death("versioned_previous_after_view")); +} + +void test_await_under_version_view_is_fatal() +{ + TS_CHECK(ts::test::expect_death("versioned_await_under_view")); +} + } // namespace void run_versioned_tests() @@ -542,4 +696,16 @@ void run_versioned_tests() run("versioned: awaited read", test_read_awaited); run("versioned: read with .queued enqueues", test_read_queued_option); run_if(with_harness, "TS_SAFETY_CHECKS=0", "versioned: destroy with a publish in flight is fatal", test_dtor_inflight_publish_is_fatal); + run("versioned: last versions rotate with each publish", test_last_versions_rotate); + run("versioned: stamps fraction", test_version_stamps_fraction); + run("versioned: last versions are lent inside a node", test_last_versions_lent_in_node); + run("versioned: last versions awaited behind a writer", test_last_versions_awaited); + run("versioned: last versions on a blue thread park behind a writer", test_last_versions_blue_thread_parks); + run("versioned: previous-version history with replay is fatal", test_history_with_replay_is_fatal); + run_if(with_rule_in_task_sync, "TS_ENABLED_RULES without in_task_sync", + "versioned: blocking last-versions read in a task is fatal", test_last_versions_blocking_in_task_is_fatal); + run_if(with_harness, "TS_SAFETY_CHECKS=0", "versioned: previous version after the view is fatal", + test_previous_after_view_is_fatal); + run_if(with_rule_await_under_guard, "TS_ENABLED_RULES without await_under_guard", + "versioned: co_await under a live version view is fatal", test_await_under_version_view_is_fatal); } diff --git a/tsan/tsan_main.cpp b/tsan/tsan_main.cpp index 6c92aa5..8106a66 100644 --- a/tsan/tsan_main.cpp +++ b/tsan/tsan_main.cpp @@ -21,6 +21,10 @@ void stress_game_frame_optimised(int frames, int workers); void run_blackboard_sample(); void stress_coloring(int frames); std::size_t physics_pose_hash(int frames); // final snapshot hash after `frames` frames +void stress_fixed_rate(int frames); +std::size_t fixed_rate_physics_hash(int ticks); +void game_frame_fixed_stats(int frames, float time_scale, + double& avg_ms, double& serial_ms, float& transform0); } #include "ts/parallel_for.h" #include "ts/scheduler.h" @@ -1057,6 +1061,18 @@ void stress_physics() (void)a; (void)b; } +// A fixed-rate graph on its own clock (`ts::Periodic`) beside a frame graph: the timer thread's +// wakeups, yield points inside the frame's bodies, and the `Versioned` pair read under +// concurrency. Then the physics graph alone, run-to-run determinism. +void stress_fixed_rate() +{ + sample::stress_fixed_rate(200); + std::size_t a = sample::fixed_rate_physics_hash(30); + std::size_t b = sample::fixed_rate_physics_hash(30); + assert(a == b); + (void)a; (void)b; +} + } // namespace // The entry point, renamed when this TU is compiled into the Windows binary. @@ -1107,6 +1123,7 @@ int main() std::puts("tsan: deferred stress"); stress_deferred(); std::puts("tsan: versioned stress"); stress_versioned(); std::puts("tsan: physics frames"); stress_physics(); + std::puts("tsan: fixed-rate graph"); stress_fixed_rate(); std::puts("tsan: blackboard frames"); sample::run_blackboard_sample(); std::puts("tsan: coloring frames"); sample::stress_coloring(10); std::puts("tsan: game_frame frames"); @@ -1126,6 +1143,15 @@ int main() float xf = 0.0f; sample::game_frame_free_stats(20, 0.2f, avg, serial, xf); } + std::puts("tsan: game_frame fixed-rate frames"); + for (int i = 0; i < 5; ++i) + { + // Physics at 60 Hz and networking at 30 Hz on their own clocks beside the frame graph: + // two tick graphs concurrent with it, their staged inputs and published snapshots. + double avg = 0.0, serial = 0.0; + float xf = 0.0f; + sample::game_frame_fixed_stats(20, 0.2f, avg, serial, xf); + } std::puts("tsan: game_frame optimised frames"); sample::stress_game_frame_optimised(40, 4); // gameplay Versioned + Deferred staging std::puts("tsan: done (no races)"); From a79ab3d461c5313fe25d5d7631387b853220f3f9 Mon Sep 17 00:00:00 2001 From: Andriy Date: Mon, 14 Sep 2026 11:59:31 +0100 Subject: [PATCH 2/3] docs: fixed-rate graphs, timers, yield points, Versioned history - guide.md: 6.6 fixed-rate graphs, 10.6 timers, yield points in 10.1 and 7, the previous-version history in 9.2, set_default_priority, tool-table rows, and two limitations. - design.md: yield points and the timer thread in 3, the history rotation in 6, and 6.1 on fixed-rate graphs as the logical-execution-time model. - example-frame-optimization.md 5.1: the fixed-rate variant measured against the baseline and optimised frames. It is neutral on frame time because the optimised frame is already core-bound. - Status updates in TODO 2.11, the timer design study, pattern-farming 2.2, the two read_pair mentions, CLAUDE.md and show_graph.bat. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QTtvN1mH7K6MWzpYiyJ88w --- CLAUDE.md | 5 +- docs/TODO.md | 8 +- docs/design.md | 88 +++++++++ docs/example-frame-optimization.md | 36 +++- docs/guide.md | 197 ++++++++++++++++++++- docs/internals/command-buffer-design.md | 5 +- docs/internals/deferred-versioned-state.md | 5 +- docs/internals/timer-primitive-design.md | 10 ++ docs/pattern-farming.md | 2 +- show_graph.bat | 5 +- 10 files changed, 347 insertions(+), 14 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f0352ab..ff10415 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,10 +31,11 @@ Two transformations landed in 2026-08 and define the current shape: the **evolve - **Access harness** (`access.{h,cpp}`) — `TS_CHECK_ACCESS()` at the top of every guarded method checks `this` against a thread-local `Access_context` the scheduler/pipe installs per task; a violation routes to `ts::fatal`. ~1 ns/call. Subtasks (e.g. `parallel_for` chunks) inherit the parent node's context. **Grant-window validity (2026-07)**: each `Pipe` carries a `write_epoch` (seqlock parity — bumped at write acquire/release under the pipe mutex, +2 on a graph write handoff; readers never bump); context entries declared under a pipe grant capture it, and `access_check` fatals with a stale-grant diagnostic when an inherited snapshot outlives its window (a non-nested `ts::launch` touching the launcher's data after the launcher's scope released — nested-gated sub-work is structurally never stale). `Access_context::check` returns granted/stale/none. Gated by `TS_SAFETY_CHECKS`. - **Waiting-rule policy** (`rules.h`, design of record `docs/internals/waiting-rule-policy.md`) — the coroutine-first waiting rules (`docs/internals/coroutine-first.md` §2) are enforced by runtime checks that fatal, and each is separately (1) compiled in via `TS_ENABLED_RULES` (a bitmask of `TS_RULE_*`) and (2) opted out of per scope via `ts::Relaxed_scope` for the rules that permit it. `ts::Rule` mirrors the bits: **`in_task_sync`** (`sync()`/`take()` inside a task), **`await_under_guard`** (`co_await` while a `Access_guard` is live), **`access_rank`** (awaiting an object out of declared rank order), **`circular_wait`** (a held-grant → awaited-pipe wait cycle), **`deadlock_net`** (quiescence with no possible external wakeup). Rule classes: `advisory` (`in_task_sync|access_rank|circular_wait` — a scoped `Relaxed_scope` opt-out, "I uphold this by means the library can't see"), `structural` (`await_under_guard` — compile-out-only, its absence corrupts rather than merely permits, so it is the ONE rule shipping keeps), `net` (`deadlock_net` — global, unscopable). `set_default_relaxed_rules(Rule)` is a process-wide advisory-relaxation baseline. `circular_wait`/`access_rank` read grant bookkeeping that exists only under `TS_SAFETY_CHECKS`, so they can't outlive it whatever the policy asks (the EFFECTIVE mask = policy ∩ build support). `rule_enforced(rule)` is the one predicate a check calls, only AFTER the cheap hazard condition is already true (the relaxation lookup stays off the common path). **`detail::relaxed_rules` is reached ONLY through `detail::relaxed_load()`/`relaxed_store(unsigned)`, both `TS_DETAIL_NO_INLINE` (2026-08-22)** - a compiler may resolve a thread-local's block address once and keep it in a coroutine frame across a suspension, so an INLINED read after a cross-thread resume answers for the SUSPENDING thread (MSVC 19.51 does; the class is cross-compiler with an incomplete upstream fix - LLVM #47179 / #63022 / D92661, and it extends to anything thread-identifying, e.g. clang's `pthread_self()` case and LLVM #72006). The barrier lives in the accessors, so every toucher (`relaxed_bits`, `Relaxed_scope` ctor/dtor, `Relaxed_carrier`, `snapshot_relaxed`, `Inherited_relaxed_scope`) inherits it and none carries its own `TS_DETAIL_NO_INLINE`; load/store BY VALUE (an `unsigned&` accessor re-opens the hazard one level up - the Rust `thread_local!`+`.with()` shape). Regression test "rules relaxed scope reads the resuming thread" (it fails again if the accessors are made inlinable - verified both directions); rationale in `docs/internals/waiting-rule-policy.md` §4.1. A `Relaxed_scope` entered in a coroutine body follows the ambient task state, not the thread (`Relaxed_carrier` on the promise re-installs it around every segment). **`ts::Rank`** (`access.h`) — a declared lock order (`Guarded(Named, Rank, args...)`), NOT defaulted, required only for objects dynamically awaited while another grant is held; the `access_rank` rule fatals on an out-of-order await. **Deadlock report, three tiers** (`docs/internals/waiting-rule-policy.md` §7): tiers 1 (the `circular_wait` cycle) and 2 (the `deadlock_net` quiescence net — the scheduler idle with nothing externally outstanding for `set_deadlock_net_window`, default 2 s) are free and always present; tier 3 is `TS_SUSPENSION_REGISTRY` — an opt-in per-suspension registry (who is suspended, what they await, what they hold), default ON in DEBUG only (a linked-list insert/remove per suspension, ~30 ns, ~8% on the suspend/resume microbench), which forces `TS_DEBUG_NAMES` on (a registry of block pointers is not a diagnostic). **`ts::External_wait`** (`task.h`) — the `deadlock_net` escape: hold one while a wakeup owned by a thread the scheduler doesn't run (OS I/O, GPU fence, a `Signal` from a dedicated thread, a `Frame_gate` `open()`) is outstanding, so the net doesn't false-positive; a FORGOTTEN registration produces a false deadlock report, which is why the report names the type. **`ts::Named`** (`named.h`) — the one unified debug-identity type for graph nodes (required), guarded objects (required), and tasks (optional, `Dispatch_options::name`/`Access_options::name` — both are a `ts::Named` since 2026-08 (M6), not a `const char*`: the call SITE is captured by the verb's own defaulted `source_location`, so an unnamed task is still identified, EXCEPT on the multi-object verbs, which end in an object pack and can carry no defaulted `source_location` — there `{.name = ts::Named{}}` is the only way to capture the site, and `detail::named_from` prefers the option's `Named` when non-empty). A `Named` is a literal OR a `{}` call-site capture (`file`+`line` only, three words); `named_display` renders it. The load-bearing library rule: a defaulted `source_location` captures the CALLER, so it must sit on the OUTERMOST function the user calls and be passed down explicitly — an inner helper re-defaulting it captures the library header (`tests/named_tests.cpp` asserts captured names point into the test file). Debug identity is gated by `TS_DEBUG_NAMES` (default = the harness, forced on by `TS_SUSPENSION_REGISTRY`). - **`fatal.{h,cpp}`** — all non-recoverable failures call `ts::fatal` (message + `std::stacktrace` + `abort`). **Exception-agnostic since 2026-08-21**: the library never throws its own exceptions (its only `try`/`catch` - and a bare `throw;` re-raise in the promise's `unhandled_exception` - sit behind `__cpp_exceptions`/`_CPPUNWIND` guards, solely to diagnose an exception escaping a user body) and builds with OR without exception support - the default CMake/vcxproj Debug+Release build now has them ON (matching what a `find_package` consumer gets), `-DMACRAME_NO_EXCEPTIONS=ON` (CMake) / the vcxproj Shipping config / `tsan/run.sh` / the pre-commit hook build them OFF, which is what keeps the source throw-free. The contract that makes both work: **an exception must not leave a user body** - every path that invokes one goes through `detail::invoke_user_body` (task_block.h; `Executable::run`, `Access_op::State::run`, the graph node body, both `parallel_for` loops) whose handlers (compiled only where the calling TU has exceptions) call the `escaped_exception_diagnose` seam (declared task_block.h, defined guarded.cpp next to `blocking_sync_diagnose`), and a coroutine body's escape reports the same way through the promise's `unhandled_exception`. The message names the running task (`task_name(current_task)`) + `what()` when the exception derives from `std::exception`, because a handler runs POST-unwind so the stacktrace starts at the seam, not the throw. Five death tests, `run_if(with_exceptions, ...)`. `MACRAME_NO_EXCEPTIONS` exports `_HAS_EXCEPTIONS=0` (+ the flag where expressible) as a PUBLIC usage requirement because the MSVC STL bakes it into declarations; `access.h` carries a `detect_mismatch` tripwire for it (verified: a mixed link fails LNK2038). Rationale in `docs/design.md` §4.6, user contract in `docs/guide.md` §10.5. Tests are the one place failures are non-fatal (the harness records and continues); fatal paths are checked via subprocess death tests. **`TS_ENSURE(expr, message)` (2026-07, `TS_SAFETY_CHECKS`-gated)** — UE-`ensure`-shaped recoverable assert macro (fatal.h): evaluates `expr` once in both configs and yields its bool; on failure bumps `ts::ensure_failure_count()` on EVERY occurrence but reports ONCE PER CALL SITE (captureless-lambda function-local static claim), through a swappable presentation handler (`ts::set_ensure_handler`, `std::set_terminate` shape; default = `ENSURE FAILED:` + stack + debugger-break if attached via `detail::is_debugger_present`/`debug_break`, the C++26 P2546 pair polyfilled Win/Linux/mac; `ts::fatal` breaks too, before `abort`). Counting is outside the handler. The harness fails on failures not consumed via `ts::test::consume_ensure_failures(n)`; `--bench`/`--stress` fail their exit code on any. Its first user was the blocking-sync diagnostic, which the coroutine-first transformation PROMOTED to a hard fatal: an in-task `sync()`/`take()` on unsettled work has no sanctioned form any more (there is no retraction to fall back on), so `blocking_sync_diagnose` (seam task.h→guarded.cpp) calls `ts::fatal` — a sharp same-object message when the target is an access on a pipe the current context holds, a general never-park message otherwise. `parallel_for`'s join is the one structural exemption (it waits on group state, never through `sync_wait`). `TS_ENSURE` remains the recoverable-assert facility for new diagnostics. -- **Samples** (`sample/`) — six SINGLE-FILE, self-contained .cpp samples (no headers — consumers forward-declare the entries; the blackboard convention, now the rule). `game_frame.cpp`: a ~30-system mock game-engine frame built in TWO variants from the SAME system bodies via `build_frame_graph(World&, Frame_variant, dot)` — `baseline` (straightforward composition) and `optimised` (levers applied in an `optimise()` section + branched node bodies), the worked optimisation exercise in [docs/example-frame-optimization.md](docs/example-frame-optimization.md). Exercises every layer — schedule derived from declared access, a frame head (camera/networking/scripting), gameplay trio, AI (speculative cancellable nav queries, trailing-`Cancellation_token` early-out), animation chain (graph→IK→skinning), split physics pipeline (broadphase→narrowphase→solver→finalize; the multi-conflict `physics→propagation` edge via velocities+bodies), a RENDER pipeline reading LAST frame's transforms via `Versioned` so it overlaps sim (the render-thread-with-one-frame-latency model), off-path leaves (vfx/replication/stats/gc/navmesh_rebuild), `ts::parallel_for` in heavy systems, streaming via a fire-and-forget coroutine that fires four `async` loads and folds them with sequential `co_await`s (the linear spelling of the old `then`/`when_all` chain), a per-frame HUD snapshot via multi-object `ts::async(fn, combat, economy)`, one bare generic-lambda node (`const auto&` probe). Priorities are IDENTICAL in both variants (kept for their visualization, NOT a lever — compilation ignores them). The optimised levers, all tool-guided from the baseline trace: (1) AI reads LAST frame's gameplay via a second `Versioned` snapshot (the gameplay trio's results staged + flipped at frame end) — deletes the `trio→AI` edges, the SINGLE BIGGEST cut (−821 µs), the marquee dependency-cut lever; (2) parallelise the fattest serial CRITICAL bars (`combat`/`ik_post`/`UI` — critical-path-bound, so splits move the makespan); (3) `Deferred` draw staging (producers go wide — utilization, NOT makespan); (4) cloth on prev-frame transforms (off the post-flip tail; neutral on the long spine, fills cores once the spine is short). Baseline ≈85% util / ≈12% dead / ≈6.5 ms → optimised ≈95% util / ≈10% dead / ≈5.6 ms on 8 workers (`variant_workers`; util/dead are the portable numbers, frame time is machine-dependent) — the picture flips from dependency-bound to CORE-bound (cores full, chain waits for a core, not a dependency; the single critical spine fragments), the "optimisation is done" signal. The older 4793→3120 µs / 60→85% util figures were a profiler artifact, not a bigger workload: `parallel_for` fan-out leaked to the global default pool, so the "6-worker" trace secretly ran on ~12 cores — frame times ~2× too fast, util undercounted the leaked work. The serial workload has been a constant ~36.6 ms since the 30-system frame landed (2026-07-24); the fixes that made the trace honest were the `parallel_for`-on-current-scheduler routing, owner attribution, time-bucketed utilization, and the body/machinery split. `transform0==5` determinism held both variants. **A THIRD composition (2026-08): `Frame_variant::graph_free` / `run_frame_graph_free(World&)`** — the baseline frame with no `Static_task_graph` at all, the measurement closing `docs/internals/coroutine-first.md` §10.5: every system is a multi-object `ts::async`, every derived edge an explicit `co_await` inside one of the frame's chain coroutines, completion joined by hand. Three findings (`docs/guide.md` §6.4, `docs/design.md` §2.4): (1) safety is the PIPE's, not the graph's — unchanged; (2) **pipe FIFO does NOT stand in for conflict edges** (the multi-object cascade enters links one at a time in canonical order, so a system blocked on its first object hasn't taken its slot on the later ones and a later launch overtakes it) — the naive declaration-order-no-awaits version mis-orders `frustum_cull` before `camera` and `submit` before `cmd_record`, losing a frame of draw commands SILENTLY (declarations all correct, harness quiet) and running 7.6% faster for it; (3) **the graph's perf edge is resume locality, not allocation amortization** — graph-free costs ~95 more allocs/frame (<2 µs) but +64 µs/frame on a 4.1 ms frame and +131 µs on a 0.45 ms one (+1.6% / +28.7%), which is ~50 coroutine suspend/resume round trips at ~1.8 µs. Exports `game_frame_free_stats(...)` (same shape as `game_frame_stats`) + `game_frame_draw_count()` (the submitted-command count — the observable that DISCRIMINATES producer/submit ordering; `transform0` cannot, since every mock system writes the same constant every frame so a one-frame skew is invisible). Wired into `--bench` (`graph 1.0`/`free 1.0`/`graph .05`/`free .05` µs/frame), `--memprofile` (`frame graph`/`frame graph-free` allocs/frame, measured as an n-frame minus 1-frame difference so setup cancels), `--stress`, tsan_main, and the `engine frame without a graph` integration test. `--trace` traces both graph variants on a `variant_workers`(=8)-worker scheduler → `sample_game_frame_avg_baseline.svg` + `sample_game_frame_avg_optimised.svg` + the DOT. Exports `game_frame_stats(frames, scale, avg&, serial&, xf0&)` (baseline, for the determinism/utilization test) + `stress_game_frame_optimised(frames, workers)` (headless optimised run — TSan covers the gameplay `Versioned` publisher + `Deferred` staging) + `run_game_frame_sample` + `dump_game_frame_dot` + `trace_game_frame`. `physics.cpp`: the machine/extract decomposition (sealed `Guarded`, single grant holder = the sim's write; `Deferred` inputs; `Versioned` outputs; snapshot-based scene query; id-reservation forward refs; batch-extract; deterministic run-twice check); exports `run_physics_sample` + `physics_pose_hash` (tsan). `blackboard.cpp`: the blackboard recipe (doc §7.4). `scope_access.cpp`: the scope-based access showcase — a tiny bank exercising the `co_await ts::read_write`/`read_only` held-grant guards in three shapes: single-object read-modify-write under a held grant; several accounts one-by-one (each its own scope, released before the next `co_await`); and a two-account transfer taken together (the multi-object hold). Runs twice, checking transfers conserve the total and both runs agree. Exports `run_scope_access_sample`. `events.cpp`: the delegate/event recipe (docs/pattern-farming.md 2.59) — how UE-style delegates decompose onto the primitives, four tiers in one toy: intra-system (a plain synchronous `move_only_function` delegate fired under the node's grant — sanctioned, zero machinery; the one rule: outbound effects via `async`/`stage` only), command (`audio.async` from the handler), lightweight notifications (`ts::Event_bus` — promoted 2026-08 from this sample's prototype to a public header, see its own bullet; the sample demonstrates the three subscribe forms, pinned auto-disconnect mid-run via the retiring commander, the explicit `dispatch.after(producers)` intent edge, and two minimal with/without-graph setup floors checked every run), and heavyweight flows (a `Guarded` batch + conflict-derived edge — O(events) with a 1000-NPC population). Determinism self-check (run-twice compare). Exports `run_events_sample`. `coloring.cpp`: interaction coloring (pattern-farming 2.39, author-flagged important) — the cross-entity case deferral cannot serve: a Gauss-Seidel cloth solver (Verlet + projection) greedy-colors its constraint graph (4 colors on a grid, disjointness validated), runs each color band as one `parallel_for` under the node's grant with bands sequential as barriers; two disconnected patches demonstrate islands need no special handling; bit-deterministic under any chunking/stealing (≤1 writer per particle per band). Exports `run_coloring_sample`. +- **Samples** (`sample/`) — six SINGLE-FILE, self-contained .cpp samples (no headers — consumers forward-declare the entries; the blackboard convention, now the rule). `game_frame.cpp`: a ~30-system mock game-engine frame built in TWO variants from the SAME system bodies via `build_frame_graph(World&, Frame_variant, dot)` — `baseline` (straightforward composition) and `optimised` (levers applied in an `optimise()` section + branched node bodies), the worked optimisation exercise in [docs/example-frame-optimization.md](docs/example-frame-optimization.md). Exercises every layer — schedule derived from declared access, a frame head (camera/networking/scripting), gameplay trio, AI (speculative cancellable nav queries, trailing-`Cancellation_token` early-out), animation chain (graph→IK→skinning), split physics pipeline (broadphase→narrowphase→solver→finalize; the multi-conflict `physics→propagation` edge via velocities+bodies), a RENDER pipeline reading LAST frame's transforms via `Versioned` so it overlaps sim (the render-thread-with-one-frame-latency model), off-path leaves (vfx/replication/stats/gc/navmesh_rebuild), `ts::parallel_for` in heavy systems, streaming via a fire-and-forget coroutine that fires four `async` loads and folds them with sequential `co_await`s (the linear spelling of the old `then`/`when_all` chain), a per-frame HUD snapshot via multi-object `ts::async(fn, combat, economy)`, one bare generic-lambda node (`const auto&` probe). Priorities are IDENTICAL in both variants (kept for their visualization, NOT a lever — compilation ignores them). The optimised levers, all tool-guided from the baseline trace: (1) AI reads LAST frame's gameplay via a second `Versioned` snapshot (the gameplay trio's results staged + flipped at frame end) — deletes the `trio→AI` edges, the SINGLE BIGGEST cut (−821 µs), the marquee dependency-cut lever; (2) parallelise the fattest serial CRITICAL bars (`combat`/`ik_post`/`UI` — critical-path-bound, so splits move the makespan); (3) `Deferred` draw staging (producers go wide — utilization, NOT makespan); (4) cloth on prev-frame transforms (off the post-flip tail; neutral on the long spine, fills cores once the spine is short). Baseline ≈85% util / ≈12% dead / ≈6.5 ms → optimised ≈95% util / ≈10% dead / ≈5.6 ms on 8 workers (`variant_workers`; util/dead are the portable numbers, frame time is machine-dependent) — the picture flips from dependency-bound to CORE-bound (cores full, chain waits for a core, not a dependency; the single critical spine fragments), the "optimisation is done" signal. The older 4793→3120 µs / 60→85% util figures were a profiler artifact, not a bigger workload: `parallel_for` fan-out leaked to the global default pool, so the "6-worker" trace secretly ran on ~12 cores — frame times ~2× too fast, util undercounted the leaked work. The serial workload has been a constant ~36.6 ms since the 30-system frame landed (2026-07-24); the fixes that made the trace honest were the `parallel_for`-on-current-scheduler routing, owner attribution, time-bucketed utilization, and the body/machinery split. `transform0==5` determinism held both variants. **A THIRD composition (2026-08): `Frame_variant::graph_free` / `run_frame_graph_free(World&)`** — the baseline frame with no `Static_task_graph` at all, the measurement closing `docs/internals/coroutine-first.md` §10.5: every system is a multi-object `ts::async`, every derived edge an explicit `co_await` inside one of the frame's chain coroutines, completion joined by hand. Three findings (`docs/guide.md` §6.4, `docs/design.md` §2.4): (1) safety is the PIPE's, not the graph's — unchanged; (2) **pipe FIFO does NOT stand in for conflict edges** (the multi-object cascade enters links one at a time in canonical order, so a system blocked on its first object hasn't taken its slot on the later ones and a later launch overtakes it) — the naive declaration-order-no-awaits version mis-orders `frustum_cull` before `camera` and `submit` before `cmd_record`, losing a frame of draw commands SILENTLY (declarations all correct, harness quiet) and running 7.6% faster for it; (3) **the graph's perf edge is resume locality, not allocation amortization** — graph-free costs ~95 more allocs/frame (<2 µs) but +64 µs/frame on a 4.1 ms frame and +131 µs on a 0.45 ms one (+1.6% / +28.7%), which is ~50 coroutine suspend/resume round trips at ~1.8 µs. Exports `game_frame_free_stats(...)` (same shape as `game_frame_stats`) + `game_frame_draw_count()` (the submitted-command count — the observable that DISCRIMINATES producer/submit ordering; `transform0` cannot, since every mock system writes the same constant every frame so a one-frame skew is invisible). Wired into `--bench` (`graph 1.0`/`free 1.0`/`graph .05`/`free .05` µs/frame), `--memprofile` (`frame graph`/`frame graph-free` allocs/frame, measured as an n-frame minus 1-frame difference so setup cancels), `--stress`, tsan_main, and the `engine frame without a graph` integration test. `--trace` traces both graph variants on a `variant_workers`(=8)-worker scheduler → `sample_game_frame_avg_baseline.svg` + `sample_game_frame_avg_optimised.svg` + the DOT. Exports `game_frame_stats(frames, scale, avg&, serial&, xf0&)` (baseline, for the determinism/utilization test) + `stress_game_frame_optimised(frames, workers)` (headless optimised run — TSan covers the gameplay `Versioned` publisher + `Deferred` staging) + `run_game_frame_sample` + `dump_game_frame_dot` + `trace_game_frame`. `physics.cpp`: the machine/extract decomposition (sealed `Guarded`, single grant holder = the sim's write; `Deferred` inputs; `Versioned` outputs; snapshot-based scene query; id-reservation forward refs; batch-extract; deterministic run-twice check); exports `run_physics_sample` + `physics_pose_hash` (tsan). `blackboard.cpp`: the blackboard recipe (doc §7.4). `scope_access.cpp`: the scope-based access showcase — a tiny bank exercising the `co_await ts::read_write`/`read_only` held-grant guards in three shapes: single-object read-modify-write under a held grant; several accounts one-by-one (each its own scope, released before the next `co_await`); and a two-account transfer taken together (the multi-object hold). Runs twice, checking transfers conserve the total and both runs agree. Exports `run_scope_access_sample`. `events.cpp`: the delegate/event recipe (docs/pattern-farming.md 2.59) — how UE-style delegates decompose onto the primitives, four tiers in one toy: intra-system (a plain synchronous `move_only_function` delegate fired under the node's grant — sanctioned, zero machinery; the one rule: outbound effects via `async`/`stage` only), command (`audio.async` from the handler), lightweight notifications (`ts::Event_bus` — promoted 2026-08 from this sample's prototype to a public header, see its own bullet; the sample demonstrates the three subscribe forms, pinned auto-disconnect mid-run via the retiring commander, the explicit `dispatch.after(producers)` intent edge, and two minimal with/without-graph setup floors checked every run), and heavyweight flows (a `Guarded` batch + conflict-derived edge — O(events) with a 1000-NPC population). Determinism self-check (run-twice compare). Exports `run_events_sample`. `coloring.cpp`: interaction coloring (pattern-farming 2.39, author-flagged important) — the cross-entity case deferral cannot serve: a Gauss-Seidel cloth solver (Verlet + projection) greedy-colors its constraint graph (4 colors on a grid, disjointness validated), runs each color band as one `parallel_for` under the node's grant with bands sequential as barriers; two disconnected patches demonstrate islands need no special handling; bit-deterministic under any chunking/stealing (≤1 writer per particle per band). Exports `run_coloring_sample`. `fixed_rate.cpp` (2026-09): a 60 Hz physics graph on its own `ts::Periodic` clock beside the frame graph, boundary = `Deferred` intents + `Versioned` poses read lent by render; gameplay's long body yields; structural self-checks every run (ticks in order, consecutive interpolated pair, fraction in [0,1], every staged intent applied once) since which tick picks up an intent is wall-clock dependent; `fixed_rate_physics_hash` is the determinism check (physics graph alone, tick-indexed intent script, same hash across worker counts). Exports `run_fixed_rate_sample` (`--fixed-rate [n]`), `fixed_rate_self_check`, `fixed_rate_physics_hash`, `stress_fixed_rate`. `game_frame.cpp` gained **`Frame_variant::fixed_rate`** (2026-09): the optimised frame with the physics chain on a 60 Hz graph and networking on a 30 Hz graph (`Fixed_rate_ticks` builds both, primes them, drives each from a `Periodic`, runs a final tick of each at teardown), frame nodes reading `body_snapshot`/`net_snapshot` and staging `physics_intents`/`net_outbox_stream`; propagation interpolates the two newest physics ticks. Exports `game_frame_fixed_stats`; third `--trace` SVG (`_fixed_rate`); `fixed 1.0`/`fixed .05` `--bench` rows; integration test `engine frame with fixed-rate physics and networking` (transform0 == 5, same draw count as the baseline). - **`task.h`** — the task layer's PUBLIC surface (split 2026-08-17, the guarded.h playbook): `Task`, the option aggregates (`Dispatch_options`, `Access_options`), the bare-scheduler task primitive `ts::launch` (dispatches through the `submit_ready` bridge, so no direct scheduler dependency; detached — inherits no grant), `Signal`, `External_wait` + `set_deadlock_net_window`, and the `core_of`/`task_from_core`/`Optional_awaitable` detail bridges. ~330 lines; includes the two split-out headers so every existing includer keeps working. **`ts/cancellation.h`** (public) — the cancellation cluster: `Cancellation_source`/`Cancellation_token`/`Cancel_callback` + `detail::Cancel_state`. **`ts/detail/task_block.h`** — the block layer: the fully-monomorphic `detail::Task_control_block`, `Task_ptr`/`Adopt_ref`/`Destroy_queue`, `Block_backed` + `Executable`/`make_executable` (wrappers derive from the block; recovery is a derived cast), `make_bare_block` + the settled/cancelled sentinel cores, the naming helpers, `current_task`/`current_scope_children` TLS, the body traits (`Task_body`, `Task_result`), `detail::add_nested`, and the scheduler-free seams (`submit_ready`, `pipe_enter_first`, `drain_serial_pending`, `blocking_sync_diagnose`, the deadlock-net internals — declared there, defined in the scheduler/pipe layers). All header-only — no `task.cpp`; `guarded.h` includes `task.h`. - **`Frame_gate`** (`frame_gate.h`, 2026-08, TODO 6.7) — the cross-frame realignment utility: `co_await gate.next()` parks a task until the frame loop's next `open()`, so work that resumed from an external completion at an arbitrary moment re-enters at a frame boundary instead of mid-update. A type rather than a documented `Signal::reset` idiom because the hand-rolled version has a missed-wakeup window (a task reading the signal just before a boundary can attach to a gate about to be re-armed) and `reset()`'s "every waiter already released" precondition (fatal otherwise); the gate hands out the CURRENT frame's signal under a mutex and installs a fresh one at `open()`, trading one bare block per frame. `open()` releases through the scheduler at `Priority::low` by default (`set_release_priority` overrides) — an inline trigger would run every parked frame on the frame loop's own thread before `open()` returned. The same hop is the sanctioned idiom for an OS completion callback triggering a `Signal` (TODO 6.6 stays a design note: the bridge is one line, the packaging question is who owns the WAITING, and that needs the platform layer). -- **Headers** — `ts/ts.h` is the umbrella (`version`, `scheduler`, `guarded`, `task`, `static_task_graph`, `parallel_for`, `recorder`, `deferred`, `versioned`, `event_bus`, `access`, `rules`, `frame_gate`, `coroutine_support`; `named.h`/`priority.h` come in transitively). `frame_gate.h` joined it 2026-08-22 (the API-stability pass, S13 — the exclusion had no technical basis: it includes only `ts/task.h` and std headers). `version.h` carries `ts::version_major/minor/patch` + `version_string()` beside the `TS_VERSION_*` macros. `ts/task_scope.h` was **removed (2026-08)** with `ts::Task_scope`. +- **Fixed-rate graphs (2026-09)** — a subsystem on its own clock is a second compiled graph beside the frame graph on the one scheduler, bounded by a `Deferred` for inputs (sampled at the tick's commit) and a `Versioned` for outputs (published at the tick's end): the logical-execution-time model (guide §6.6, design §6.1). No bundled rate-domain type, by decision - ownership split, boundary contents and overload policy are engine choices; the library ships four mechanisms. (1) **`ts/timer.h`**: `ts::sleep`/`sleep_until` return a bare-block `Task` settled by one lazily created timer thread (min-heap; delivery is a `ts::launch` at `Sleep_options::priority`, never inline; cancellation through a `Cancel_callback` settles cancelled promptly; each armed sleep holds an `External_wait`; Windows waits on a `CREATE_WAITABLE_TIMER_HIGH_RESOLUTION` timer plus an auto-reset wake event; worker-less mode is fatal; `destroy_scheduler` stops the thread first and fatals on a live sleep under `TS_SAFETY_CHECKS`; program exit stops it through a static in guarded.cpp declared after `g_scheduler`). `ts::Periodic`: grid deadlines, `next()` returns grid points passed, 0 once its token trips. (2) **`Versioned`** (`History` is a template parameter; opaque-declared in journal.h so the recorder friends can name the two-parameter template): a third replica rotated in `swap_replicas` under the write grant, `Version_stamps` (publish instants, serial, `fraction_at`), copy resync by default, `Resync::replay` a construction fatal. `read_last_versions()` returns a `Version_view` (non-movable guard installing its own context, tuple protocol for `auto [previous, current, stamps]`; lent when the context grants the front, a blocking read turn on a blue thread, fatal under `Rule::in_task_sync` in a grant-less task, a live guard for `await_under_guard` in a coroutine); the free `co_await ts::read_last_versions(v)` uses `detail::Version_awaiter`, which derives from `Access_awaiter` (its resume bookkeeping is now `finish_acquire()`). (3) **`ts::yield()`** (task.h): reads `detail::high_queued` (relaxed; `Scheduler::submit` increments before a `high` push, `find_work` decrements after a pop); the slow path `detail::yield_to_high(Priority)`, a `Scheduler` friend, pops one `high` entry and runs it inline under a `Nested_dispatch_scope` that clears Current_task, access, scope children, relaxed rules and trace owner, and books the nested span to `Nested_span_state`, which `Trace_busy_scope` subtracts; no-op off a worker, in worker-less mode, and at `high`. `parallel_for` (flat and colored loops) yields at every chunk claim at `Parallel_base::priority`. (4) **`Static_task_graph::set_default_priority`** (`Node::priority_set`; `add_node` seeds from the default; the move ctor lists `default_priority_`). Tests: `timer_tests` group; versioned, scheduler (`yield:`), parallel, graph (`default priority`) and integration additions; each fatal has a death scenario. Negative checks done: yield no-op, rotation without the second swap, sleep without `External_wait`. Measured 2026-09, 8 workers, 100 traced frames: optimised 16.93 ms vs fixed_rate 17.12 ms - neutral, because the optimised frame is already core-bound; the gain on this fixture is structural (one frame shape, tick count owned by the clock, overload policy local to the driver), not throughput. +- **Headers** — `ts/ts.h` is the umbrella (`version`, `scheduler`, `guarded`, `task`, `static_task_graph`, `parallel_for`, `recorder`, `deferred`, `versioned`, `event_bus`, `access`, `rules`, `frame_gate`, `timer`, `coroutine_support`; `named.h`/`priority.h` come in transitively). `frame_gate.h` joined it 2026-08-22 (the API-stability pass, S13 — the exclusion had no technical basis: it includes only `ts/task.h` and std headers). `version.h` carries `ts::version_major/minor/patch` + `version_string()` beside the `TS_VERSION_*` macros. `ts/task_scope.h` was **removed (2026-08)** with `ts::Task_scope`. Docs map. Roadmap and open design questions live in `docs/TODO.md`. Two designs of record must be read before touching their areas: **`docs/internals/coroutine-first.md`** (the end-state model, the waiting rules (suspension under access), what was deleted and what replaces it, the fatal+companion test matrix, and §11's remaining action list — read before touching `task.h`/`coroutine_support.h`) and **`docs/internals/pipe-rebase.md`** §0 (the evolved pipe, why the lock-free chain was retired, the unified cascade and commit ladder — read before touching `guarded.cpp`/the pipe; its test plan is `docs/internals/pipe-rebase-tests.md`). Then: the dynamic-task internals (lifecycle, lock-counter, the access invariant, allocation, the graph's §10 scenarios) in `docs/internals/task-internals.md`; the deferred-write layer's state handoff (contracts, load-bearing mechanisms, ranked future plan — read before touching `journal.h`/`deferred.h`/`versioned.h`) in `docs/internals/deferred-versioned-state.md`; the waiting-rule check policy (the `ts::Rule` bitmask, `Relaxed_scope`, `ts::Rank`, the deadlock report's three tiers — read before touching `rules.h`/the rule checks) in `docs/internals/waiting-rule-policy.md`; the engine-comparison research in `docs/task-systems-comparison.md`; the production lock-contention research + the paired mutex/library benches behind it in `docs/internals/lock-contention-research.md`; the planned caller-owned-operation-state redesign of `access` (zero-alloc attended path, TODO 1.19 — read before touching `guarded.h`'s access fast path) in `docs/internals/access-op-design.md`; the original interface sketch in `design/guarded_sketch.h`. diff --git a/docs/TODO.md b/docs/TODO.md index 5b3b59c..b5e2955 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -379,7 +379,7 @@ IDs — when an item is done, mark it, don't renumber. per-frame rebuild a measured option instead of an anti-pattern and answers the first sophisticated-evaluator question ([research-deepdive.md](internals/research-deepdive.md) §9.4, §12.2). Ties to 10.1. - 11. `[ ]` **(P2, author 2026-07) Yield points inside long-running nodes.** The 4-worker + 11. `[x]` **(P2, author 2026-07 — DONE 2026-09) Yield points inside long-running nodes.** The 4-worker game_frame trace made the failure concrete: a ready critical-path node (economy) waited ~0.9 ms behind a long off-path runner (audio) — nothing evicts a runner, and priority cannot help work that is already running (measured: audio at `low` changed neither its @@ -389,6 +389,12 @@ IDs — when an item is done, mark it, don't renumber. `ts::yield()` polling a "critical work pending" signal vs coroutine nodes (`co_await` suspension already exists for tasks) vs auto-slicing via `parallel_for` guidance. Relates to 2.4's keep-out-zone hypothesis and 2.5's rank (what "higher-rank pending" means). + **Landed (2026-09):** `ts::yield()` (task.h) runs one queued `Priority::high` entry inline + on the yielding worker's stack and returns - no suspension, so functor nodes and + `parallel_for` bodies can yield, and the continuation keeps its core; `parallel_for` + yields at every chunk claim. Rationale: design.md §3 "Yield points". Open: a pending + signal finer than the `high` class (2.5's rank), and a yield inside a resumed coroutine + segment defers any resume the nested task triggers until the segment returns (guide §13). 12. `[ ]` **(P2, author 2026-07 — raised from the 2.3 adjacency) Frame-boundary overlap for designated nodes.** Distinct from 2.3 (whole-graph pipelining): let specific off-path tails (audio mix, streaming finalization) spill past the run's settle into the next diff --git a/docs/design.md b/docs/design.md index f4b3ec2..d00b0e9 100644 --- a/docs/design.md +++ b/docs/design.md @@ -493,6 +493,54 @@ relaxations would be explicit staleness opt-ins, and the real reader-throughput answer is structural (`Versioned`, §6) rather than queue policy. +#### Yield points + +Priority orders queues but cannot evict a running task, so a ready `high` task +can wait behind long `normal` bodies for their whole duration. The game-frame +dry runs measured exactly that +([profiler-guided-optimization.md](internals/profiler-guided-optimization.md), +experiment 1), and a fixed-rate graph with a deadline (§6.1) turns it from a +tuning question into a requirement. The cooperative remedy is a yield point, +and the design question was what a yield point does. + +The obvious shape suspends the yielding task, requeues it, and lets the worker +take the urgent one. That pays a requeue and a resume hop per yield, lets a +thief move the continuation to a cold core, and needs a coroutine. The shape +taken runs the urgent task inline instead: `ts::yield()` pops one queued +`high` entry and executes it on the yielding worker's stack, then returns. The +continuation never leaves the stack, and the yield works in a plain functor +body. The common case, nothing queued, reads one relaxed counter the scheduler +keeps beside the `high` queue, incremented before a push and decremented after +a pop, so it never under-counts. + +Three properties make the nesting safe. A queued entry has already taken its +pipe turns, so it cannot wait on grants the yielder holds. The nested dispatch +runs inside a scope that clears the thread's ambient task state (current task, +grants, scope children, rule relaxation, trace owner) and restores it +afterwards, so the nested task starts as it would at the top of the worker +loop. And a `high` task's yield point does nothing, so nesting is one level +deep and ordering among `high` tasks stays the queue's. The trace subtracts +the nested span from the yielding body's credited time, so body time is +counted once. `parallel_for` has a yield point at every chunk claim, which +covers the most common long body without a change to user code. + +#### The timer thread + +Deadlines are kept by one timer thread with a min-heap, created on first use +([timer-primitive-design.md](internals/timer-primitive-design.md)). The +alternative considered was folding the heap into the scheduler, as Go and +Tokio do: workers check it before the queues and park with a timeout to the +nearest deadline. That saves a thread and one wake hop per fire, but the +eventcount park has no timed wait in standard C++, the `spin` and `handoff` +idle policies would each need their own polling, and deadline ownership would +move between parked workers. On Windows it also costs timing quality: a parked +worker's timeout has the system timer's granularity, while a dedicated thread +can wait on a high-resolution waitable timer. For a 60 Hz tick the saved hop is +tens of microseconds against a 16.7 ms period, well below the jitter that +granularity would add, so the dedicated thread stays. It never runs user code: +a fire is delivered as a task at the sleep's priority, for the same reason +`Frame_gate::open()` releases through the scheduler. + --- ## 4. The task core @@ -1093,6 +1141,21 @@ and `publish()` flips atomically. The load-bearing choices: provides per-worker slots for staging from inside a `parallel_for`, is the explicit, localized surrender of cross-thread reproducibility. +`History::current_and_previous` adds a third replica, the version before the +current one, so a consumer can interpolate between the two newest versions of +a producer that publishes on its own clock. The swap becomes a rotation under +the same write grant, with front, shadow and previous exchanging contents, and +the publish stamps its instant and serial. Two choices follow. The resync is by +copy: after the rotation the shadow holds the version before last, and +replaying one batch would bring it only as far as the previous version. And the +pair is read through a view that is itself the read grant, the `Access_guard` +shape, so both references are valid for exactly the bindings' lifetime. The +history is a template parameter, so the third replica and the pair read exist +only where asked for. An earlier sketch took the reading instant as an +argument, which read as a lookup by time although the versions returned never +depend on it; the stamps and a `fraction_at` helper keep the choice of instant +with the caller. + The UE research grounded several choices. `ENQUEUE_RENDER_COMMAND`'s linear-allocated coarse queue and its splice-in-submit-order parallel recording, where determinism comes from splice position rather than thread @@ -1102,6 +1165,31 @@ tier. The Render Dependency Graph, with passes declaring resource access and order derived, is production validation of this library's central premise, applied to GPU resources. +### 6.1 Fixed-rate graphs + +A fixed-rate subsystem is a second compiled graph with its own clock, beside +the frame graph on the same scheduler, and the boundary between them is one +`Deferred` for inputs and one `Versioned` for outputs. That is the logical +execution time model of real-time control (Giotto, AUTOSAR's timing +extensions): a task reads its inputs at release and publishes its outputs at +its deadline, whatever its actual duration. The time-triggered architecture's +state message, a version replaced at a known instant and read without +blocking, is what `Versioned` already is. The game-loop accumulator is the +special case where the schedule is inferred from the measured frame time, +which is also why it can spiral: a slow tick raises the next frame's tick +count, which slows the tick further. With a clock of its own, the overload +policy, whether to run missed ticks, drop them or slow simulated time, is one +decision in the driver, separate from rendering. + +The library supplies mechanisms only: the timer, the history on `Versioned`, +the graph default priority, and yield points. There is no fixed-rate-graph +type. Which state a subsystem owns, what crosses the boundary, the overload +policy, and how the tick relates to networking are engine decisions a bundled +type would have to guess, and the composition is a few lines +(`sample/fixed_rate.cpp`). The substantial cost of the model sits in the +engine, not the library: a subsystem whose gameplay code writes the same state +from both rates has to be split into owned state and published versions first. + --- ## 7. Rejected alternatives diff --git a/docs/example-frame-optimization.md b/docs/example-frame-optimization.md index ca5feef..c636cdb 100644 --- a/docs/example-frame-optimization.md +++ b/docs/example-frame-optimization.md @@ -33,15 +33,16 @@ two variants of the same ~30-system frame, built from the *same* system bodies: - **optimised** — the same frame after reading its own trace, with the levers the visualization makes obvious applied in an `optimise()` section. -Generate both traces (on an 8-worker scheduler) plus the structure dump with: +Generate the traces (on an 8-worker scheduler) plus the structure dump with: ``` macrame_playground --trace 200 -show_graph.bat # renders the DOT and opens both average-run SVGs +show_graph.bat # renders the DOT and opens the average-run SVGs ``` producing `sample_game_frame_avg_baseline.svg`, -`sample_game_frame_avg_optimised.svg`, and `sample_game_frame.dot`. +`sample_game_frame_avg_optimised.svg`, `sample_game_frame_avg_fixed_rate.svg` +(the third variant, §5.1), and `sample_game_frame.dot`. The point of the exercise is not the exact percentage it ends up saving. It is *which* optimisations the trace says are worth trying, and which ones it says to skip. It ends by showing what a *finished* @@ -193,6 +194,35 @@ faster means **cutting work** (a smaller scene, cheaper systems) or **adding cores**, not restructuring the graph. The trace tells you when you have reached that point instead of guessing. +### 5.1 A third variant: physics and networking on their own clocks + +`Frame_variant::fixed_rate` is the optimised frame with the physics pipeline +(broadphase → narrowphase → solver → finalize) moved into a 60 Hz graph and +networking into a 30 Hz graph, each driven by a `ts::Periodic` clock beside the +frame loop ([guide.md §6.6](guide.md)). The frame stages physics inputs and +outgoing network data through `Deferred`, reads the ticks' published snapshots +through `Versioned`, and propagation interpolates between the two newest +physics ticks. `--trace` writes its average run as +`sample_game_frame_avg_fixed_rate.svg`; the tick graphs run on the same +workers, untraced. + +| | baseline | optimised | fixed rate | +|---|---|---|---| +| frame time | 19.01 ms | 16.93 ms | 17.12 ms | +| core utilization | 86.2 % | 95.4 % | 95.6 % | +| critical path dead time | 9.8 % | 9.3 % | 10.6 % | + +*(2026-09, 8 workers, 100 traced frames, one run each.)* + +The physics chain leaves the frame's critical path, and the frame time does not +move. That is the core-bound reading from above applied to a new lever: the +optimised frame already keeps every core busy, and at one tick per frame the +tick's work still needs those cores, whichever graph it belongs to. On this +fixture the variant's value is structural: the frame graph has one shape +however many ticks fall into a frame, the tick rate no longer depends on the +frame rate, and the overload policy lives in one driver. A frame with idle cores +or a physics chain on its critical path would also gain time. + ## 6. What carries over to a real frame - **Read utilization and dead time together.** Low dead time + low utilization = diff --git a/docs/guide.md b/docs/guide.md index efebddd..b1318d0 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -851,7 +851,10 @@ Node capabilities: form reads as intent: `submit.after(cmd, particles, ui)` makes `submit` depend on all three, the same as `.after(cmd).after(particles).after(ui)` but without reading like a sequence among them. -- `node.set_priority(p)` sets the node's queue priority. +- `node.set_priority(p)` sets the node's queue priority, and + `graph.set_default_priority(p)` sets it for every node that has not set its + own, including nodes added later. The latter is the spelling for a graph + whose whole run is urgent, such as a fixed-rate graph with a deadline (§6.6). - `node.set_inline()` runs the node on the thread that readied it when its objects are immediately available, which gives low-latency chaining for small nodes. @@ -1206,6 +1209,99 @@ This also reserves room. Because direction carries no meaning, a future critical path. Programs that wrote their intent down keep working; programs that leaned on declaration order would silently change behaviour. +### 6.6 Fixed-rate graphs + +Some subsystems want a clock of their own rather than the frame's. Physics +has a solver tuned to a fixed step, and its results should not depend on +frame timing; networking sends and consumes snapshots at a fixed tick. The +usual game loop runs them inside the frame with an accumulator, zero, one or +several steps per frame. That couples the two rates: the frame graph has to +express a variable number of ticks, the frame that owes catch-up ticks +stalls on them, and a slow step can force more catch-up, which slows the step +further. + +The alternative is a second compiled graph with its own clock, beside the +frame graph on the same scheduler. The tick graph owns its state and talks to +the frame through two objects, a `Deferred` for inputs and a `Versioned` for +outputs: + +```cpp +ts::Guarded world{ ts::Named{"world"} }; +ts::Deferred intents{ world }; // frame -> tick +ts::Versioned poses{ ts::Named{"poses"} }; // tick -> frame + +ts::Static_task_graph physics; +auto step = physics.add_node("step", [&intents, out = poses.recorder()](Physics_world& w) mutable +{ + (void)intents.commit(); // Inline: this node holds the write grant. + w.step(1.0f / 60.0f); + out.stage([p = w.positions()](Poses& s) { s.assign(p); }); +}, world); +physics.add_node("publish", ts::publish_fn(poses), poses.state()).after(step); +physics.set_default_priority(ts::Priority::high); +physics.compile(); +``` + +A coroutine drives it from a `ts::Periodic` clock (§10.6) and owns the +overload policy, which here runs at most two missed ticks per wake and lets +simulated time slow down beyond that: + +```cpp +ts::Task run_ticks(ts::Static_task_graph& graph, ts::Periodic& clock) +{ + for (;;) + { + int due = co_await clock.next(); // Grid points since the last tick; 0 once stopped. + if (due == 0) + co_return; + for (int i = 0; i < std::min(due, 2); ++i) + co_await graph.execute(); + } +} +``` + +The frame graph stages into `intents` from any node and reads `poses`. A node +that declared a read on `poses.state()` reads the two newest versions lent +(§9.2) and interpolates by where the frame falls between them: + +```cpp +frame.add_node("render", [&poses](const Poses&, Frame_out& out) +{ + auto [previous, current, stamps] = poses.read_last_versions(); + out.draw(lerp(previous, current, stamps.fraction_at(std::chrono::steady_clock::now()))); +}, poses.state(), frame_out); +``` + +The boundary has four rules: + +- The tick owns its state. The world has one accessor, the tick's step node, + and no frame node declares it. +- Inputs are sampled at the tick's start. Whatever the frame staged before the + step's commit belongs to that tick, anything later to the next one: up to a + tick of latency from input to simulation. +- Outputs are published at the tick's end. The frame reads the last published + version, so what it shows trails the simulation by up to a tick, which + interpolation between the two newest versions smooths. +- Everything that crosses is a staged command or published data, never a call + from one graph's body into the other's state. + +This is the logical-execution-time model from real-time control: a task reads +its inputs at release and publishes its outputs at its deadline, whatever its +actual duration. The frame graph keeps one shape, the number of ticks per +frame is the clock's business, the tick's chain leaves the frame's critical +path, and the tick is deterministic given its sequence of input cuts. + +The tick has a deadline and the frame has long bodies. Give the tick graph +`high` priority (`set_default_priority`) and put yield points in long frame +work (§10.1), so a tick that comes due while every worker runs frame work +starts within one yield interval. Runs of different graphs may overlap; the +pipe serializes their conflicting accesses, and the two boundary objects are +the only state both graphs touch. + +`sample/fixed_rate.cpp` is the minimal version, checking itself as it runs. +The `game_frame` sample's `fixed_rate` variant moves its physics pipeline to a +60 Hz graph and its networking to a 30 Hz graph. + --- ## 7. `parallel_for` @@ -1251,6 +1347,10 @@ body, deadlock-free even when every worker is occupied. Chunks inherit the caller's access grants, so a `parallel_for` inside a graph node may touch the node's declared objects. +Chunk boundaries are yield points (§10.1). When a `high` task is queued, the +thread that finishes a chunk runs it before claiming the next one, so a long +`normal` loop does not hold off urgent work for its whole duration. + Cross-item mutation, where item *i* writes item *j*, is not synchronized by `parallel_for` itself; see the WIP note in §13 and the staging tools in §9, which cover the common cases today. For the case staging cannot serve, @@ -1662,6 +1762,32 @@ Key properties: run is the sanctioned pattern, and it is checked rather than just documented. +`Versioned` also keeps the version +published before the current one. `read_last_versions()` returns both, with +their publish instants, under one read grant: + +```cpp +ts::Versioned poses{ ts::Named{"poses"} }; + +auto [previous, current, stamps] = poses.read_last_versions(); +draw(lerp(previous, current, stamps.fraction_at(std::chrono::steady_clock::now()))); +``` + +`stamps.fraction_at(t)` is 0 at the previous publish and 1 at the current one, +clamped to that range. The returned view is the grant: it lives as long as the +bindings, both versions pass the harness while it does, and in a coroutine a +`co_await` while it is alive is fatal, as with any held guard. Where the +context already grants the front, such as a node that declared +`poses.state()`, the read is lent and takes no turn of its own. On a blue +thread it takes a read turn. Inside a task that holds nothing, use +`co_await ts::read_last_versions(poses)`; the blocking form is fatal there. + +The history costs a third replica, rotated at every publish, and it requires +`Resync::copy` (the default for this history) or `Resync::overwrite`: after the +rotation the shadow holds the version before last, which one replayed batch +cannot bring forward. Without the history, `read_last_versions` does not +exist. + ### 9.3 Choosing between them | your state | use | @@ -1734,6 +1860,28 @@ settled what it awaited. It inherits the priority of the task that created it, which matters only for what the body launches. A `parallel_for` inside a coroutine called from a `high` node dispatches its helpers at `high`. +Priority orders the queues, but it cannot evict work already running, so a +long `normal` body can hold off a `high` task that became ready while every +worker was busy. `ts::yield()` is the remedy: when a `high` task is queued it +runs that task on the current thread and returns, and the yielding body then +continues on the same stack. With nothing queued it costs one relaxed load, +so it can sit in an inner loop: + +```cpp +for (Chunk& chunk : chunks) +{ + process(chunk); + ts::yield(); // A tick that came due runs here; then the loop continues. +} +``` + +It never suspends, so it is legal in any body, including a functor node and a +`parallel_for` body, and grants held across it are safe: the task it runs was +queued with its own turns already taken, so it cannot wait on them. It is a +no-op off a worker, in worker-less mode, and in a task already running at +`high`, and it runs only queued `high` work. Chunk boundaries of +`parallel_for` are yield points already (§7). + ### 10.2 Scheduler configuration There is one process-wide scheduler, brought up explicitly with @@ -1883,6 +2031,43 @@ compiler flag where a flag can express it, as a usage requirement, and a link that mixes the two settings fails with a `_HAS_EXCEPTIONS` mismatch instead of corrupting quietly. +### 10.6 Timers + +`ts::sleep(duration)` and `ts::sleep_until(deadline)` (`ts/timer.h`, in the +umbrella header) return a task that settles at the deadline: + +```cpp +co_await ts::sleep(100ms); // In a coroutine. +ts::sleep_until(deadline, { .token = stop.token() }).sync(); // On a blue thread. +``` + +A token cancels the wait promptly, settling the task cancelled rather than at +the deadline. The priority option sets the priority of the task that delivers +the wakeup, which is where an awaiting coroutine resumes; unset, it is the +calling task's. + +`ts::Periodic` is a fixed-rate tick source. Its deadlines sit on a grid fixed +at construction, so late delivery never drifts the rate, and `next()` reports +how many grid points passed since the previous call: 1 in steady state, more +when the consumer fell behind, 0 once its token is requested. What to do with a +count above 1, whether to run the missed ticks, drop them, or slow simulated +time, is the consumer's policy (§6.6 shows one): + +```cpp +ts::Periodic tick{ 16'667us, { .token = stop.token(), .priority = ts::Priority::high } }; +int due = co_await tick.next(); +``` + +One timer thread keeps the deadlines. It is created on first use, stopped by +`destroy_scheduler`, and never runs user code: a wakeup is delivered as a task +on a worker. On Windows it waits on a high-resolution waitable timer, so a +deadline is not rounded up to the system timer tick. An armed sleep counts as +an external wait for the deadlock net (§5.0.3), so a program idle while it +waits is not reported as deadlocked. Two constraints: worker-less mode has no +worker to deliver on, so a sleep there is fatal, and every sleep must be +awaited or cancelled before `destroy_scheduler` (fatal under +`TS_SAFETY_CHECKS`). + --- ## 11. Patterns and rules of thumb @@ -1925,6 +2110,10 @@ result, completion, or cancellation. | ordering gate between phases | `Signal` | | reusing a sub-graph inside a frame | `co_await inner.execute()` (§6.3) | | realigning cross-frame work to a frame start | `Frame_gate` (§10.4) | +| a subsystem on its own fixed clock | a fixed-rate graph (§6.6) | +| interpolating between the two newest versions | `Versioned` (§9.2) | +| a long body that must not hold off urgent work | `ts::yield()` (§10.1) | +| waiting for a time, or a periodic tick | `ts::sleep`, `ts::Periodic` (§10.6) | --- @@ -1979,6 +2168,12 @@ Stated plainly; each is on the roadmap (`docs/TODO.md`): (checked, §6.3), so a sub-graph shared by two concurrently running parents needs one instance per caller. Queued and pipelined runs are on the roadmap. +- Timers need workers. In worker-less mode a sleep is fatal; a virtual clock + the program advances, for deterministic tests, is planned. +- Yield points run queued `high` work only, and only on workers. A task run + at a yield point that resumes a coroutine hands the resume to the thread's + resume trampoline, so when the yield point is itself inside a resumed + coroutine segment, that resume waits until the segment returns. --- diff --git a/docs/internals/command-buffer-design.md b/docs/internals/command-buffer-design.md index 0061272..fc6dbcd 100644 --- a/docs/internals/command-buffer-design.md +++ b/docs/internals/command-buffer-design.md @@ -651,8 +651,9 @@ version the output extract, not the machine.** - `Access::append` edge derivation (`add_flush_node`) as compile-time sugar over hand-wired `after`. - Reserved handles from `stage()` (the id-allocator pattern is the documented - answer), multi-target buffers, `read_pair()` on `Versioned` for - interpolation. + answer), multi-target buffers. (`read_pair()` on `Versioned` for + interpolation landed 2026-09 as `History::current_and_previous` + + `read_last_versions`.) - ~~Single-publisher discipline on `Versioned` is documented, not enforced~~ **Enforced** (follow-up to the initial ship): a graph/inline publish that catches a dynamic publish still unresolved is fatal at flip entry under diff --git a/docs/internals/deferred-versioned-state.md b/docs/internals/deferred-versioned-state.md index ee0a427..05c8abb 100644 --- a/docs/internals/deferred-versioned-state.md +++ b/docs/internals/deferred-versioned-state.md @@ -251,8 +251,9 @@ bytewise hash/copy for trivially-copyable `T`; `Versioned` ctor arg forwarding **Parked, awaiting a forcing use case:** `Access::append` derivation / `add_flush_node`; cross-target commands (physics says: decompose instead); reserved handles from `stage()` (id-allocator pattern is the documented -answer); `read_pair()` for interpolation; triple-buffer `Versioned` (the name -already permits it silently); parallel apply inside a commit (API-invisible; +answer); triple-buffer `Versioned` (the name already permits it silently; the +interpolation case landed 2026-09 as `History::current_and_previous` + +`read_last_versions`, which is exactly a third replica); parallel apply inside a commit (API-invisible; only if a flush becomes the critical path). ## 7. Selection guidance (when to use what — condensed from the fixtures) diff --git a/docs/internals/timer-primitive-design.md b/docs/internals/timer-primitive-design.md index b2b0bbd..4c5d9d7 100644 --- a/docs/internals/timer-primitive-design.md +++ b/docs/internals/timer-primitive-design.md @@ -1,5 +1,15 @@ # Timer / delayed-dispatch primitive: design study +**Status (2026-09): option (a) landed** as `ts/timer.h` / `src/timer.cpp`: `ts::sleep`, +`ts::sleep_until`, and `ts::Periodic` (§3.4's `every`, whose `next()` returns the number of +grid points passed since the previous tick). Deviations from this study: on Windows the timer +thread waits on a high-resolution waitable timer, because a condition-variable timeout wakes on +the ~15.6 ms system tick, a whole period of a 60 Hz clock; delivery is a `ts::launch` at the +sleep's own priority rather than always `low`; worker-less mode is fatal for now rather than +virtual-clock driven (§4 and §5 remain the plan); `launch_after` and `Deadline` (§3.2, §3.3) are +not built. The alternative of folding deadlines into the workers' park (§2.3) was reconsidered +and rejected again; the reasoning is in design.md §3, "The timer thread". + Design-only doc for pattern-farming item **2.2** (the timer / delayed-dispatch primitive — "foundational, we have none"). No implementation here; this is the option analysis, the recommended shape, and a rigorous performance-impact diff --git a/docs/pattern-farming.md b/docs/pattern-farming.md index f1e0f13..9ffdf4f 100644 --- a/docs/pattern-farming.md +++ b/docs/pattern-farming.md @@ -228,7 +228,7 @@ over drop-cancellation systems. #### 2.2 Timer / delayed-dispatch primitive — **foundational, we have none** -**Status: 🔬 design delegated — doc ready: [timer-primitive-design.md](internals/timer-primitive-design.md).** +**Status: ✅ landed (2026-09) as `ts::sleep` / `ts::sleep_until` / `ts::Periodic` (`ts/timer.h`); design and deviations in [timer-primitive-design.md](internals/timer-primitive-design.md).** Recommendation: a lazily-created, scheduler-owned **timer thread with a `steady_clock` min-heap**, each fire delivered as a `Signal` trigger handed to the scheduler via a low-priority `launch` — structurally identical to diff --git a/show_graph.bat b/show_graph.bat index fd17cf9..23b2691 100644 --- a/show_graph.bat +++ b/show_graph.bat @@ -3,8 +3,8 @@ setlocal rem Render a Graphviz .dot file to .svg (next to it) and open it with the default app. rem Usage: show_graph.bat [path\to\graph.dot] rem No argument: renders sample_game_frame.dot AND opens the average-run trace SVGs -rem (sample_game_frame_avg_baseline.svg / _optimised.svg) if present -- the set -rem macrame_playground --dot / --trace produces. +rem (sample_game_frame_avg_baseline.svg / _optimised.svg / _fixed_rate.svg) if present -- +rem the set macrame_playground --dot / --trace produces. rem If Graphviz is missing, offers to install it via winget. set "DOTFILE=%~1" @@ -32,6 +32,7 @@ start "" "%SVGFILE%" if not "%~1"=="" exit /b 0 if exist "sample_game_frame_avg_baseline.svg" start "" "sample_game_frame_avg_baseline.svg" if exist "sample_game_frame_avg_optimised.svg" start "" "sample_game_frame_avg_optimised.svg" +if exist "sample_game_frame_avg_fixed_rate.svg" start "" "sample_game_frame_avg_fixed_rate.svg" exit /b 0 :install_graphviz From a02552aa140d58e098ec5b959b3f967415702fd8 Mon Sep 17 00:00:00 2001 From: Andriy Date: Mon, 14 Sep 2026 15:18:59 +0100 Subject: [PATCH 3/3] fixed-rate review: yield by class, resumes at yield points, one-allocation timer Yield points - `detail::yield_signal`: the queued `high` and global-`normal` counts, alone on one cache line; incremented before a global push, decremented after every global pop. Local-deque pushes stay uncounted. - `yield_to_higher` runs one entry of a higher class: `normal` -> `high`; `low` -> `high`, then the global `normal` queue; `high` never yields. - The nested dispatch detaches the resume and inline-dispatch trampolines (by value, out of line, a spare buffer keeps the nested drain allocation-free) and reattaches them afterwards, so a coroutine or inline node the nested task releases runs at the yield point, also inside a resumed segment or an inline drain. `Yield_nesting` makes yield points inside the nested task no-ops (one level deep). Timer - A wait is one `Timed_executable` block: `Executable` plus the timer bookkeeping. A fire or a cancel submits that block itself; a passed deadline or a requested token runs it in the call. `Periodic::next` is no longer a coroutine. - mem_profile rows for sleep and `Periodic::next`; TODO 4.9 for allocation-free ticks. Versioned - Document `Version_view`'s access-context contribution (one or two entries). Samples and tests - fixed_rate checks the version stamps and that the clock kept up with the grid. - The engine fixed-rate test runs 60 frames and checks both clocks ticked. - New tests: low yields to normal, normal does not yield to normal, a resume caused at a yield point runs there, an inline node released at a yield point runs there, no nested yielding; TSan stage `yield resumes`. Docs: guide 7, 9.2, 10.1, 10.6, 13; design 3 (yield points, timer thread); the timer design status; TODO; CLAUDE.md. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QTtvN1mH7K6MWzpYiyJ88w --- CLAUDE.md | 4 +- docs/TODO.md | 22 ++- docs/design.md | 50 +++-- docs/guide.md | 45 +++-- docs/internals/timer-primitive-design.md | 5 +- include/ts/coroutine_support.h | 36 ++++ include/ts/detail/task_block.h | 66 ++++++- include/ts/parallel_for.h | 12 +- include/ts/scheduler.h | 8 +- include/ts/task.h | 24 ++- include/ts/timer.h | 4 + include/ts/versioned.h | 3 + sample/fixed_rate.cpp | 40 ++-- sample/game_frame.cpp | 15 ++ src/mem_profile.cpp | 20 ++ src/scheduler.cpp | 78 ++++++-- src/timer.cpp | 236 ++++++++++++++--------- tests/graph_tests.cpp | 39 ++++ tests/integration_tests.cpp | 12 +- tests/scheduler_tests.cpp | 134 +++++++++++++ tsan/tsan_main.cpp | 52 +++++ 21 files changed, 720 insertions(+), 185 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ff10415..58159c6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,10 +31,10 @@ Two transformations landed in 2026-08 and define the current shape: the **evolve - **Access harness** (`access.{h,cpp}`) — `TS_CHECK_ACCESS()` at the top of every guarded method checks `this` against a thread-local `Access_context` the scheduler/pipe installs per task; a violation routes to `ts::fatal`. ~1 ns/call. Subtasks (e.g. `parallel_for` chunks) inherit the parent node's context. **Grant-window validity (2026-07)**: each `Pipe` carries a `write_epoch` (seqlock parity — bumped at write acquire/release under the pipe mutex, +2 on a graph write handoff; readers never bump); context entries declared under a pipe grant capture it, and `access_check` fatals with a stale-grant diagnostic when an inherited snapshot outlives its window (a non-nested `ts::launch` touching the launcher's data after the launcher's scope released — nested-gated sub-work is structurally never stale). `Access_context::check` returns granted/stale/none. Gated by `TS_SAFETY_CHECKS`. - **Waiting-rule policy** (`rules.h`, design of record `docs/internals/waiting-rule-policy.md`) — the coroutine-first waiting rules (`docs/internals/coroutine-first.md` §2) are enforced by runtime checks that fatal, and each is separately (1) compiled in via `TS_ENABLED_RULES` (a bitmask of `TS_RULE_*`) and (2) opted out of per scope via `ts::Relaxed_scope` for the rules that permit it. `ts::Rule` mirrors the bits: **`in_task_sync`** (`sync()`/`take()` inside a task), **`await_under_guard`** (`co_await` while a `Access_guard` is live), **`access_rank`** (awaiting an object out of declared rank order), **`circular_wait`** (a held-grant → awaited-pipe wait cycle), **`deadlock_net`** (quiescence with no possible external wakeup). Rule classes: `advisory` (`in_task_sync|access_rank|circular_wait` — a scoped `Relaxed_scope` opt-out, "I uphold this by means the library can't see"), `structural` (`await_under_guard` — compile-out-only, its absence corrupts rather than merely permits, so it is the ONE rule shipping keeps), `net` (`deadlock_net` — global, unscopable). `set_default_relaxed_rules(Rule)` is a process-wide advisory-relaxation baseline. `circular_wait`/`access_rank` read grant bookkeeping that exists only under `TS_SAFETY_CHECKS`, so they can't outlive it whatever the policy asks (the EFFECTIVE mask = policy ∩ build support). `rule_enforced(rule)` is the one predicate a check calls, only AFTER the cheap hazard condition is already true (the relaxation lookup stays off the common path). **`detail::relaxed_rules` is reached ONLY through `detail::relaxed_load()`/`relaxed_store(unsigned)`, both `TS_DETAIL_NO_INLINE` (2026-08-22)** - a compiler may resolve a thread-local's block address once and keep it in a coroutine frame across a suspension, so an INLINED read after a cross-thread resume answers for the SUSPENDING thread (MSVC 19.51 does; the class is cross-compiler with an incomplete upstream fix - LLVM #47179 / #63022 / D92661, and it extends to anything thread-identifying, e.g. clang's `pthread_self()` case and LLVM #72006). The barrier lives in the accessors, so every toucher (`relaxed_bits`, `Relaxed_scope` ctor/dtor, `Relaxed_carrier`, `snapshot_relaxed`, `Inherited_relaxed_scope`) inherits it and none carries its own `TS_DETAIL_NO_INLINE`; load/store BY VALUE (an `unsigned&` accessor re-opens the hazard one level up - the Rust `thread_local!`+`.with()` shape). Regression test "rules relaxed scope reads the resuming thread" (it fails again if the accessors are made inlinable - verified both directions); rationale in `docs/internals/waiting-rule-policy.md` §4.1. A `Relaxed_scope` entered in a coroutine body follows the ambient task state, not the thread (`Relaxed_carrier` on the promise re-installs it around every segment). **`ts::Rank`** (`access.h`) — a declared lock order (`Guarded(Named, Rank, args...)`), NOT defaulted, required only for objects dynamically awaited while another grant is held; the `access_rank` rule fatals on an out-of-order await. **Deadlock report, three tiers** (`docs/internals/waiting-rule-policy.md` §7): tiers 1 (the `circular_wait` cycle) and 2 (the `deadlock_net` quiescence net — the scheduler idle with nothing externally outstanding for `set_deadlock_net_window`, default 2 s) are free and always present; tier 3 is `TS_SUSPENSION_REGISTRY` — an opt-in per-suspension registry (who is suspended, what they await, what they hold), default ON in DEBUG only (a linked-list insert/remove per suspension, ~30 ns, ~8% on the suspend/resume microbench), which forces `TS_DEBUG_NAMES` on (a registry of block pointers is not a diagnostic). **`ts::External_wait`** (`task.h`) — the `deadlock_net` escape: hold one while a wakeup owned by a thread the scheduler doesn't run (OS I/O, GPU fence, a `Signal` from a dedicated thread, a `Frame_gate` `open()`) is outstanding, so the net doesn't false-positive; a FORGOTTEN registration produces a false deadlock report, which is why the report names the type. **`ts::Named`** (`named.h`) — the one unified debug-identity type for graph nodes (required), guarded objects (required), and tasks (optional, `Dispatch_options::name`/`Access_options::name` — both are a `ts::Named` since 2026-08 (M6), not a `const char*`: the call SITE is captured by the verb's own defaulted `source_location`, so an unnamed task is still identified, EXCEPT on the multi-object verbs, which end in an object pack and can carry no defaulted `source_location` — there `{.name = ts::Named{}}` is the only way to capture the site, and `detail::named_from` prefers the option's `Named` when non-empty). A `Named` is a literal OR a `{}` call-site capture (`file`+`line` only, three words); `named_display` renders it. The load-bearing library rule: a defaulted `source_location` captures the CALLER, so it must sit on the OUTERMOST function the user calls and be passed down explicitly — an inner helper re-defaulting it captures the library header (`tests/named_tests.cpp` asserts captured names point into the test file). Debug identity is gated by `TS_DEBUG_NAMES` (default = the harness, forced on by `TS_SUSPENSION_REGISTRY`). - **`fatal.{h,cpp}`** — all non-recoverable failures call `ts::fatal` (message + `std::stacktrace` + `abort`). **Exception-agnostic since 2026-08-21**: the library never throws its own exceptions (its only `try`/`catch` - and a bare `throw;` re-raise in the promise's `unhandled_exception` - sit behind `__cpp_exceptions`/`_CPPUNWIND` guards, solely to diagnose an exception escaping a user body) and builds with OR without exception support - the default CMake/vcxproj Debug+Release build now has them ON (matching what a `find_package` consumer gets), `-DMACRAME_NO_EXCEPTIONS=ON` (CMake) / the vcxproj Shipping config / `tsan/run.sh` / the pre-commit hook build them OFF, which is what keeps the source throw-free. The contract that makes both work: **an exception must not leave a user body** - every path that invokes one goes through `detail::invoke_user_body` (task_block.h; `Executable::run`, `Access_op::State::run`, the graph node body, both `parallel_for` loops) whose handlers (compiled only where the calling TU has exceptions) call the `escaped_exception_diagnose` seam (declared task_block.h, defined guarded.cpp next to `blocking_sync_diagnose`), and a coroutine body's escape reports the same way through the promise's `unhandled_exception`. The message names the running task (`task_name(current_task)`) + `what()` when the exception derives from `std::exception`, because a handler runs POST-unwind so the stacktrace starts at the seam, not the throw. Five death tests, `run_if(with_exceptions, ...)`. `MACRAME_NO_EXCEPTIONS` exports `_HAS_EXCEPTIONS=0` (+ the flag where expressible) as a PUBLIC usage requirement because the MSVC STL bakes it into declarations; `access.h` carries a `detect_mismatch` tripwire for it (verified: a mixed link fails LNK2038). Rationale in `docs/design.md` §4.6, user contract in `docs/guide.md` §10.5. Tests are the one place failures are non-fatal (the harness records and continues); fatal paths are checked via subprocess death tests. **`TS_ENSURE(expr, message)` (2026-07, `TS_SAFETY_CHECKS`-gated)** — UE-`ensure`-shaped recoverable assert macro (fatal.h): evaluates `expr` once in both configs and yields its bool; on failure bumps `ts::ensure_failure_count()` on EVERY occurrence but reports ONCE PER CALL SITE (captureless-lambda function-local static claim), through a swappable presentation handler (`ts::set_ensure_handler`, `std::set_terminate` shape; default = `ENSURE FAILED:` + stack + debugger-break if attached via `detail::is_debugger_present`/`debug_break`, the C++26 P2546 pair polyfilled Win/Linux/mac; `ts::fatal` breaks too, before `abort`). Counting is outside the handler. The harness fails on failures not consumed via `ts::test::consume_ensure_failures(n)`; `--bench`/`--stress` fail their exit code on any. Its first user was the blocking-sync diagnostic, which the coroutine-first transformation PROMOTED to a hard fatal: an in-task `sync()`/`take()` on unsettled work has no sanctioned form any more (there is no retraction to fall back on), so `blocking_sync_diagnose` (seam task.h→guarded.cpp) calls `ts::fatal` — a sharp same-object message when the target is an access on a pipe the current context holds, a general never-park message otherwise. `parallel_for`'s join is the one structural exemption (it waits on group state, never through `sync_wait`). `TS_ENSURE` remains the recoverable-assert facility for new diagnostics. -- **Samples** (`sample/`) — six SINGLE-FILE, self-contained .cpp samples (no headers — consumers forward-declare the entries; the blackboard convention, now the rule). `game_frame.cpp`: a ~30-system mock game-engine frame built in TWO variants from the SAME system bodies via `build_frame_graph(World&, Frame_variant, dot)` — `baseline` (straightforward composition) and `optimised` (levers applied in an `optimise()` section + branched node bodies), the worked optimisation exercise in [docs/example-frame-optimization.md](docs/example-frame-optimization.md). Exercises every layer — schedule derived from declared access, a frame head (camera/networking/scripting), gameplay trio, AI (speculative cancellable nav queries, trailing-`Cancellation_token` early-out), animation chain (graph→IK→skinning), split physics pipeline (broadphase→narrowphase→solver→finalize; the multi-conflict `physics→propagation` edge via velocities+bodies), a RENDER pipeline reading LAST frame's transforms via `Versioned` so it overlaps sim (the render-thread-with-one-frame-latency model), off-path leaves (vfx/replication/stats/gc/navmesh_rebuild), `ts::parallel_for` in heavy systems, streaming via a fire-and-forget coroutine that fires four `async` loads and folds them with sequential `co_await`s (the linear spelling of the old `then`/`when_all` chain), a per-frame HUD snapshot via multi-object `ts::async(fn, combat, economy)`, one bare generic-lambda node (`const auto&` probe). Priorities are IDENTICAL in both variants (kept for their visualization, NOT a lever — compilation ignores them). The optimised levers, all tool-guided from the baseline trace: (1) AI reads LAST frame's gameplay via a second `Versioned` snapshot (the gameplay trio's results staged + flipped at frame end) — deletes the `trio→AI` edges, the SINGLE BIGGEST cut (−821 µs), the marquee dependency-cut lever; (2) parallelise the fattest serial CRITICAL bars (`combat`/`ik_post`/`UI` — critical-path-bound, so splits move the makespan); (3) `Deferred` draw staging (producers go wide — utilization, NOT makespan); (4) cloth on prev-frame transforms (off the post-flip tail; neutral on the long spine, fills cores once the spine is short). Baseline ≈85% util / ≈12% dead / ≈6.5 ms → optimised ≈95% util / ≈10% dead / ≈5.6 ms on 8 workers (`variant_workers`; util/dead are the portable numbers, frame time is machine-dependent) — the picture flips from dependency-bound to CORE-bound (cores full, chain waits for a core, not a dependency; the single critical spine fragments), the "optimisation is done" signal. The older 4793→3120 µs / 60→85% util figures were a profiler artifact, not a bigger workload: `parallel_for` fan-out leaked to the global default pool, so the "6-worker" trace secretly ran on ~12 cores — frame times ~2× too fast, util undercounted the leaked work. The serial workload has been a constant ~36.6 ms since the 30-system frame landed (2026-07-24); the fixes that made the trace honest were the `parallel_for`-on-current-scheduler routing, owner attribution, time-bucketed utilization, and the body/machinery split. `transform0==5` determinism held both variants. **A THIRD composition (2026-08): `Frame_variant::graph_free` / `run_frame_graph_free(World&)`** — the baseline frame with no `Static_task_graph` at all, the measurement closing `docs/internals/coroutine-first.md` §10.5: every system is a multi-object `ts::async`, every derived edge an explicit `co_await` inside one of the frame's chain coroutines, completion joined by hand. Three findings (`docs/guide.md` §6.4, `docs/design.md` §2.4): (1) safety is the PIPE's, not the graph's — unchanged; (2) **pipe FIFO does NOT stand in for conflict edges** (the multi-object cascade enters links one at a time in canonical order, so a system blocked on its first object hasn't taken its slot on the later ones and a later launch overtakes it) — the naive declaration-order-no-awaits version mis-orders `frustum_cull` before `camera` and `submit` before `cmd_record`, losing a frame of draw commands SILENTLY (declarations all correct, harness quiet) and running 7.6% faster for it; (3) **the graph's perf edge is resume locality, not allocation amortization** — graph-free costs ~95 more allocs/frame (<2 µs) but +64 µs/frame on a 4.1 ms frame and +131 µs on a 0.45 ms one (+1.6% / +28.7%), which is ~50 coroutine suspend/resume round trips at ~1.8 µs. Exports `game_frame_free_stats(...)` (same shape as `game_frame_stats`) + `game_frame_draw_count()` (the submitted-command count — the observable that DISCRIMINATES producer/submit ordering; `transform0` cannot, since every mock system writes the same constant every frame so a one-frame skew is invisible). Wired into `--bench` (`graph 1.0`/`free 1.0`/`graph .05`/`free .05` µs/frame), `--memprofile` (`frame graph`/`frame graph-free` allocs/frame, measured as an n-frame minus 1-frame difference so setup cancels), `--stress`, tsan_main, and the `engine frame without a graph` integration test. `--trace` traces both graph variants on a `variant_workers`(=8)-worker scheduler → `sample_game_frame_avg_baseline.svg` + `sample_game_frame_avg_optimised.svg` + the DOT. Exports `game_frame_stats(frames, scale, avg&, serial&, xf0&)` (baseline, for the determinism/utilization test) + `stress_game_frame_optimised(frames, workers)` (headless optimised run — TSan covers the gameplay `Versioned` publisher + `Deferred` staging) + `run_game_frame_sample` + `dump_game_frame_dot` + `trace_game_frame`. `physics.cpp`: the machine/extract decomposition (sealed `Guarded`, single grant holder = the sim's write; `Deferred` inputs; `Versioned` outputs; snapshot-based scene query; id-reservation forward refs; batch-extract; deterministic run-twice check); exports `run_physics_sample` + `physics_pose_hash` (tsan). `blackboard.cpp`: the blackboard recipe (doc §7.4). `scope_access.cpp`: the scope-based access showcase — a tiny bank exercising the `co_await ts::read_write`/`read_only` held-grant guards in three shapes: single-object read-modify-write under a held grant; several accounts one-by-one (each its own scope, released before the next `co_await`); and a two-account transfer taken together (the multi-object hold). Runs twice, checking transfers conserve the total and both runs agree. Exports `run_scope_access_sample`. `events.cpp`: the delegate/event recipe (docs/pattern-farming.md 2.59) — how UE-style delegates decompose onto the primitives, four tiers in one toy: intra-system (a plain synchronous `move_only_function` delegate fired under the node's grant — sanctioned, zero machinery; the one rule: outbound effects via `async`/`stage` only), command (`audio.async` from the handler), lightweight notifications (`ts::Event_bus` — promoted 2026-08 from this sample's prototype to a public header, see its own bullet; the sample demonstrates the three subscribe forms, pinned auto-disconnect mid-run via the retiring commander, the explicit `dispatch.after(producers)` intent edge, and two minimal with/without-graph setup floors checked every run), and heavyweight flows (a `Guarded` batch + conflict-derived edge — O(events) with a 1000-NPC population). Determinism self-check (run-twice compare). Exports `run_events_sample`. `coloring.cpp`: interaction coloring (pattern-farming 2.39, author-flagged important) — the cross-entity case deferral cannot serve: a Gauss-Seidel cloth solver (Verlet + projection) greedy-colors its constraint graph (4 colors on a grid, disjointness validated), runs each color band as one `parallel_for` under the node's grant with bands sequential as barriers; two disconnected patches demonstrate islands need no special handling; bit-deterministic under any chunking/stealing (≤1 writer per particle per band). Exports `run_coloring_sample`. `fixed_rate.cpp` (2026-09): a 60 Hz physics graph on its own `ts::Periodic` clock beside the frame graph, boundary = `Deferred` intents + `Versioned` poses read lent by render; gameplay's long body yields; structural self-checks every run (ticks in order, consecutive interpolated pair, fraction in [0,1], every staged intent applied once) since which tick picks up an intent is wall-clock dependent; `fixed_rate_physics_hash` is the determinism check (physics graph alone, tick-indexed intent script, same hash across worker counts). Exports `run_fixed_rate_sample` (`--fixed-rate [n]`), `fixed_rate_self_check`, `fixed_rate_physics_hash`, `stress_fixed_rate`. `game_frame.cpp` gained **`Frame_variant::fixed_rate`** (2026-09): the optimised frame with the physics chain on a 60 Hz graph and networking on a 30 Hz graph (`Fixed_rate_ticks` builds both, primes them, drives each from a `Periodic`, runs a final tick of each at teardown), frame nodes reading `body_snapshot`/`net_snapshot` and staging `physics_intents`/`net_outbox_stream`; propagation interpolates the two newest physics ticks. Exports `game_frame_fixed_stats`; third `--trace` SVG (`_fixed_rate`); `fixed 1.0`/`fixed .05` `--bench` rows; integration test `engine frame with fixed-rate physics and networking` (transform0 == 5, same draw count as the baseline). +- **Samples** (`sample/`) — six SINGLE-FILE, self-contained .cpp samples (no headers — consumers forward-declare the entries; the blackboard convention, now the rule). `game_frame.cpp`: a ~30-system mock game-engine frame built in TWO variants from the SAME system bodies via `build_frame_graph(World&, Frame_variant, dot)` — `baseline` (straightforward composition) and `optimised` (levers applied in an `optimise()` section + branched node bodies), the worked optimisation exercise in [docs/example-frame-optimization.md](docs/example-frame-optimization.md). Exercises every layer — schedule derived from declared access, a frame head (camera/networking/scripting), gameplay trio, AI (speculative cancellable nav queries, trailing-`Cancellation_token` early-out), animation chain (graph→IK→skinning), split physics pipeline (broadphase→narrowphase→solver→finalize; the multi-conflict `physics→propagation` edge via velocities+bodies), a RENDER pipeline reading LAST frame's transforms via `Versioned` so it overlaps sim (the render-thread-with-one-frame-latency model), off-path leaves (vfx/replication/stats/gc/navmesh_rebuild), `ts::parallel_for` in heavy systems, streaming via a fire-and-forget coroutine that fires four `async` loads and folds them with sequential `co_await`s (the linear spelling of the old `then`/`when_all` chain), a per-frame HUD snapshot via multi-object `ts::async(fn, combat, economy)`, one bare generic-lambda node (`const auto&` probe). Priorities are IDENTICAL in both variants (kept for their visualization, NOT a lever — compilation ignores them). The optimised levers, all tool-guided from the baseline trace: (1) AI reads LAST frame's gameplay via a second `Versioned` snapshot (the gameplay trio's results staged + flipped at frame end) — deletes the `trio→AI` edges, the SINGLE BIGGEST cut (−821 µs), the marquee dependency-cut lever; (2) parallelise the fattest serial CRITICAL bars (`combat`/`ik_post`/`UI` — critical-path-bound, so splits move the makespan); (3) `Deferred` draw staging (producers go wide — utilization, NOT makespan); (4) cloth on prev-frame transforms (off the post-flip tail; neutral on the long spine, fills cores once the spine is short). Baseline ≈85% util / ≈12% dead / ≈6.5 ms → optimised ≈95% util / ≈10% dead / ≈5.6 ms on 8 workers (`variant_workers`; util/dead are the portable numbers, frame time is machine-dependent) — the picture flips from dependency-bound to CORE-bound (cores full, chain waits for a core, not a dependency; the single critical spine fragments), the "optimisation is done" signal. The older 4793→3120 µs / 60→85% util figures were a profiler artifact, not a bigger workload: `parallel_for` fan-out leaked to the global default pool, so the "6-worker" trace secretly ran on ~12 cores — frame times ~2× too fast, util undercounted the leaked work. The serial workload has been a constant ~36.6 ms since the 30-system frame landed (2026-07-24); the fixes that made the trace honest were the `parallel_for`-on-current-scheduler routing, owner attribution, time-bucketed utilization, and the body/machinery split. `transform0==5` determinism held both variants. **A THIRD composition (2026-08): `Frame_variant::graph_free` / `run_frame_graph_free(World&)`** — the baseline frame with no `Static_task_graph` at all, the measurement closing `docs/internals/coroutine-first.md` §10.5: every system is a multi-object `ts::async`, every derived edge an explicit `co_await` inside one of the frame's chain coroutines, completion joined by hand. Three findings (`docs/guide.md` §6.4, `docs/design.md` §2.4): (1) safety is the PIPE's, not the graph's — unchanged; (2) **pipe FIFO does NOT stand in for conflict edges** (the multi-object cascade enters links one at a time in canonical order, so a system blocked on its first object hasn't taken its slot on the later ones and a later launch overtakes it) — the naive declaration-order-no-awaits version mis-orders `frustum_cull` before `camera` and `submit` before `cmd_record`, losing a frame of draw commands SILENTLY (declarations all correct, harness quiet) and running 7.6% faster for it; (3) **the graph's perf edge is resume locality, not allocation amortization** — graph-free costs ~95 more allocs/frame (<2 µs) but +64 µs/frame on a 4.1 ms frame and +131 µs on a 0.45 ms one (+1.6% / +28.7%), which is ~50 coroutine suspend/resume round trips at ~1.8 µs. Exports `game_frame_free_stats(...)` (same shape as `game_frame_stats`) + `game_frame_draw_count()` (the submitted-command count — the observable that DISCRIMINATES producer/submit ordering; `transform0` cannot, since every mock system writes the same constant every frame so a one-frame skew is invisible). Wired into `--bench` (`graph 1.0`/`free 1.0`/`graph .05`/`free .05` µs/frame), `--memprofile` (`frame graph`/`frame graph-free` allocs/frame, measured as an n-frame minus 1-frame difference so setup cancels), `--stress`, tsan_main, and the `engine frame without a graph` integration test. `--trace` traces both graph variants on a `variant_workers`(=8)-worker scheduler → `sample_game_frame_avg_baseline.svg` + `sample_game_frame_avg_optimised.svg` + the DOT. Exports `game_frame_stats(frames, scale, avg&, serial&, xf0&)` (baseline, for the determinism/utilization test) + `stress_game_frame_optimised(frames, workers)` (headless optimised run — TSan covers the gameplay `Versioned` publisher + `Deferred` staging) + `run_game_frame_sample` + `dump_game_frame_dot` + `trace_game_frame`. `physics.cpp`: the machine/extract decomposition (sealed `Guarded`, single grant holder = the sim's write; `Deferred` inputs; `Versioned` outputs; snapshot-based scene query; id-reservation forward refs; batch-extract; deterministic run-twice check); exports `run_physics_sample` + `physics_pose_hash` (tsan). `blackboard.cpp`: the blackboard recipe (doc §7.4). `scope_access.cpp`: the scope-based access showcase — a tiny bank exercising the `co_await ts::read_write`/`read_only` held-grant guards in three shapes: single-object read-modify-write under a held grant; several accounts one-by-one (each its own scope, released before the next `co_await`); and a two-account transfer taken together (the multi-object hold). Runs twice, checking transfers conserve the total and both runs agree. Exports `run_scope_access_sample`. `events.cpp`: the delegate/event recipe (docs/pattern-farming.md 2.59) — how UE-style delegates decompose onto the primitives, four tiers in one toy: intra-system (a plain synchronous `move_only_function` delegate fired under the node's grant — sanctioned, zero machinery; the one rule: outbound effects via `async`/`stage` only), command (`audio.async` from the handler), lightweight notifications (`ts::Event_bus` — promoted 2026-08 from this sample's prototype to a public header, see its own bullet; the sample demonstrates the three subscribe forms, pinned auto-disconnect mid-run via the retiring commander, the explicit `dispatch.after(producers)` intent edge, and two minimal with/without-graph setup floors checked every run), and heavyweight flows (a `Guarded` batch + conflict-derived edge — O(events) with a 1000-NPC population). Determinism self-check (run-twice compare). Exports `run_events_sample`. `coloring.cpp`: interaction coloring (pattern-farming 2.39, author-flagged important) — the cross-entity case deferral cannot serve: a Gauss-Seidel cloth solver (Verlet + projection) greedy-colors its constraint graph (4 colors on a grid, disjointness validated), runs each color band as one `parallel_for` under the node's grant with bands sequential as barriers; two disconnected patches demonstrate islands need no special handling; bit-deterministic under any chunking/stealing (≤1 writer per particle per band). Exports `run_coloring_sample`. `fixed_rate.cpp` (2026-09): a 60 Hz physics graph on its own `ts::Periodic` clock beside the frame graph, boundary = `Deferred` intents + `Versioned` poses read lent by render; gameplay's long body yields; structural self-checks every run (ticks in order, consecutive interpolated pair, `Version_stamps` consistent with the pair - serial = the newest tick, publish instants ordered - the clock kept up with at least half the grid points that passed, every staged intent applied once) since which tick picks up an intent is wall-clock dependent; `fixed_rate_physics_hash` is the determinism check (physics graph alone, tick-indexed intent script, same hash across worker counts). Exports `run_fixed_rate_sample` (`--fixed-rate [n]`), `fixed_rate_self_check`, `fixed_rate_physics_hash`, `stress_fixed_rate`. `game_frame.cpp` gained **`Frame_variant::fixed_rate`** (2026-09): the optimised frame with the physics chain on a 60 Hz graph and networking on a 30 Hz graph (`Fixed_rate_ticks` builds both, primes them, drives each from a `Periodic`, runs a final tick of each at teardown), frame nodes reading `body_snapshot`/`net_snapshot` and staging `physics_intents`/`net_outbox_stream`; propagation interpolates the two newest physics ticks. Exports `game_frame_fixed_stats`; third `--trace` SVG (`_fixed_rate`); `fixed 1.0`/`fixed .05` `--bench` rows; integration test `engine frame with fixed-rate physics and networking` (60 frames: transform0 == 5, same draw count as the baseline, both clocks ticked - `game_frame_fixed_tick_counts`). - **`task.h`** — the task layer's PUBLIC surface (split 2026-08-17, the guarded.h playbook): `Task`, the option aggregates (`Dispatch_options`, `Access_options`), the bare-scheduler task primitive `ts::launch` (dispatches through the `submit_ready` bridge, so no direct scheduler dependency; detached — inherits no grant), `Signal`, `External_wait` + `set_deadlock_net_window`, and the `core_of`/`task_from_core`/`Optional_awaitable` detail bridges. ~330 lines; includes the two split-out headers so every existing includer keeps working. **`ts/cancellation.h`** (public) — the cancellation cluster: `Cancellation_source`/`Cancellation_token`/`Cancel_callback` + `detail::Cancel_state`. **`ts/detail/task_block.h`** — the block layer: the fully-monomorphic `detail::Task_control_block`, `Task_ptr`/`Adopt_ref`/`Destroy_queue`, `Block_backed` + `Executable`/`make_executable` (wrappers derive from the block; recovery is a derived cast), `make_bare_block` + the settled/cancelled sentinel cores, the naming helpers, `current_task`/`current_scope_children` TLS, the body traits (`Task_body`, `Task_result`), `detail::add_nested`, and the scheduler-free seams (`submit_ready`, `pipe_enter_first`, `drain_serial_pending`, `blocking_sync_diagnose`, the deadlock-net internals — declared there, defined in the scheduler/pipe layers). All header-only — no `task.cpp`; `guarded.h` includes `task.h`. - **`Frame_gate`** (`frame_gate.h`, 2026-08, TODO 6.7) — the cross-frame realignment utility: `co_await gate.next()` parks a task until the frame loop's next `open()`, so work that resumed from an external completion at an arbitrary moment re-enters at a frame boundary instead of mid-update. A type rather than a documented `Signal::reset` idiom because the hand-rolled version has a missed-wakeup window (a task reading the signal just before a boundary can attach to a gate about to be re-armed) and `reset()`'s "every waiter already released" precondition (fatal otherwise); the gate hands out the CURRENT frame's signal under a mutex and installs a fresh one at `open()`, trading one bare block per frame. `open()` releases through the scheduler at `Priority::low` by default (`set_release_priority` overrides) — an inline trigger would run every parked frame on the frame loop's own thread before `open()` returned. The same hop is the sanctioned idiom for an OS completion callback triggering a `Signal` (TODO 6.6 stays a design note: the bridge is one line, the packaging question is who owns the WAITING, and that needs the platform layer). -- **Fixed-rate graphs (2026-09)** — a subsystem on its own clock is a second compiled graph beside the frame graph on the one scheduler, bounded by a `Deferred` for inputs (sampled at the tick's commit) and a `Versioned` for outputs (published at the tick's end): the logical-execution-time model (guide §6.6, design §6.1). No bundled rate-domain type, by decision - ownership split, boundary contents and overload policy are engine choices; the library ships four mechanisms. (1) **`ts/timer.h`**: `ts::sleep`/`sleep_until` return a bare-block `Task` settled by one lazily created timer thread (min-heap; delivery is a `ts::launch` at `Sleep_options::priority`, never inline; cancellation through a `Cancel_callback` settles cancelled promptly; each armed sleep holds an `External_wait`; Windows waits on a `CREATE_WAITABLE_TIMER_HIGH_RESOLUTION` timer plus an auto-reset wake event; worker-less mode is fatal; `destroy_scheduler` stops the thread first and fatals on a live sleep under `TS_SAFETY_CHECKS`; program exit stops it through a static in guarded.cpp declared after `g_scheduler`). `ts::Periodic`: grid deadlines, `next()` returns grid points passed, 0 once its token trips. (2) **`Versioned`** (`History` is a template parameter; opaque-declared in journal.h so the recorder friends can name the two-parameter template): a third replica rotated in `swap_replicas` under the write grant, `Version_stamps` (publish instants, serial, `fraction_at`), copy resync by default, `Resync::replay` a construction fatal. `read_last_versions()` returns a `Version_view` (non-movable guard installing its own context, tuple protocol for `auto [previous, current, stamps]`; lent when the context grants the front, a blocking read turn on a blue thread, fatal under `Rule::in_task_sync` in a grant-less task, a live guard for `await_under_guard` in a coroutine); the free `co_await ts::read_last_versions(v)` uses `detail::Version_awaiter`, which derives from `Access_awaiter` (its resume bookkeeping is now `finish_acquire()`). (3) **`ts::yield()`** (task.h): reads `detail::high_queued` (relaxed; `Scheduler::submit` increments before a `high` push, `find_work` decrements after a pop); the slow path `detail::yield_to_high(Priority)`, a `Scheduler` friend, pops one `high` entry and runs it inline under a `Nested_dispatch_scope` that clears Current_task, access, scope children, relaxed rules and trace owner, and books the nested span to `Nested_span_state`, which `Trace_busy_scope` subtracts; no-op off a worker, in worker-less mode, and at `high`. `parallel_for` (flat and colored loops) yields at every chunk claim at `Parallel_base::priority`. (4) **`Static_task_graph::set_default_priority`** (`Node::priority_set`; `add_node` seeds from the default; the move ctor lists `default_priority_`). Tests: `timer_tests` group; versioned, scheduler (`yield:`), parallel, graph (`default priority`) and integration additions; each fatal has a death scenario. Negative checks done: yield no-op, rotation without the second swap, sleep without `External_wait`. Measured 2026-09, 8 workers, 100 traced frames: optimised 16.93 ms vs fixed_rate 17.12 ms - neutral, because the optimised frame is already core-bound; the gain on this fixture is structural (one frame shape, tick count owned by the clock, overload policy local to the driver), not throughput. +- **Fixed-rate graphs (2026-09)** — a subsystem on its own clock is a second compiled graph beside the frame graph on the one scheduler, bounded by a `Deferred` for inputs (sampled at the tick's commit) and a `Versioned` for outputs (published at the tick's end): the logical-execution-time model (guide §6.6, design §6.1). No bundled rate-domain type, by decision - ownership split, boundary contents and overload policy are engine choices; the library ships four mechanisms. (1) **`ts/timer.h`**: `ts::sleep`/`sleep_until` return a `Task` whose block is a `Timed_executable` (timer.cpp: `Executable` + `Timed_fields` - live/delivered flags and the `Cancel_callback`, declared last so destroyed first; one allocation per wait), kept by one lazily created timer thread (min-heap of block refs; a fire or a cancel submits that same block through `submit_ready` at `Sleep_options::priority`, never inline, and `Executable::run` settles it cancelled when its token was requested; a deadline already passed or a token already requested runs the block in the call; an armed wait counts in `outstanding_external_waits` from arm to delivery; a wait dropped by shutdown destroys its unrun body in `~Timed_executable`; Windows waits on a `CREATE_WAITABLE_TIMER_HIGH_RESOLUTION` timer plus an auto-reset wake event; worker-less mode is fatal; `destroy_scheduler` stops the thread first and fatals on a live sleep under `TS_SAFETY_CHECKS`; program exit stops it through a static in guarded.cpp declared after `g_scheduler`). `ts::Periodic`: grid deadlines; `next()` is not a coroutine - it arms a timed block whose body `advance()` returns the grid points passed (0 once its token trips; the block carries an empty token, so a cancelled tick settles completed with 0, never cancelled); allocation-free ticks are TODO 4.9. (2) **`Versioned`** (`History` is a template parameter; opaque-declared in journal.h so the recorder friends can name the two-parameter template): a third replica rotated in `swap_replicas` under the write grant, `Version_stamps` (publish instants, serial, `fraction_at`), copy resync by default, `Resync::replay` a construction fatal. `read_last_versions()` returns a `Version_view` (non-movable guard installing its own context - the running one plus `previous` and, unless lent, the front, so one or two of `Access_context::max_entries`; tuple protocol for `auto [previous, current, stamps]`; lent when the context grants the front, a blocking read turn on a blue thread, fatal under `Rule::in_task_sync` in a grant-less task, a live guard for `await_under_guard` in a coroutine); the free `co_await ts::read_last_versions(v)` uses `detail::Version_awaiter`, which derives from `Access_awaiter` (its resume bookkeeping is now `finish_acquire()`). (3) **`ts::yield()`** (task.h): reads `detail::yield_signal` (task_block.h: an `alignas(64)` pair of relaxed counters alone on a line, `high` and `normal_global`; `Scheduler::submit` increments before a global push, every global pop decrements - both `find_work` normal-queue pops and `yield_to_higher`'s; local-deque pushes are not counted, keeping the producer fast path free of shared writes); the slow path `detail::yield_to_higher(Priority)`, a `Scheduler` friend, pops one entry of a higher class (`normal` → `high`; `low` → `high`, then the global `normal` queue; `high` never) and runs it inline under a `Nested_dispatch_scope` that clears Current_task, access, scope children, relaxed rules and trace owner, detaches the `Resume_queue` and `Inline_queue` drains (`detach`/`reattach`: by value, out of line, the nested drain on a `spare_` buffer so it allocates nothing in steady state, a `TS_SAFETY_CHECKS` fatal if the nested drain did not finish; the destroy trampoline stays attached) so a resume the nested task causes runs at the yield point even inside a resumed segment, sets `Yield_nesting` (a `Tls_scalar` local to scheduler.cpp; yield points inside the nested task are no-ops, one level deep), and books the nested span to `Nested_span_state`, which `Trace_busy_scope` subtracts; no-op off a worker and in worker-less mode. Why running resumes at a yield point collides with none of the old settle races (c8c7688's `Signal::trigger` pin, the graph's `done` keep-alive, the awaiter handshake): design §3 "Yield points". `parallel_for` (flat and colored loops) yields at every chunk claim at `Parallel_base::priority`. (4) **`Static_task_graph::set_default_priority`** (`Node::priority_set`; `add_node` seeds from the default; the move ctor lists `default_priority_`). Tests: `timer_tests` group; versioned, scheduler (`yield:` - incl. low-to-normal, normal-not-to-normal, resume-at-yield-point, no-nesting), parallel, graph (`default priority`) and integration additions (`test_engine_fixed_rate` runs 60 frames and checks both clocks ticked via `game_frame_fixed_tick_counts`; the fixed_rate self-check verifies the stamps and that ticks plus drops cover the grid points passed); tsan stage `yield resumes`; each fatal has a death scenario. Negative checks done: yield no-op, rotation without the second swap, sleep without `External_wait`; review round: normal yields to normal, no trampoline detach, no nesting bound, uncounted ticks, stamp serial off by one. Measured 2026-09, 8 workers, 100 traced frames: optimised 16.93 ms vs fixed_rate 17.12 ms - neutral, because the optimised frame is already core-bound; the gain on this fixture is structural (one frame shape, tick count owned by the clock, overload policy local to the driver), not throughput. - **Headers** — `ts/ts.h` is the umbrella (`version`, `scheduler`, `guarded`, `task`, `static_task_graph`, `parallel_for`, `recorder`, `deferred`, `versioned`, `event_bus`, `access`, `rules`, `frame_gate`, `timer`, `coroutine_support`; `named.h`/`priority.h` come in transitively). `frame_gate.h` joined it 2026-08-22 (the API-stability pass, S13 — the exclusion had no technical basis: it includes only `ts/task.h` and std headers). `version.h` carries `ts::version_major/minor/patch` + `version_string()` beside the `TS_VERSION_*` macros. `ts/task_scope.h` was **removed (2026-08)** with `ts::Task_scope`. Docs map. Roadmap and open design questions live in `docs/TODO.md`. Two designs of record must be read before touching their areas: **`docs/internals/coroutine-first.md`** (the end-state model, the waiting rules (suspension under access), what was deleted and what replaces it, the fatal+companion test matrix, and §11's remaining action list — read before touching `task.h`/`coroutine_support.h`) and **`docs/internals/pipe-rebase.md`** §0 (the evolved pipe, why the lock-free chain was retired, the unified cascade and commit ladder — read before touching `guarded.cpp`/the pipe; its test plan is `docs/internals/pipe-rebase-tests.md`). Then: the dynamic-task internals (lifecycle, lock-counter, the access invariant, allocation, the graph's §10 scenarios) in `docs/internals/task-internals.md`; the deferred-write layer's state handoff (contracts, load-bearing mechanisms, ranked future plan — read before touching `journal.h`/`deferred.h`/`versioned.h`) in `docs/internals/deferred-versioned-state.md`; the waiting-rule check policy (the `ts::Rule` bitmask, `Relaxed_scope`, `ts::Rank`, the deadlock report's three tiers — read before touching `rules.h`/the rule checks) in `docs/internals/waiting-rule-policy.md`; the engine-comparison research in `docs/task-systems-comparison.md`; the production lock-contention research + the paired mutex/library benches behind it in `docs/internals/lock-contention-research.md`; the planned caller-owned-operation-state redesign of `access` (zero-alloc attended path, TODO 1.19 — read before touching `guarded.h`'s access fast path) in `docs/internals/access-op-design.md`; the original interface sketch in `design/guarded_sketch.h`. diff --git a/docs/TODO.md b/docs/TODO.md index b5e2955..0500b60 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -389,12 +389,14 @@ IDs — when an item is done, mark it, don't renumber. `ts::yield()` polling a "critical work pending" signal vs coroutine nodes (`co_await` suspension already exists for tasks) vs auto-slicing via `parallel_for` guidance. Relates to 2.4's keep-out-zone hypothesis and 2.5's rank (what "higher-rank pending" means). - **Landed (2026-09):** `ts::yield()` (task.h) runs one queued `Priority::high` entry inline - on the yielding worker's stack and returns - no suspension, so functor nodes and - `parallel_for` bodies can yield, and the continuation keeps its core; `parallel_for` - yields at every chunk claim. Rationale: design.md §3 "Yield points". Open: a pending - signal finer than the `high` class (2.5's rank), and a yield inside a resumed coroutine - segment defers any resume the nested task triggers until the segment returns (guide §13). + **Landed (2026-09):** `ts::yield()` (task.h) runs one queued entry of a higher class + inline (`normal` -> `high`; `low` -> `high`, then the global `normal` queue) on the + yielding worker's stack and returns - no suspension, so functor nodes and `parallel_for` + bodies can yield, and the continuation keeps its core; `parallel_for` yields at every + chunk claim. Rationale: design.md §3 "Yield points". A resume the nested task causes runs + at the yield point, also inside a resumed segment (the nested dispatch sets the resume + and inline trampolines aside; nesting is bounded to one level). Open: a pending signal + finer than the priority classes (2.5's rank). 12. `[ ]` **(P2, author 2026-07 — raised from the 2.3 adjacency) Frame-boundary overlap for designated nodes.** Distinct from 2.3 (whole-graph pipelining): let specific off-path tails (audio mix, streaming finalization) spill past the run's settle into the next @@ -651,6 +653,14 @@ IDs — when an item is done, mark it, don't renumber. concrete. 8. `[ ]` **(P2, author 2026-08) Cache-line alignment audit across components.** A systematic pass over every hot shared structure for cache-line placement: separate fields written by different threads onto distinct lines (`alignas(std::hardware_destructive_interference_size)` where warranted), keep fields read/written together on one line, and check array elements for false sharing between adjacent entries. Inventory to cover: `Task_control_block` (the size-ordered cluster is packing-motivated, not sharing-motivated — line 0 packs the write-hot atomics `refcount`/`num_locks` together AND with the read-only dispatch-read set `execute`/`result_ptr`/`pipe_links`/`flags`/`token`, trading one-line dispatch locality against handle-churn/indegree writes false-sharing that line; `refcount` is ~23% of per-node machinery per the graph-regression callgrind, so measure whether isolating the write-hot atomics onto their own line beats the locality win), the evolved `Pipe` (all queue state now lives under one mutex, so the interesting question shifted: is `writer_owner` — read lock-free by every `commit()` ownership check — on the right line relative to the mutex and the queue head, and does a coroutine frame's embedded link share a line with hot promise state), `Pipe_link` arrays (adjacent links of one task live on one line; different lines' traffic collides — measure before padding, links are per-task not global), scheduler queues/deques (Chase-Lev top/bottom, MPMC slots; `Busy_slot`/`Bucket_row` are already padded — verify the rest), journal slots, `Event_count`. Measure with the existing benchmarks (contention series + R10 pipe fixture) — padding trades memory for isolation, so each change needs a number, not a vibe. **Reaffirmed + rescoped (author, 2026-08): the WHOLE implementation, not just the inventory above** — treat every hot or shared structure in `include/`+`src/` as in scope, and rederive the field-level analysis from the CURRENT layout rather than the specifics written above, which are now stale: `Task_control_block` changed twice since (the coroutine-first slimming to 264 B, then B2 dropping `dispatch_arg`/collapsing `run_state`→`body_claimed` to 248 B, plus the `TS_DEBUG_NAMES`-gated `Named` field), so the `num_locks`/`refcount` line-sharing claim must be re-checked against the real struct. Dump the actual offsets (a `static_assert(offsetof(...))` probe or the debugger's layout view) as the first step, identify which fields are touched by which threads on the hot paths (settle vs dispatch vs refcount vs the harness), then place the destructive-interference boundaries — measured, per the rule above. Do this after the perf-baseline/graph-regression work settles (it may itself surface a false-sharing culprit worth folding in). + 9. `[ ]` **(P3, 2026-09) Allocation-free `Periodic::next()`.** Each `next()` allocates one + timed block: the returned task, with the timer bookkeeping embedded. A `Periodic` could own + one block and re-arm it every tick, the way `Signal::reset()` re-arms a signal, which makes a + steady-state tick allocation-free. It needs a reusable settle for an executable block (result + and state reset while no awaiter is attached), which blocks lack since executable-task reuse + was deleted with the coroutine-first transformation. A 60 Hz clock costs 60 allocations a + second today, so this waits for a workload with many clocks, or for the per-type free-list + (4.1), which would absorb the cost without an API change. 5. **Fork-join / parallel_for** 1. `[ ]` **(P2) Intra-system entity interactions** — ship the primitive menu: `parallel_gather_apply` (mailbox), `parallel_for_colored` + `Interaction_coloring`, `Accumulator` (commutative), `Union_find` helper, + triage docs. Open author questions. [§D5] diff --git a/docs/design.md b/docs/design.md index d00b0e9..944cf3c 100644 --- a/docs/design.md +++ b/docs/design.md @@ -507,21 +507,47 @@ The obvious shape suspends the yielding task, requeues it, and lets the worker take the urgent one. That pays a requeue and a resume hop per yield, lets a thief move the continuation to a cold core, and needs a coroutine. The shape taken runs the urgent task inline instead: `ts::yield()` pops one queued -`high` entry and executes it on the yielding worker's stack, then returns. The -continuation never leaves the stack, and the yield works in a plain functor -body. The common case, nothing queued, reads one relaxed counter the scheduler -keeps beside the `high` queue, incremented before a push and decremented after -a pop, so it never under-counts. +entry of a higher class than the yielder's and executes it on the yielding +worker's stack, then returns. The continuation never leaves the stack, and the +yield works in a plain functor body. `normal` yields to `high`; `low` yields to +`high` and then to the global `normal` queue; `high` never yields. A worker's +own `normal` deque stays invisible to a `low` yielder: counting its entries +would add a shared write to the deque push, the scheduler's fast path, and +idle workers steal from it anyway. The common case, nothing queued, reads two +relaxed counters the scheduler keeps for the `high` queue and the global +`normal` queue, each incremented before a push and decremented after a pop, so +neither under-counts. The pair sits alone on one cache line, which every +worker reads at every yield point. Three properties make the nesting safe. A queued entry has already taken its pipe turns, so it cannot wait on grants the yielder holds. The nested dispatch runs inside a scope that clears the thread's ambient task state (current task, -grants, scope children, rule relaxation, trace owner) and restores it +grants, scope children, rule relaxation, trace owner), sets aside the pending +work of the resume and inline-dispatch trampolines, and restores both afterwards, so the nested task starts as it would at the top of the worker -loop. And a `high` task's yield point does nothing, so nesting is one level -deep and ordering among `high` tasks stays the queue's. The trace subtracts -the nested span from the yielding body's credited time, so body time is -counted once. `parallel_for` has a yield point at every chunk claim, which +loop. And yield points reached inside the nested task do nothing, so nesting +is one level deep. + +The trampoline part is what lets a yield point release a coroutine. A resumed +coroutine segment runs inside the resume trampoline's drain, and a resume +queued during a drain runs when the drain reaches it. Without the set-aside, a +resume the nested task causes from a yield point inside a resumed segment +would wait for the yielding segment to return, so a tick driver released at a +yield point would wait for the body it was meant to interrupt. With it, the +resume starts a drain of its own at the yield point, and the outer drain +continues where it stood once the nested task returns. The one-level bound is +what keeps that finite: a coroutine resumed at a yield point could otherwise +yield in turn and run another task, one stack level per queued task. + +Running a resume at a yield point adds no interleaving the settle paths do not +already face. The awaiter handshake decides which thread resumes a coroutine, +not when, and a resume that runs while its settler is still on the stack is +the ordinary case at the top of a worker loop: `Signal::trigger` keeps its +block alive across the settle, and a graph run's last node keeps the run's +completion handle alive across it, for exactly that reason. The yielding +segment's own pending resumes are delayed by the nested task, as they would +be by any longer segment. The trace subtracts the nested span from the +yielding body's credited time, so body time is counted once. `parallel_for` has a yield point at every chunk claim, which covers the most common long body without a change to user code. #### The timer thread @@ -539,7 +565,9 @@ can wait on a high-resolution waitable timer. For a 60 Hz tick the saved hop is tens of microseconds against a 16.7 ms period, well below the jitter that granularity would add, so the dedicated thread stays. It never runs user code: a fire is delivered as a task at the sleep's priority, for the same reason -`Frame_gate::open()` releases through the scheduler. +`Frame_gate::open()` releases through the scheduler. The delivered task is the +one the sleep returned, whose block also carries the timer's bookkeeping, so a +wait is one allocation and a fire queues no second task. --- diff --git a/docs/guide.md b/docs/guide.md index b1318d0..cb2f5c0 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -1347,9 +1347,10 @@ body, deadlock-free even when every worker is occupied. Chunks inherit the caller's access grants, so a `parallel_for` inside a graph node may touch the node's declared objects. -Chunk boundaries are yield points (§10.1). When a `high` task is queued, the -thread that finishes a chunk runs it before claiming the next one, so a long -`normal` loop does not hold off urgent work for its whole duration. +Chunk boundaries are yield points (§10.1). When work of a higher class than +the loop's is queued, the thread that finishes a chunk runs one such task +before claiming the next chunk, so a long `normal` loop does not hold off +urgent work for its whole duration. Cross-item mutation, where item *i* writes item *j*, is not synchronized by `parallel_for` itself; see the WIP note in §13 and the staging tools in §9, @@ -1781,6 +1782,11 @@ context already grants the front, such as a node that declared `poses.state()`, the read is lent and takes no turn of its own. On a blue thread it takes a read turn. Inside a task that holds nothing, use `co_await ts::read_last_versions(poses)`; the blocking form is fatal there. +The view extends the running access context by one entry, the previous +version, when the read is lent, and by two otherwise. A context holds at most +`Access_context::max_entries` objects, so a node that already declares that +many, or one fewer for an unlent read, cannot take a view: the overflow is +fatal. The history costs a third replica, rotated at every publish, and it requires `Resync::copy` (the default for this history) or `Resync::overwrite`: after the @@ -1862,10 +1868,11 @@ coroutine called from a `high` node dispatches its helpers at `high`. Priority orders the queues, but it cannot evict work already running, so a long `normal` body can hold off a `high` task that became ready while every -worker was busy. `ts::yield()` is the remedy: when a `high` task is queued it -runs that task on the current thread and returns, and the yielding body then -continues on the same stack. With nothing queued it costs one relaxed load, -so it can sit in an inner loop: +worker was busy. `ts::yield()` is the remedy: when work of a higher class +than the calling task's is queued, it runs one such task on the current thread +and returns, and the yielding body then continues on the same stack. With +nothing queued it costs two relaxed loads of one cache line, so it can sit in +an inner loop: ```cpp for (Chunk& chunk : chunks) @@ -1875,11 +1882,21 @@ for (Chunk& chunk : chunks) } ``` +A `normal` task yields to queued `high` work. A `low` task yields to `high` +work and to `normal` work in the scheduler's global queue, which holds +submissions from outside the workers and overflow from their local queues. +`normal` work a worker queued on its own local queue is left to idle workers +to steal. A `high` task never yields. + +The task run at a yield point runs as a worker would run it. If its +completion resumes a coroutine, the coroutine resumes right there, also when +the yield point is itself inside a resumed coroutine. Yield points reached +inside that task do nothing, so nesting is one level deep. + It never suspends, so it is legal in any body, including a functor node and a `parallel_for` body, and grants held across it are safe: the task it runs was queued with its own turns already taken, so it cannot wait on them. It is a -no-op off a worker, in worker-less mode, and in a task already running at -`high`, and it runs only queued `high` work. Chunk boundaries of +no-op off a worker and in worker-less mode. Chunk boundaries of `parallel_for` are yield points already (§7). ### 10.2 Scheduler configuration @@ -2060,7 +2077,8 @@ int due = co_await tick.next(); One timer thread keeps the deadlines. It is created on first use, stopped by `destroy_scheduler`, and never runs user code: a wakeup is delivered as a task -on a worker. On Windows it waits on a high-resolution waitable timer, so a +on a worker. That task is the one the call returned, so a wait costs one +allocation. On Windows it waits on a high-resolution waitable timer, so a deadline is not rounded up to the system timer tick. An armed sleep counts as an external wait for the deadlock net (§5.0.3), so a program idle while it waits is not reported as deadlocked. Two constraints: worker-less mode has no @@ -2170,10 +2188,9 @@ Stated plainly; each is on the roadmap (`docs/TODO.md`): roadmap. - Timers need workers. In worker-less mode a sleep is fatal; a virtual clock the program advances, for deterministic tests, is planned. -- Yield points run queued `high` work only, and only on workers. A task run - at a yield point that resumes a coroutine hands the resume to the thread's - resume trampoline, so when the yield point is itself inside a resumed - coroutine segment, that resume waits until the segment returns. +- Yield points work only on workers. A `low` task's yield point reaches + `normal` work in the global queue but not work a worker queued on its own + local queue, which is left to stealing. --- diff --git a/docs/internals/timer-primitive-design.md b/docs/internals/timer-primitive-design.md index 4c5d9d7..bec2aae 100644 --- a/docs/internals/timer-primitive-design.md +++ b/docs/internals/timer-primitive-design.md @@ -4,8 +4,9 @@ `ts::sleep_until`, and `ts::Periodic` (§3.4's `every`, whose `next()` returns the number of grid points passed since the previous tick). Deviations from this study: on Windows the timer thread waits on a high-resolution waitable timer, because a condition-variable timeout wakes on -the ~15.6 ms system tick, a whole period of a 60 Hz clock; delivery is a `ts::launch` at the -sleep's own priority rather than always `low`; worker-less mode is fatal for now rather than +the ~15.6 ms system tick, a whole period of a 60 Hz clock; delivery submits the wait's own +task block, which embeds the timer bookkeeping (one allocation per wait), at the sleep's own +priority rather than a `ts::launch` at `low`; worker-less mode is fatal for now rather than virtual-clock driven (§4 and §5 remain the plan); `launch_after` and `Deadline` (§3.2, §3.3) are not built. The alternative of folding deadlines into the workers' park (§2.3) was reconsidered and rejected again; the reasoning is in design.md §3, "The timer thread". diff --git a/include/ts/coroutine_support.h b/include/ts/coroutine_support.h index ef89c03..f332fda 100644 --- a/include/ts/coroutine_support.h +++ b/include/ts/coroutine_support.h @@ -278,8 +278,21 @@ class Resume_queue // chain iteratively. static void push_and_drain(Resume_item item); // noinline: see the definition + // A drain's state, set aside while a task runs at a yield point and put back when it + // returns (`Nested_dispatch_scope`, scheduler.cpp). While detached, a resume starts a + // drain of its own, as at the top of a worker loop. By value and out of line, like + // `push_and_drain`, so no caller's frame holds the queue's address. + struct Drain_state + { + std::vector pending; + bool draining = false; + }; + static Drain_state detach() noexcept; // noinline: see the definition + static void reattach(Drain_state state) noexcept; // noinline: see the definition + private: inline static thread_local std::vector pending_; + inline static thread_local std::vector spare_; // a nested drain's buffer (`detach`) inline static thread_local bool draining_ = false; }; @@ -298,6 +311,29 @@ TS_DETAIL_NO_INLINE inline void Resume_queue::push_and_drain(Resume_item item) draining_ = false; } +// The outer drain's loop reads `pending_` by index after each resume returns, so the whole +// vector is set aside and put back unchanged; a drain started in between has finished by +// then, since `push_and_drain` returns only when its chain is empty. The nested drain runs on +// `spare_`, which keeps its capacity from one yield point to the next, so it allocates nothing +// in steady state. +TS_DETAIL_NO_INLINE inline Resume_queue::Drain_state Resume_queue::detach() noexcept +{ + Drain_state state{ std::exchange(pending_, std::move(spare_)), draining_ }; + pending_.clear(); // `spare_` is empty; a moved-from vector only promises valid + draining_ = false; + return state; +} + +TS_DETAIL_NO_INLINE inline void Resume_queue::reattach(Drain_state state) noexcept +{ +#if TS_SAFETY_CHECKS + if (draining_ || !pending_.empty()) + ts::fatal("Resume_queue::reattach over an unfinished drain - a nested drain did not complete"); +#endif + spare_ = std::exchange(pending_, std::move(state.pending)); + draining_ = state.draining; +} + template void resume_thunk(void* addr) { diff --git a/include/ts/detail/task_block.h b/include/ts/detail/task_block.h index 05a1a0f..2283d54 100644 --- a/include/ts/detail/task_block.h +++ b/include/ts/detail/task_block.h @@ -82,15 +82,34 @@ struct Task_control_block; // other diagnostics. [[noreturn]] void escaped_exception_diagnose(const char* what) noexcept; -// `Priority::high` entries currently queued, maintained by the scheduler: incremented before -// the push and decremented after a successful pop, so it never under-counts a queued entry. -// `ts::yield()` reads it relaxed - the whole cost of a yield point with nothing pending. -inline std::atomic high_queued{ 0 }; +// Entries queued where a yield point can take them, maintained by the scheduler: each count is +// incremented before the push and decremented after a successful pop, so it never under-counts +// a queued entry. `high` counts the `high` queue; `normal_global` counts the global `normal` +// queue (external submits and deque overflow), not the per-worker deques, whose pushes stay +// free of shared writes. Every worker reads this line at each yield point and each +// `parallel_for` chunk claim, so it holds nothing else: a neighbour written elsewhere would +// invalidate it in every reader's cache. +struct alignas(64) Yield_signal +{ + std::atomic high{ 0 }; + std::atomic normal_global{ 0 }; +}; +static_assert(sizeof(Yield_signal) == 64, "Yield_signal must own its cache line"); +inline Yield_signal yield_signal; + +// Whether anything a yield point could run is queued: the whole cost of a yield point with +// nothing pending. Relaxed - a stale answer costs one missed or one empty slow path. +inline bool yield_work_queued() noexcept +{ + return (yield_signal.high.load(std::memory_order_relaxed) + | yield_signal.normal_global.load(std::memory_order_relaxed)) != 0; +} -// The slow half of a yield point (defined in scheduler.cpp): on a worker, when the caller runs -// below `high` (`own`), pop one queued `high` entry and run it on this thread, then return. -// A no-op otherwise. -void yield_to_high(Priority own) noexcept; +// The slow half of a yield point (defined in scheduler.cpp): on a worker, run one queued task +// of a class above `own` on this thread, then return. `normal` yields to `high`; `low` yields +// to `high`, then to the global `normal` queue; `high` never yields. A no-op off a worker, and +// inside a task that is itself running at a yield point. +void yield_to_higher(Priority own) noexcept; #if TS_RULE_ON(TS_RULE_DEADLOCK_NET) // Work that only a non-worker thread can complete, currently outstanding (see @@ -429,8 +448,19 @@ struct Task_control_block public: static void push_and_drain(const Task_ptr& blk); // noinline: see the definition + // A drain's state, set aside while a task runs at a yield point and put back when it + // returns - `Resume_queue::Drain_state`'s counterpart, for inline dispatches. + struct Drain_state + { + std::vector pending; + bool draining = false; + }; + static Drain_state detach() noexcept; // noinline: see the definition + static void reattach(Drain_state state) noexcept; // noinline: see the definition + private: inline static thread_local std::vector pending_; + inline static thread_local std::vector spare_; // a nested drain's buffer (`detach`) inline static thread_local bool draining_ = false; }; @@ -628,6 +658,26 @@ TS_DETAIL_NO_INLINE inline void Task_control_block::Inline_queue::push_and_drain draining_ = false; } +// See `Resume_queue::detach`: the same contract for the inline-dispatch drain. +TS_DETAIL_NO_INLINE inline Task_control_block::Inline_queue::Drain_state +Task_control_block::Inline_queue::detach() noexcept +{ + Drain_state state{ std::exchange(pending_, std::move(spare_)), draining_ }; + pending_.clear(); // `spare_` is empty; a moved-from vector only promises valid + draining_ = false; + return state; +} + +TS_DETAIL_NO_INLINE inline void Task_control_block::Inline_queue::reattach(Drain_state state) noexcept +{ +#if TS_SAFETY_CHECKS + if (draining_ || !pending_.empty()) + ts::fatal("Inline_queue::reattach over an unfinished drain - a nested drain did not complete"); +#endif + spare_ = std::exchange(pending_, std::move(state.pending)); + draining_ = state.draining; +} + // `Task_ptr` refcount ops (block is complete here). `dec` at 0 runs the wrapper's `destroy`. inline void intrusive_inc(Task_control_block* p) noexcept { diff --git a/include/ts/parallel_for.h b/include/ts/parallel_for.h index fc62636..5332418 100644 --- a/include/ts/parallel_for.h +++ b/include/ts/parallel_for.h @@ -125,10 +125,10 @@ void run_loop(Parallel_state* st) { for (;;) { - // A yield point between chunks (`ts::yield`): a queued `high` task runs here before the - // next claim. One relaxed load when nothing is pending. - if (high_queued.load(std::memory_order_relaxed) != 0) - yield_to_high(st->priority); + // A yield point between chunks (`ts::yield`): queued work of a higher class than the + // loop's runs here before the next claim. One cache line read when nothing is pending. + if (yield_work_queued()) + yield_to_higher(st->priority); int start, stop; if (st->token.is_cancel_requested()) { @@ -199,9 +199,9 @@ void run_loop(Colored_state* st) { // A yield point between chunks, as in the flat loop; the phase is re-read afterwards, // since the band may have moved on while the nested task ran. - if (high_queued.load(std::memory_order_relaxed) != 0) + if (yield_work_queued()) { - yield_to_high(st->priority); + yield_to_higher(st->priority); cur = st->phase_next.load(std::memory_order_acquire); } int ph = static_cast(cur >> 32); diff --git a/include/ts/scheduler.h b/include/ts/scheduler.h index 3f26232..fee960e 100644 --- a/include/ts/scheduler.h +++ b/include/ts/scheduler.h @@ -138,9 +138,9 @@ namespace detail // `create_scheduler`. Returns a `unique_ptr` because `Scheduler` is non-movable. std::unique_ptr make_scheduler(Scheduler_config config = {}); -// A yield point's slow half (declared again beside `high_queued` in task_block.h, defined in -// scheduler.cpp); a friend of `Scheduler` so it can pop the `high` queue directly. -void yield_to_high(Priority own) noexcept; +// A yield point's slow half (declared again beside `yield_signal` in task_block.h, defined in +// scheduler.cpp); a friend of `Scheduler` so it can pop the queues directly. +void yield_to_higher(Priority own) noexcept; // The slot behind `ts::current_worker_index()`, written only by a worker thread's entry and // exit. Behind the thread-local barrier (ts/detail/thread_local.h) like every other @@ -160,7 +160,7 @@ class Scheduler { friend class detail::Worker_thread; friend std::unique_ptr detail::make_scheduler(Scheduler_config); - friend void detail::yield_to_high(Priority own) noexcept; + friend void detail::yield_to_higher(Priority own) noexcept; public: ~Scheduler(); diff --git a/include/ts/task.h b/include/ts/task.h index 94fb542..2305ae5 100644 --- a/include/ts/task.h +++ b/include/ts/task.h @@ -305,18 +305,22 @@ auto launch(Fn&& fn, Dispatch_options opts = {}, return detail::build_bare_task(std::forward(fn), std::move(opts), site); } -// A yield point for long-running work: if a `Priority::high` task is queued, run it now, on -// this thread, and return; otherwise return at once. With nothing pending the cost is one -// relaxed load, so it can sit in an inner loop. It never suspends, so it is legal in any body -// (a functor node, a `parallel_for` body, a coroutine segment), and grants held across it are -// safe: the task it runs was queued with its own turns already taken, so it cannot wait on -// them. The yielding work continues on the same stack afterwards. A no-op off a worker, in -// worker-less mode (nothing queues there), and in a task already running at `high`. Only -// queued `high` work is run; `normal` and `low` work never preempts through a yield point. +// A yield point for long-running work: if work of a higher class than the calling task's is +// queued, run one such task now, on this thread, and return; otherwise return at once. A +// `normal` task yields to queued `high` work; a `low` task also to `normal` work in the global +// queue (external submits and deque overflow - `normal` work in a worker's own deque is left +// to stealing); a `high` task never yields. With nothing queued the cost is two relaxed loads +// of one cache line, so it can sit in an inner loop. It never suspends, so it is legal in any +// body (a functor node, a `parallel_for` body, a coroutine segment), and grants held across it +// are safe: the task it runs was queued with its own turns already taken, so it cannot wait on +// them. That task runs as a worker would run it - a coroutine its completion resumes is resumed +// here too - and the yielding work continues on the same stack afterwards. Yield points reached +// inside it are no-ops, so nesting is one level deep. A no-op off a worker and in worker-less +// mode, where nothing queues. inline void yield() { - if (detail::high_queued.load(std::memory_order_relaxed) != 0) - detail::yield_to_high(detail::resolved_priority(std::nullopt)); + if (detail::yield_work_queued()) + detail::yield_to_higher(detail::resolved_priority(std::nullopt)); } // Declares that something the task system is waiting on will be completed by a thread the diff --git a/include/ts/timer.h b/include/ts/timer.h index 6dd8dff..5bdf1a9 100644 --- a/include/ts/timer.h +++ b/include/ts/timer.h @@ -71,6 +71,10 @@ class Periodic std::chrono::steady_clock::duration period() const noexcept { return period_; } private: + // Grid points passed since the previous tick, advancing the grid past now; 0 once the + // token is requested. The body of the task `next()` returns. + int advance() noexcept; + std::chrono::steady_clock::duration period_; std::chrono::steady_clock::time_point next_deadline_; Sleep_options opts_; diff --git a/include/ts/versioned.h b/include/ts/versioned.h index ea16d61..c39e04c 100644 --- a/include/ts/versioned.h +++ b/include/ts/versioned.h @@ -112,6 +112,9 @@ inline void check_version_read_may_block() // Non-copyable and non-movable: the view is the grant, and it installs its own access context // (like `Access_guard`), so both versions pass the harness while it lives and neither does // after. In a coroutine it counts as a live guard: `co_await` while one is alive is fatal. +// Its context is the running one plus `previous` and, when the read is not lent, the front: +// one or two more of the `Access_context::max_entries` objects a context can hold, so taking +// a view in a body whose context is at or near that cap overflows it, which is fatal. template class Version_view { diff --git a/sample/fixed_rate.cpp b/sample/fixed_rate.cpp index 85de1fb..27d3c3f 100644 --- a/sample/fixed_rate.cpp +++ b/sample/fixed_rate.cpp @@ -16,11 +16,12 @@ // every `parallel_for` chunk boundary are yield points (`ts::yield`), so a tick that comes due // while the workers are busy with frame work starts within one chunk of the frame's. // -// Checked every run: ticks run one at a time in order, the interpolated pair is always two -// consecutive ticks, the interpolation fraction stays within [0, 1], and every staged intent is -// applied exactly once. Which tick picks up a given intent depends on wall time, as it would with -// a real input device, so the full run is not bit-reproducible; the physics graph alone, driven -// by a tick-indexed intent script, is (`fixed_rate_physics_hash`). +// Checked every run: the clock kept up with the periods that passed (ticking or dropping them +// by policy), ticks run one at a time in order, the interpolated pair is always two consecutive +// ticks whose stamps agree with them, and every staged intent is applied exactly once. Which +// tick picks up a given intent depends on wall time, as it would with a real input device, so +// the full run is not bit-reproducible; the physics graph alone, driven by a tick-indexed +// intent script, is (`fixed_rate_physics_hash`). #include "ts/coroutine_support.h" #include "ts/deferred.h" @@ -191,7 +192,7 @@ struct Run_stats std::atomic intents_staged{ 0 }; std::atomic render_frames{ 0 }; std::atomic pairs_not_consecutive{ 0 }; - std::atomic fraction_out_of_range{ 0 }; + std::atomic stamps_inconsistent{ 0 }; long long ticks = 0; // written by the driver only long long wakes = 0; long long dropped = 0; @@ -270,8 +271,9 @@ ts::Static_task_graph build_frame_graph(Domains& domains, Costs costs, Run_stats const double alpha = stamps.fraction_at(Clock::now()); if (current.tick() != previous.tick() + 1) stats.pairs_not_consecutive.fetch_add(1, std::memory_order_relaxed); - if (alpha < 0.0 || alpha > 1.0) - stats.fraction_out_of_range.fetch_add(1, std::memory_order_relaxed); + // Every tick publishes exactly once, so the newest version's serial is its tick. + if (stamps.previous_published > stamps.current_published || stamps.current_serial != current.tick()) + stats.stamps_inconsistent.fetch_add(1, std::memory_order_relaxed); const float y0 = previous.at(0).y; const float y1 = current.at(0).y; render.body0_y = y0 + static_cast(alpha) * (y1 - y0); @@ -349,14 +351,23 @@ Outcome run_domains(int frames, float scale, bool print) return std::pair{ static_cast(world.tick()), world.impulses_applied() }; }).sync(); + // The clock ran: the ticks run or dropped by the overload policy cover the grid points that + // passed while the frames ran, short of those still undelivered at the end. That shortfall + // is bounded by the wake latency, which a loaded machine stretches to several periods, so + // the check asks for half - a stalled clock still fails it. + const double period_s = std::chrono::duration(tick_period).count(); + const long long grid_points = static_cast(elapsed_s / period_s); + const bool clock_ran = grid_points < 4 || stats.ticks + stats.dropped >= grid_points / 2; + Outcome outcome; outcome.frame_ms = 1000.0 * elapsed_s / frames; outcome.ticks_per_second = elapsed_s > 0.0 ? static_cast(stats.ticks) / elapsed_s : 0.0; outcome.ok = world_ticks == ticks_total && applied == stats.intents_staged.load() && stats.pairs_not_consecutive.load() == 0 - && stats.fraction_out_of_range.load() == 0 - && stats.render_frames.load() == frames; + && stats.stamps_inconsistent.load() == 0 + && stats.render_frames.load() == frames + && clock_ran; if (print) { @@ -369,9 +380,11 @@ Outcome run_domains(int frames, float scale, bool print) std::printf(" wake intervals %.2f .. %.2f ms, %lld ticks dropped by the overload policy\n", ms(stats.shortest_wake_interval), ms(stats.longest_wake_interval), stats.dropped); } - std::printf(" %lld intents staged, %lld applied; pairs consecutive: %s; fraction in [0, 1]: %s -> %s\n", - stats.intents_staged.load(), applied, stats.pairs_not_consecutive.load() == 0 ? "yes" : "no", - stats.fraction_out_of_range.load() == 0 ? "yes" : "no", outcome.ok ? "ok" : "FAILED"); + std::printf(" %lld grid points passed, %lld ticked or dropped; %lld intents staged, %lld applied; " + "pairs consecutive: %s; stamps consistent: %s -> %s\n", + grid_points, stats.ticks + stats.dropped, stats.intents_staged.load(), applied, + stats.pairs_not_consecutive.load() == 0 ? "yes" : "no", + stats.stamps_inconsistent.load() == 0 ? "yes" : "no", outcome.ok ? "ok" : "FAILED"); } return outcome; } @@ -397,7 +410,6 @@ bool fixed_rate_self_check(int frames, float scale) // count - the determinism a fixed-rate graph keeps when its input cut sequence is fixed. std::size_t fixed_rate_physics_hash(int ticks) { - Run_stats stats; Domains domains; ts::Static_task_graph physics = build_physics_graph(domains, costs_at(0.02f)); { diff --git a/sample/game_frame.cpp b/sample/game_frame.cpp index 2f4185c..03657f9 100644 --- a/sample/game_frame.cpp +++ b/sample/game_frame.cpp @@ -126,6 +126,9 @@ std::atomic streamed{ 0 }; std::atomic batches{ 0 }; std::atomic drawn{ 0 }; std::atomic hud_snapshots{ 0 }; +// Ticks the fixed-rate variant's two clocks ran in the last run (primers and teardown excluded). +std::atomic fixed_physics_ticks{ 0 }; +std::atomic fixed_network_ticks{ 0 }; // --- the stores ------------------------------------------------------------------- @@ -702,6 +705,8 @@ class Fixed_rate_ticks stop_.request_cancel(); physics_driver_.sync(); network_driver_.sync(); + fixed_physics_ticks.store(physics_ticks_); + fixed_network_ticks.store(network_ticks_); physics_.execute().sync(); network_.execute().sync(); } @@ -874,6 +879,8 @@ void reset_stats() batches.store(0); drawn.store(0); hud_snapshots.store(0); + fixed_physics_ticks.store(0); + fixed_network_ticks.store(0); } // Mock a system's CPU cost: spin-wait for the budget. Precise (unlike @@ -1735,6 +1742,14 @@ long long game_frame_draw_count() return drawn.load(); } +// Ticks the fixed-rate variant's clocks ran in the last `game_frame_fixed_stats` run, primers and +// the teardown tick excluded - how a test tells clocks that ran from clocks that did not. +void game_frame_fixed_tick_counts(long long& physics, long long& network) +{ + physics = fixed_physics_ticks.load(); + network = fixed_network_ticks.load(); +} + // Compile the frame graph and write its structure as Graphviz DOT (no frames run). void dump_game_frame_dot(const char* path) { diff --git a/src/mem_profile.cpp b/src/mem_profile.cpp index 4b9bdc8..f0740f2 100644 --- a/src/mem_profile.cpp +++ b/src/mem_profile.cpp @@ -10,6 +10,7 @@ #include "ts/coroutine_support.h" #include "ts/parallel_for.h" #include "ts/static_task_graph.h" +#include "ts/timer.h" // The game-frame sample is a single self-contained .cpp (no header). Both compositions of // the same frame are profiled: the compiled graph amortizes its per-run state, the @@ -248,6 +249,25 @@ void run_mem_profile() }); } + // A wait is one allocation: the returned task's block carries the timer's bookkeeping. + measure("sleep, passed", k, [] + { + ts::sleep_until(std::chrono::steady_clock::now()).sync(); + }); + + measure("sleep 20us", k / 8, [] + { + ts::sleep(std::chrono::microseconds(20)).sync(); + }); + + { + ts::Periodic tick{ std::chrono::microseconds(50) }; + measure("Periodic::next", k / 8, [&tick] + { + (void)tick.next().sync(); + }); + } + std::printf("\ngame frame (sample/game_frame.cpp, 1000 entities, ~30 systems):\n"); measure_frame("frame graph", &sample::game_frame_stats); measure_frame("frame graph-free", &sample::game_frame_free_stats); diff --git a/src/scheduler.cpp b/src/scheduler.cpp index 6473a75..4e2cdd9 100644 --- a/src/scheduler.cpp +++ b/src/scheduler.cpp @@ -1,4 +1,5 @@ #include "ts/scheduler.h" +#include "ts/coroutine_support.h" // the resume trampoline a yield point's nested dispatch detaches #include "ts/detail/task_block.h" // the ambient task state a yield point's nested dispatch resets #include "ts/detail/worker_thread.h" @@ -170,8 +171,11 @@ void Scheduler::submit(Task_func_ptr func, void* data, Priority priority) return; } + // The yield-point counts, before the push so they never under-count (see `Yield_signal`). if (priority == Priority::high) - detail::high_queued.fetch_add(1, std::memory_order_relaxed); // before the push: never under-counts + detail::yield_signal.high.fetch_add(1, std::memory_order_relaxed); + else if (priority == Priority::normal) + detail::yield_signal.normal_global.fetch_add(1, std::memory_order_relaxed); queues_[static_cast(priority)].push({ func, data }); signal_submit(); @@ -292,7 +296,7 @@ bool Scheduler::find_work(int worker_index, detail::Task_entry& out) if (queues_[0].pop(out)) // global high (strict) { - detail::high_queued.fetch_sub(1, std::memory_order_relaxed); + detail::yield_signal.high.fetch_sub(1, std::memory_order_relaxed); ++since_low; return true; } @@ -318,6 +322,7 @@ bool Scheduler::find_work(int worker_index, detail::Task_entry& out) since_global = 0; if (queues_[1].pop(out)) { + detail::yield_signal.normal_global.fetch_sub(1, std::memory_order_relaxed); ++since_low; return true; } @@ -330,6 +335,7 @@ bool Scheduler::find_work(int worker_index, detail::Task_entry& out) } if (queues_[1].pop(out)) // global normal (overflow + external) { + detail::yield_signal.normal_global.fetch_sub(1, std::memory_order_relaxed); ++since_low; since_global = 0; return true; @@ -475,12 +481,30 @@ namespace detail namespace { -// Clears the thread's ambient task state for a task run inside another task's yield point and -// restores it afterwards, so the nested task starts as it would at the top of the worker loop: -// no current task, no grants, no scope children, no rule relaxation, no trace owner. Its own -// dispatch installs what it needs. Under a traced run the nested span is added to -// `Nested_span_state`, which the yielding body's `Trace_busy_scope` subtracts, so the span is -// credited as body time once - by the nested task's own scope. +// Set while this thread runs a task at a yield point. A yield point reached inside that task, +// or inside a coroutine its completion resumes here, is then a no-op, so nesting stays one +// level deep however many tasks are queued. Touched only from this translation unit, which +// defines no coroutine, so no frame can hold its address. +struct Yield_nesting : Tls_scalar {}; + +// Makes a task run at a yield point start as it would at the top of the worker loop, and +// restores the yielding code's state when it returns. +// - Cleared: the current task, grants, scope children, rule relaxation and trace owner. The +// nested task's own dispatch installs what it needs. +// - Detached: the resume and inline-dispatch trampolines. The yielding code may be running +// inside a drain of either (a resumed coroutine segment, an inline-dispatched node), and a +// resume or inline dispatch the nested task causes would then queue behind the yielding +// code and run only after it returns - a tick released at a yield point would wait for the +// very body it was meant to interrupt. Detached, it starts its own drain here, nested, as +// at the top of the loop, and the outer drain carries on from where it was once the scope +// ends. The destroy trampoline stays attached: deferring a free is harmless. +// - Bounded: `Yield_nesting` makes yield points inside the nested task no-ops. Without it a +// coroutine resumed here could yield, run another task, resume another coroutine, and so +// on, one stack level per queued task; the deferral the detach removes used to be what +// stopped that. +// Under a traced run the nested span is added to `Nested_span_state`, which the yielding +// body's `Trace_busy_scope` subtracts, so the span is credited as body time once - by the +// nested task's own scope. class Nested_dispatch_scope { public: @@ -488,7 +512,10 @@ class Nested_dispatch_scope : task_(Current_task::exchange(Task_ptr{})) , access_(access_load()) , scope_children_(Scope_children::exchange(nullptr)) + , resume_drain_(Resume_queue::detach()) + , inline_drain_(Task_control_block::Inline_queue::detach()) { + Yield_nesting::store(true); access_store(nullptr); #if TS_RULES_ANY relaxed_ = relaxed_load(); @@ -513,9 +540,12 @@ class Nested_dispatch_scope #if TS_RULES_ANY relaxed_store(relaxed_); #endif + Task_control_block::Inline_queue::reattach(std::move(inline_drain_)); + Resume_queue::reattach(std::move(resume_drain_)); Scope_children::store(scope_children_); access_store(access_); (void)Current_task::exchange(std::move(task_)); + Yield_nesting::store(false); } Nested_dispatch_scope(const Nested_dispatch_scope&) = delete; @@ -525,6 +555,8 @@ class Nested_dispatch_scope Task_ptr task_; const Access_context* access_; std::vector* scope_children_; + Resume_queue::Drain_state resume_drain_; + Task_control_block::Inline_queue::Drain_state inline_drain_; #if TS_RULES_ANY unsigned relaxed_ = 0; #endif @@ -537,19 +569,29 @@ class Nested_dispatch_scope } // namespace -// The entry runs exactly as a worker would run it, minus the busy timing: the yielding task's -// `run_task` span already covers this thread. One entry per call, so a yield point's latency -// is bounded by one task. A task it runs that settles a coroutine hands the resume to this -// thread's resume trampoline; if the yield point is itself inside a resumed segment, that -// resume runs once the yielding segment returns to the trampoline. -void yield_to_high(Priority own) noexcept +// One entry per call, run exactly as a worker would run it, minus the busy timing: the +// yielding task's `run_task` span already covers this thread. The queues are tried in +// `find_work`'s order among the classes the caller may yield to: `high` first, then, for a +// `low` caller, the global `normal` queue. An entry taken by another worker since the counts +// were read leaves nothing to run, and the call returns. +void yield_to_higher(Priority own) noexcept { - if (own == Priority::high || current_worker_index() < 0) + if (own == Priority::high || Yield_nesting::load() || current_worker_index() < 0) return; + Scheduler& scheduler = global_scheduler(); Task_entry task; - if (!global_scheduler().queues_[static_cast(Priority::high)].pop(task)) - return; // taken by another worker since the counter was read - high_queued.fetch_sub(1, std::memory_order_relaxed); + if (scheduler.queues_[static_cast(Priority::high)].pop(task)) + { + yield_signal.high.fetch_sub(1, std::memory_order_relaxed); + } + else if (own == Priority::low && scheduler.queues_[static_cast(Priority::normal)].pop(task)) + { + yield_signal.normal_global.fetch_sub(1, std::memory_order_relaxed); + } + else + { + return; + } Nested_dispatch_scope scope; task.func_(task.data_); } diff --git a/src/timer.cpp b/src/timer.cpp index f274f21..4fa5c95 100644 --- a/src/timer.cpp +++ b/src/timer.cpp @@ -1,14 +1,13 @@ #include "ts/timer.h" -#include "ts/coroutine_support.h" #include "ts/fatal.h" -#include "ts/guarded.h" // global_scheduler +#include "ts/scheduler.h" // global_scheduler #include #include #include #include -#include #include +#include #include #include #include @@ -35,24 +34,80 @@ namespace using Clock = std::chrono::steady_clock; -// One armed wait. Owned by its heap entry; the cancel callback it carries captures a raw -// pointer to it, so the callback's lifetime is nested inside the state's. -struct Sleep_state : detail::Ref_counted +// The timer's bookkeeping, a second base of every timed block. The heap and the cancel +// callback reach the block through it; the block's own refcount keeps it alive. +struct Timed_fields { - detail::Task_ptr block; // the returned task's block - Priority priority = Priority::normal; // the delivery task's priority - bool live = true; // neither fired nor cancelled; guarded by the timer mutex - // Held while live: the wakeup comes from a thread the scheduler does not run, so the - // deadlock net must not read a quiescent pool as a deadlock (`External_wait`). - std::unique_ptr outstanding; - std::optional on_cancel; + detail::Task_control_block* timer_block = nullptr; // the block this is a base of + bool timer_live = false; // armed, neither delivered nor cancelled; guarded by the timer mutex + bool timer_delivered = false; // handed to the scheduler, or run in the call; its body will run + // Declared last, so destroyed first: its destructor waits out a callback firing on another + // thread, and that callback reads the fields above. + std::optional timer_on_cancel; }; +// One allocation per wait: the task the caller holds, its body and result, and the timer's +// bookkeeping. Delivery submits this block itself, so a wakeup costs no second task. +template +struct Timed_executable : detail::Executable, Timed_fields +{ + explicit Timed_executable(Body body) + : detail::Executable(std::move(body)) + { + timer_block = this; + } + + // A wait dropped by `timer_shutdown` is never delivered, so its body never ran; `Executable` + // expects `run` to have destroyed it. + ~Timed_executable() + { + if (!timer_delivered) + this->destroy_body(); + } +}; + +template +detail::Task_ptr make_timed(Body body, Cancellation_token token, Priority priority, Named name, Timed_fields*& fields) +{ + using Exec = detail::Executable; + using Wrapper = Timed_executable; + auto* timed = new Wrapper(std::move(body)); + timed->destroy = [](detail::Task_control_block* c) { delete static_cast(c); }; + timed->execute = &Exec::run; + timed->token = std::move(token); + timed->flags.priority = priority; + fields = timed; + detail::Task_ptr block(timed); + detail::set_task_name(block, name); + return block; +} + +// An armed wait counts as an `External_wait` from arm to delivery - its wakeup comes from a +// thread the scheduler does not run - without storing one. +void external_wait_add([[maybe_unused]] int delta) noexcept +{ +#if TS_RULE_ON(TS_RULE_DEADLOCK_NET) + detail::outstanding_external_waits.fetch_add(delta, std::memory_order_acq_rel); +#endif +} + +// Hand a due or cancelled block to the scheduler, never run it here: running it settles the +// task, and settling resumes awaiting coroutines on the settling thread. `Executable::run` +// settles a block whose token was requested as cancelled, so a fire and a sleep's cancel share +// this path. The external-wait registration is released once the block is queued, so the +// deadlock net never sees a window with neither. +void deliver(detail::Task_ptr block) noexcept +{ + detail::submit_ready(std::move(block)); + external_wait_add(-1); +} + struct Entry { Clock::time_point deadline; std::uint64_t serial; // FIFO among equal deadlines - detail::Ref_ptr state; + detail::Task_ptr block; // the heap's reference + Timed_fields* fields; }; // Heap order for `std::push_heap`/`pop_heap`: the earliest deadline at the front. @@ -64,28 +119,7 @@ struct Later } }; -// A wakeup to deliver once the timer mutex is released. -struct Wakeup -{ - detail::Task_ptr block; - Priority priority; - std::unique_ptr outstanding; // released only after the delivery is queued -}; - -// Settle `block` from a task at `priority`, never inline: settling resumes awaiting coroutines -// on the settling thread, and neither the timer thread nor a cancelling thread may run them. -void deliver(Wakeup wakeup, bool cancelled) -{ - (void)ts::launch([block = std::move(wakeup.block), cancelled]() mutable - { - if (cancelled) - block->cancel(); - else - block->complete(); - }, { .priority = wakeup.priority, .name = "ts::sleep wakeup" }); -} - -// The deadline keeper: a min-heap of armed sleeps and the one thread that waits for its head. +// The deadline keeper: a min-heap of armed waits and the one thread that waits for its head. class Timer_service { public: @@ -101,8 +135,8 @@ class Timer_service #endif } - // Arm `state` for `deadline`, starting the thread if it is not running. - void arm(Clock::time_point deadline, detail::Ref_ptr state) + // Arm `block` for `deadline`, starting the thread if it is not running. + void arm(Clock::time_point deadline, detail::Task_ptr block, Timed_fields* fields) { bool earlier; { @@ -112,35 +146,37 @@ class Timer_service stopping_ = false; thread_ = std::thread([this] { run(); }); } - state->outstanding = std::make_unique(); + fields->timer_live = true; ++live_; + external_wait_add(1); earlier = heap_.empty() || deadline < heap_.front().deadline; - heap_.push_back(Entry{ deadline, next_serial_++, std::move(state) }); + heap_.push_back(Entry{ deadline, next_serial_++, std::move(block), fields }); std::push_heap(heap_.begin(), heap_.end(), Later{}); } if (earlier) wake(); } - // The cancel callback's body: settle the wait cancelled now. The entry stays in the heap - // until its deadline and is dropped there. - void cancel(Sleep_state* state) + // The cancel callback's body: deliver the wait now. Its entry stays in the heap until the + // deadline and is dropped there. + void cancel(Timed_fields* fields) { - Wakeup wakeup; + detail::Task_ptr block; { std::scoped_lock lock(mutex_); - if (!state->live) + if (!fields->timer_live) return; - state->live = false; + fields->timer_live = false; + fields->timer_delivered = true; --live_; - wakeup = Wakeup{ state->block, state->priority, std::move(state->outstanding) }; + block = detail::Task_ptr(fields->timer_block); // safe: live means the heap still holds a reference } - deliver(std::move(wakeup), true); + deliver(std::move(block)); } void shutdown(bool check_armed) noexcept { - std::vector dropped; // destroyed after the lock: a state's cancel callback takes it + std::vector dropped; // released after the lock: a block's cancel callback takes it { std::scoped_lock lock(mutex_); if (!thread_.joinable()) @@ -156,7 +192,13 @@ class Timer_service #endif stopping_ = true; for (Entry& e : heap_) - e.state->live = false; + { + if (e.fields->timer_live) + { + e.fields->timer_live = false; + external_wait_add(-1); + } + } dropped = std::move(heap_); heap_.clear(); live_ = 0; @@ -169,7 +211,7 @@ class Timer_service void run() { std::vector due; - std::vector fire; + std::vector fire; std::unique_lock lock(mutex_); while (!stopping_) { @@ -193,18 +235,19 @@ class Timer_service } for (Entry& e : due) { - if (e.state->live) + if (e.fields->timer_live) { - e.state->live = false; + e.fields->timer_live = false; + e.fields->timer_delivered = true; --live_; - fire.push_back(Wakeup{ e.state->block, e.state->priority, std::move(e.state->outstanding) }); + fire.push_back(std::move(e.block)); } } lock.unlock(); - for (Wakeup& wakeup : fire) - deliver(std::move(wakeup), false); + for (detail::Task_ptr& block : fire) + deliver(std::move(block)); fire.clear(); - due.clear(); + due.clear(); // the heap's references to cancelled waits, released outside the lock lock.lock(); } } @@ -287,37 +330,44 @@ Timer_service& timer_service() return *fresh; } -} // namespace - -Task sleep_until(Clock::time_point deadline, Sleep_options opts, std::source_location site) +void require_workers() { if (global_scheduler().single_threaded()) { ts::fatal("ts::sleep in worker-less mode - there is no worker to deliver the wakeup on, and the " "timer thread must not run the waiting task itself"); } - detail::Task_ptr block = detail::make_bare_block(); - detail::set_task_name(block, Named(site)); +} + +// Arm `block` for `deadline` and register the cancel callback on `token`. Registered after +// arming, so a token requested in between is caught by the callback, which then runs in its +// constructor. +void arm_with_cancel(Clock::time_point deadline, detail::Task_ptr block, Timed_fields* fields, + const Cancellation_token& token) +{ + Timer_service& service = timer_service(); + service.arm(deadline, std::move(block), fields); + fields->timer_on_cancel.emplace(token, [&service, fields] { service.cancel(fields); }); +} + +} // namespace + +Task sleep_until(Clock::time_point deadline, Sleep_options opts, std::source_location site) +{ + require_workers(); + Timed_fields* fields = nullptr; + // The block carries the caller's token, so `Executable::run` settles it cancelled when the + // token was requested - on delivery, or here in the call. + const Priority priority = detail::resolved_priority(opts.priority); + detail::Task_ptr block = make_timed([] {}, opts.token, priority, Named(site), fields); Task result(block); - if (opts.token.is_cancel_requested()) - { - block->cancel(); - return result; - } - if (deadline <= Clock::now()) + if (opts.token.is_cancel_requested() || deadline <= Clock::now()) { - block->complete(); + fields->timer_delivered = true; + block->execute(block); // settles in the call return result; } - - detail::Ref_ptr state = detail::make_ref(); - state->block = block; - state->priority = detail::resolved_priority(opts.priority); - Timer_service& service = timer_service(); - service.arm(deadline, state); - // Registered after arming, so a token requested in between settles through `cancel`. A - // token already requested runs the callback here, in the constructor. - state->on_cancel.emplace(opts.token, [&service, raw = state.get()] { service.cancel(raw); }); + arm_with_cancel(deadline, std::move(block), fields, opts.token); return result; } @@ -338,19 +388,31 @@ Periodic::Periodic(Clock::duration period, Sleep_options opts, std::source_locat Task Periodic::next() { - if (opts_.token.is_cancel_requested()) - co_return 0; - Clock::time_point now = Clock::now(); - if (now < next_deadline_) + require_workers(); + Timed_fields* fields = nullptr; + // An empty token on the block: a cancelled tick settles completed with 0 (`advance`), + // never cancelled - awaiting a cancelled value task is fatal. + const Priority priority = detail::resolved_priority(opts_.priority); + detail::Task_ptr block = make_timed([this] { return advance(); }, {}, priority, Named(site_), fields); + Task result(block); + if (opts_.token.is_cancel_requested() || Clock::now() >= next_deadline_) { - co_await sleep_until(next_deadline_, opts_, site_); - if (opts_.token.is_cancel_requested()) - co_return 0; - now = Clock::now(); + fields->timer_delivered = true; + block->execute(block); // settles in the call + return result; } + arm_with_cancel(next_deadline_, std::move(block), fields, opts_.token); + return result; +} + +int Periodic::advance() noexcept +{ + if (opts_.token.is_cancel_requested()) + return 0; + const Clock::time_point now = Clock::now(); const int due = 1 + static_cast((now - next_deadline_) / period_); next_deadline_ += period_ * due; - co_return due; + return due; } void Periodic::reset() diff --git a/tests/graph_tests.cpp b/tests/graph_tests.cpp index 86c224c..303944e 100644 --- a/tests/graph_tests.cpp +++ b/tests/graph_tests.cpp @@ -1405,6 +1405,44 @@ void test_death_nested_run_mode_conflict() { TS_CHECK(ts::test::expect_death("gr void test_death_nested_run_unquiet_scope() { TS_CHECK(ts::test::expect_death("graph_lend_unquiet_scope")); } void test_death_execute_in_flight() { TS_CHECK(ts::test::expect_death("graph_execute_in_flight")); } +// A yield point inside an inline-dispatched node runs inside the inline trampoline's drain. A +// task run at that yield point that makes another inline node ready runs that node there, not +// after the yielding node returns. +void test_graph_inline_node_released_at_yield_point() +{ + ts::Scheduler_scope pool{ { .num_workers = 1 } }; + ts::Guarded x{ ts::Named{"x"}, 0 }; + ts::Guarded y{ ts::Named{"y"}, 0 }; + std::atomic yielding{ false }; + std::atomic released_ran{ false }; + bool ran_inside = false; + + ts::Static_task_graph outer; + outer.add_node(ts::Named{"opener"}, [](int& v) { v = 1; }, x); + outer.add_node(ts::Named{"yielder"}, [&](const int&) + { + yielding.store(true); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (!released_ran.load() && std::chrono::steady_clock::now() < deadline) + ts::yield(); + ran_inside = released_ran.load(); + }, x).set_inline(); // released by `opener`'s completion on the worker, inside a drain + outer.compile(); + + ts::Static_task_graph urgent; + urgent.add_node(ts::Named{"trigger"}, [](int& v) { v = 1; }, y).set_priority(ts::Priority::high); + urgent.add_node(ts::Named{"released"}, [&](const int&) { released_ran.store(true); }, y).set_inline(); + urgent.compile(); + + ts::Task outer_run = outer.execute(); + while (!yielding.load()) + std::this_thread::yield(); + ts::Task urgent_run = urgent.execute(); + outer_run.sync(); + urgent_run.sync(); + TS_CHECK(ran_inside); +} + // The graph's default priority applies to every node without its own, including nodes added // after it was set; a node's own `set_priority` wins. void test_graph_default_priority() @@ -1497,4 +1535,5 @@ void run_graph_tests() run_if(with_rule_in_task_sync, "TS_RULE_IN_TASK_SYNC off", "death: sync own object (sharp diagnostic)", test_death_sync_own_object); run("lifetime registration balance", test_lifetime_registration_balance); run("default priority", test_graph_default_priority); + run("an inline node released at a yield point runs there", test_graph_inline_node_released_at_yield_point); } diff --git a/tests/integration_tests.cpp b/tests/integration_tests.cpp index 12bd755..873cb32 100644 --- a/tests/integration_tests.cpp +++ b/tests/integration_tests.cpp @@ -18,6 +18,7 @@ void game_frame_free_stats(int frames, float time_scale, long long game_frame_draw_count(); void game_frame_fixed_stats(int frames, float time_scale, double& avg_ms, double& serial_ms, float& transform0); +void game_frame_fixed_tick_counts(long long& physics, long long& network); bool fixed_rate_self_check(int frames, float scale); std::size_t fixed_rate_physics_hash(int ticks); } @@ -788,17 +789,22 @@ void test_fixed_rate_beside_frame() } // The game frame with physics and networking on their own clocks publishes the same transforms -// and submits the same draw commands as the baseline frame. +// and submits the same draw commands as the baseline frame. Enough frames for both clocks to +// tick - 100 ms or more even on a wide machine - so this checks the clocks, not only the primers. void test_engine_fixed_rate() { double avg_ms = 0.0, serial_ms = 0.0; float graph_xf = 0.0f, fixed_xf = 0.0f; - sample::game_frame_stats(5, 0.3f, avg_ms, serial_ms, graph_xf); + sample::game_frame_stats(60, 0.3f, avg_ms, serial_ms, graph_xf); long long graph_drawn = sample::game_frame_draw_count(); - sample::game_frame_fixed_stats(5, 0.3f, avg_ms, serial_ms, fixed_xf); + sample::game_frame_fixed_stats(60, 0.3f, avg_ms, serial_ms, fixed_xf); long long fixed_drawn = sample::game_frame_draw_count(); + long long physics_ticks = 0, network_ticks = 0; + sample::game_frame_fixed_tick_counts(physics_ticks, network_ticks); TS_CHECK(fixed_xf == 5.0f); TS_CHECK(fixed_drawn == graph_drawn); + TS_CHECK(physics_ticks >= 3); + TS_CHECK(network_ticks >= 1); } // Given the same intent sequence the fixed-rate world is the same, whatever the worker count. diff --git a/tests/scheduler_tests.cpp b/tests/scheduler_tests.cpp index 7029912..5b6c9b0 100644 --- a/tests/scheduler_tests.cpp +++ b/tests/scheduler_tests.cpp @@ -1,5 +1,6 @@ #include "scheduler_scope.h" #include "scheduler_tests.h" +#include "ts/coroutine_support.h" #include "ts/scheduler.h" #include "ts/task.h" #include "harness.h" @@ -426,6 +427,135 @@ static void test_yield_high_does_not_yield() TS_CHECK(!second_ran_inside); } +// A low task's yield point runs normal work from the global queue: the class above its own. +static void test_yield_low_to_normal() +{ + ts::Scheduler_scope pool{ { .num_workers = 1 } }; + std::atomic started{ false }; + std::atomic normal_ran{ false }; + bool normal_ran_inside = false; + ts::Task low = ts::launch([&] + { + started.store(true); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (!normal_ran.load() && std::chrono::steady_clock::now() < deadline) + ts::yield(); + normal_ran_inside = normal_ran.load(); + }, { .priority = ts::Priority::low }); + while (!started.load()) + std::this_thread::yield(); + ts::Task normal = ts::launch([&] { normal_ran.store(true); }); + low.sync(); + normal.sync(); + TS_CHECK(normal_ran_inside); +} + +// A normal task's yield point does not run another normal task. +static void test_yield_normal_not_to_normal() +{ + ts::Scheduler_scope pool{ { .num_workers = 1 } }; + std::atomic started{ false }; + std::atomic second_ran{ false }; + bool second_ran_inside = true; + ts::Task first = ts::launch([&] + { + started.store(true); + const auto until = std::chrono::steady_clock::now() + std::chrono::milliseconds(50); + while (std::chrono::steady_clock::now() < until) + ts::yield(); + second_ran_inside = second_ran.load(); + }); + while (!started.load()) + std::this_thread::yield(); + ts::Task second = ts::launch([&] { second_ran.store(true); }); + first.sync(); + second.sync(); + TS_CHECK(!second_ran_inside); + TS_CHECK(second_ran.load()); +} + +static ts::Task await_then_flag(ts::Signal gate, std::atomic& flag) +{ + co_await gate; + flag.store(true); +} + +static ts::Task resumed_segment_that_yields(ts::Signal start, std::atomic& started, + std::atomic& other_ran, bool& saw_other_inside) +{ + co_await start; // resumed on the worker, inside that thread's resume drain + started.store(true); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (!other_ran.load() && std::chrono::steady_clock::now() < deadline) + ts::yield(); + saw_other_inside = other_ran.load(); +} + +// A coroutine resumed by a task that ran at a yield point resumes there, inside the yield - +// including when the yield point is itself inside a resumed segment, where the resume would +// otherwise queue on the thread's resume trampoline behind the yielding segment. +static void test_yield_resume_runs_at_yield_point() +{ + ts::Scheduler_scope pool{ { .num_workers = 1 } }; + ts::Signal gate; + ts::Signal start; + std::atomic started{ false }; + std::atomic other_ran{ false }; + bool saw_other_inside = false; + ts::Task other = await_then_flag(gate, other_ran); // suspends on `gate` + ts::Task segment = resumed_segment_that_yields(start, started, other_ran, saw_other_inside); + ts::Task opener = ts::launch([start]() mutable { start.trigger(); }); // resumes `segment` on the worker + while (!started.load()) + std::this_thread::yield(); + ts::Task releaser = ts::launch([gate]() mutable { gate.trigger(); }, { .priority = ts::Priority::high }); + segment.sync(); + other.sync(); + opener.sync(); + releaser.sync(); + TS_CHECK(saw_other_inside); +} + +static ts::Task resumed_then_yields(ts::Signal gate, std::atomic& second_ran, bool& second_ran_inside, + std::atomic& done) +{ + co_await gate; // resumed inside the task a yield point runs + const auto until = std::chrono::steady_clock::now() + std::chrono::milliseconds(50); + while (std::chrono::steady_clock::now() < until) + ts::yield(); // one level deep already: a no-op + second_ran_inside = second_ran.load(); + done.store(true); +} + +// A task run at a yield point does not yield in turn: a coroutine its completion resumes runs +// straight through its own yield points, so nesting stays one level deep. +static void test_yield_does_not_nest() +{ + ts::Scheduler_scope pool{ { .num_workers = 1 } }; + ts::Signal gate; + std::atomic started{ false }; + std::atomic second_ran{ false }; + std::atomic done{ false }; + bool second_ran_inside = true; + ts::Task resumed = resumed_then_yields(gate, second_ran, second_ran_inside, done); + ts::Task yielder = ts::launch([&] + { + started.store(true); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (!done.load() && std::chrono::steady_clock::now() < deadline) + ts::yield(); + }); + while (!started.load()) + std::this_thread::yield(); + ts::Task first = ts::launch([gate]() mutable { gate.trigger(); }, { .priority = ts::Priority::high }); + ts::Task second = ts::launch([&] { second_ran.store(true); }, { .priority = ts::Priority::high }); + yielder.sync(); + resumed.sync(); + first.sync(); + second.sync(); + TS_CHECK(!second_ran_inside); + TS_CHECK(second_ran.load()); +} + // Off a worker a yield point is a no-op: the blue thread does not run queued work. static void test_yield_off_worker_is_noop() { @@ -479,4 +609,8 @@ void run_scheduler_tests() run("yield: with nothing pending it is cheap", test_yield_without_pending_is_cheap); run("yield: a high task does not yield to high", test_yield_high_does_not_yield); run("yield: off a worker it is a no-op", test_yield_off_worker_is_noop); + run("yield: a low task yields to queued normal work", test_yield_low_to_normal); + run("yield: a normal task does not yield to normal", test_yield_normal_not_to_normal); + run("yield: a resume caused at a yield point runs there", test_yield_resume_runs_at_yield_point); + run("yield: a task run at a yield point does not yield further", test_yield_does_not_nest); } diff --git a/tsan/tsan_main.cpp b/tsan/tsan_main.cpp index 8106a66..7ef8406 100644 --- a/tsan/tsan_main.cpp +++ b/tsan/tsan_main.cpp @@ -1061,6 +1061,57 @@ void stress_physics() (void)a; (void)b; } +ts::Task await_count_and_yield(ts::Signal gate, std::atomic& resumed) +{ + co_await gate; // resumed inside a task a yield point runs + resumed.fetch_add(1); + for (int i = 0; i < 100; ++i) + ts::yield(); // nested one level already: no-ops +} + +// Yield points under concurrency: normal tasks spinning on yield points while high tasks +// trigger signals that resume coroutines inside those yields - the nested dispatch, its +// trampoline detach and reattach, and the one-level bound, raced across four workers. +void stress_yield_resumes() +{ + ts::Scheduler_scope pool{ { .num_workers = 4 } }; + for (int round = 0; round < 50; ++round) + { + constexpr int n = 16; + std::vector gates(n); + std::atomic resumed{ 0 }; + std::atomic remaining{ n }; + std::vector> waiters; + std::vector> yielders; + std::vector> triggers; + for (int i = 0; i < n; ++i) + waiters.push_back(await_count_and_yield(gates[static_cast(i)], resumed)); + for (int i = 0; i < 4; ++i) + { + yielders.push_back(ts::launch([&remaining] + { + while (remaining.load() > 0) + ts::yield(); + })); + } + for (int i = 0; i < n; ++i) + { + triggers.push_back(ts::launch([gate = gates[static_cast(i)], &remaining]() mutable + { + gate.trigger(); + remaining.fetch_sub(1); + }, { .priority = ts::Priority::high })); + } + for (ts::Task& t : triggers) + t.sync(); + for (ts::Task& t : yielders) + t.sync(); + for (ts::Task& t : waiters) + t.sync(); + assert(resumed.load() == n); + } +} + // A fixed-rate graph on its own clock (`ts::Periodic`) beside a frame graph: the timer thread's // wakeups, yield points inside the frame's bodies, and the `Versioned` pair read under // concurrency. Then the physics graph alone, run-to-run determinism. @@ -1124,6 +1175,7 @@ int main() std::puts("tsan: versioned stress"); stress_versioned(); std::puts("tsan: physics frames"); stress_physics(); std::puts("tsan: fixed-rate graph"); stress_fixed_rate(); + std::puts("tsan: yield resumes"); stress_yield_resumes(); std::puts("tsan: blackboard frames"); sample::run_blackboard_sample(); std::puts("tsan: coloring frames"); sample::stress_coloring(10); std::puts("tsan: game_frame frames");