From f34dc7f7e0b77944fd59b1de0d833f0909611bbe Mon Sep 17 00:00:00 2001 From: aethernet-io Date: Wed, 2 Sep 2026 16:02:29 -0700 Subject: [PATCH 01/11] Add per-server local presence windows with prefix retries. Confirm receive schedules only after Pong, keep ONLINE until the last confirmed window closes, and retry at 1.5R/0.5R plus a 30ms guard. Co-authored-by: Cursor --- aether/client.cpp | 7 + aether/client.h | 2 + aether/client_connectivity_policy.cpp | 165 +++- aether/client_connectivity_policy.h | 64 +- .../local_presence_schedule.h | 157 +++ .../cloud_connections/ping_cloud_servers.cpp | 366 +++++-- aether/cloud_connections/ping_cloud_servers.h | 43 +- aether/types/statistic_counter.h | 29 +- tests/CMakeLists.txt | 2 + tests/test-local-presence/CMakeLists.txt | 29 + tests/test-local-presence/main.cpp | 900 ++++++++++++++++++ 11 files changed, 1665 insertions(+), 99 deletions(-) create mode 100644 aether/cloud_connections/local_presence_schedule.h create mode 100644 tests/test-local-presence/CMakeLists.txt create mode 100644 tests/test-local-presence/main.cpp diff --git a/aether/client.cpp b/aether/client.cpp index cd31717d..a4a05469 100644 --- a/aether/client.cpp +++ b/aether/client.cpp @@ -92,6 +92,13 @@ ClientConnectivityPolicy::ptr const& Client::connectivity_policy() { return connectivity_policy_; } +bool Client::IsLocallyOnline() const { + if (!connectivity_policy_.is_valid()) { + return false; + } + return connectivity_policy_.Load()->IsLocallyOnline(); +} + P2pMessageStreamManager& Client::message_stream_manager() { if (!message_stream_manager_) { message_stream_manager_ = std::make_unique( diff --git a/aether/client.h b/aether/client.h index fe8206c5..a82c298c 100644 --- a/aether/client.h +++ b/aether/client.h @@ -63,6 +63,8 @@ class Client : public Obj { ServerConnectionManager& server_connection_manager(); CloudServerConnections& cloud_connection(); ClientConnectivityPolicy::ptr const& connectivity_policy(); + // Read-only aggregate Local ONLINE (no side effects). + bool IsLocallyOnline() const; P2pMessageStreamManager& message_stream_manager(); void SetConfig(std::string client_id, Uid parent_uid, Uid uid, diff --git a/aether/client_connectivity_policy.cpp b/aether/client_connectivity_policy.cpp index 7b0ce875..69e33c7d 100644 --- a/aether/client_connectivity_policy.cpp +++ b/aether/client_connectivity_policy.cpp @@ -17,6 +17,7 @@ #include "aether/client_connectivity_policy.h" #include +#include namespace ae { @@ -25,7 +26,6 @@ constexpr auto kDefaultTiming = RxTiming{ .conf = RxTimingConf::Every(std::chrono::milliseconds{AE_PING_INTERVAL_MS}), .next_rx_point = {}, .recordet_at = {}}; -; std::array MakeDefaultRxTimings() { std::array timings{}; @@ -45,6 +45,7 @@ ClientConnectivityPolicy::RxTimingConfig::ForAllPriorities(RxTimingConf conf) { for (auto& item : policy_->rx_timings_) { item.conf = conf; } + policy_->ApplyDesiredToBoundServers(conf); return *this; } @@ -92,6 +93,32 @@ auto ClientConnectivityPolicy::ConfigureRxTimings( return RxTimingConfig{*this, std::move(targets)}; } +void ClientConnectivityPolicy::ConfigureServerRxTiming( + ServerId server_id, RxTimingConf conf, + std::uint8_t rtt_reliability_percentile) { + auto& state = EnsureServerPresence(server_id); + auto const timing_changed = (state.desired.interval != conf.interval) || + (state.desired.rx_window != conf.rx_window); + state.desired = conf; + state.has_user_rx_timing = true; + state.rtt_reliability_percentile = + rtt_reliability_percentile == 0 ? kDefaultRttReliabilityPercentile + : rtt_reliability_percentile; + if (state.rtt_reliability_percentile > 100) { + state.rtt_reliability_percentile = 100; + } + // Confirmed schedule stays old until a Pong for a Ping carrying the new conf. + if (timing_changed) { + state.config_change_pending = true; + } + server_rx_timing_changed_event_.Emit(server_id); +} + +void ClientConnectivityPolicy::SetServerSelectedForAggregate(ServerId server_id, + bool selected) { + EnsureServerPresence(server_id).selected_for_aggregate = selected; +} + ClientConnectivityPolicy::SuspendBlocker ClientConnectivityPolicy::AcquireSuspendBlock() { return SuspendBlocker{*this}; @@ -105,6 +132,14 @@ ConnectivityStatus ClientConnectivityPolicy::GetStatus() const noexcept { next_service_time, (t.recordet_at > current_time) ? current_time : t.next_rx_point); } + for (auto const& [id, state] : server_presence_) { + static_cast(id); + if (state.has_confirmed_schedule && + state.confirmed_window_open_local != TimePoint{}) { + next_service_time = + std::min(next_service_time, state.confirmed_window_open_local); + } + } return ConnectivityStatus{.can_suspend = can_suspend_, .suspend_block_count = suspend_block_count_, .next_service_time = next_service_time}; @@ -126,15 +161,141 @@ void ClientConnectivityPolicy::ReportNextServiceTime( t.recordet_at = Now(); } +ServerPresenceState& ClientConnectivityPolicy::EnsureServerPresence( + ServerId server_id) { + auto it = server_presence_.find(server_id); + if (it == server_presence_.end()) { + ServerPresenceState state{}; + // Seed desired from priority-0 default / first priority slot. + state.desired = rx_timings_.front().conf; + it = server_presence_.emplace(server_id, state).first; + } + return it->second; +} + +ServerPresenceState const* ClientConnectivityPolicy::FindServerPresence( + ServerId server_id) const noexcept { + auto it = server_presence_.find(server_id); + return it == server_presence_.end() ? nullptr : &it->second; +} + +ServerPresenceState* ClientConnectivityPolicy::FindServerPresence( + ServerId server_id) noexcept { + auto it = server_presence_.find(server_id); + return it == server_presence_.end() ? nullptr : &it->second; +} + +void ClientConnectivityPolicy::ConfirmServerPong(ServerId server_id, + TimePoint send_time, + TimePoint pong_time, + Duration interval, + Duration rx_window) { + auto& state = EnsureServerPresence(server_id); + auto const schedule = + MakeConfirmedSchedule(send_time, pong_time, interval, rx_window); + state.has_confirmed_schedule = true; + state.confirmed_interval = schedule.interval; + state.confirmed_rx_window = schedule.rx_window; + state.confirmed_ping_send_time = schedule.ping_send_time; + state.confirmed_pong_receive_time = schedule.pong_receive_time; + state.confirmed_window_open_local = schedule.window_open_local; + state.confirmed_window_close_local = schedule.window_close_local; + state.online = true; + state.config_change_pending = (state.desired.interval != interval) || + (state.desired.rx_window != rx_window); +} + +void ClientConnectivityPolicy::MarkServerOffline(ServerId server_id, + TimePoint now) { + auto* state = FindServerPresence(server_id); + if (state == nullptr) { + return; + } + if (state->has_confirmed_schedule && now > state->confirmed_window_close_local) { + state->online = false; + } +} + +void ClientConnectivityPolicy::ClearServerPresence(ServerId server_id) { + server_presence_.erase(server_id); +} + +void ClientConnectivityPolicy::InvalidateConfirmedSchedule(ServerId server_id) { + auto* state = FindServerPresence(server_id); + if (state == nullptr) { + return; + } + state->has_confirmed_schedule = false; + state->online = false; + state->confirmed_window_open_local = {}; + state->confirmed_window_close_local = {}; +} + +bool ClientConnectivityPolicy::IsLocallyOnline() const noexcept { + return IsLocallyOnline(Now()); +} + +bool ClientConnectivityPolicy::IsLocallyOnline(TimePoint now) const noexcept { + for (auto const& [id, state] : server_presence_) { + static_cast(id); + if (!state.selected_for_aggregate) { + continue; + } + if (IsConfirmedWindowOnline(state.has_confirmed_schedule, now, + state.confirmed_window_close_local)) { + return true; + } + } + return false; +} + +bool ClientConnectivityPolicy::IsServerLocallyOnline( + ServerId server_id, TimePoint now) const noexcept { + auto const* state = FindServerPresence(server_id); + if (state == nullptr) { + return false; + } + return IsConfirmedWindowOnline(state->has_confirmed_schedule, now, + state->confirmed_window_close_local); +} + +void ClientConnectivityPolicy::RefreshOnlineFlags(TimePoint now) { + for (auto& [id, state] : server_presence_) { + static_cast(id); + state.online = IsConfirmedWindowOnline(state.has_confirmed_schedule, now, + state.confirmed_window_close_local); + } +} + void ClientConnectivityPolicy::ResetRuntimeState() { auto current_time = Now(); for (auto& t : rx_timings_) { - // if clock was reset, also reset next rx points if (current_time < t.recordet_at) { t.next_rx_point = {}; t.recordet_at = {}; } } + for (auto& [id, state] : server_presence_) { + static_cast(id); + if (current_time < state.confirmed_pong_receive_time) { + state.has_confirmed_schedule = false; + state.online = false; + state.confirmed_window_open_local = {}; + state.confirmed_window_close_local = {}; + } + } +} + +void ClientConnectivityPolicy::ApplyDesiredToBoundServers(RxTimingConf conf) { + for (auto& [id, state] : server_presence_) { + static_cast(id); + auto const timing_changed = (state.desired.interval != conf.interval) || + (state.desired.rx_window != conf.rx_window); + state.desired = conf; + if (timing_changed) { + state.config_change_pending = true; + } + } } void ClientConnectivityPolicy::IncrementSuspendBlock() { diff --git a/aether/client_connectivity_policy.h b/aether/client_connectivity_policy.h index d038af72..88413ca4 100644 --- a/aether/client_connectivity_policy.h +++ b/aether/client_connectivity_policy.h @@ -21,12 +21,17 @@ #include #include #include +#include +#include +#include "aether/cloud_connections/local_presence_schedule.h" +#include "aether/cloud_connections/request_policy.h" #include "aether/config.h" #include "aether/events/events.h" #include "aether/obj/obj.h" +#include "aether/types/server_id.h" -#include "aether/cloud_connections/request_policy.h" +#include namespace ae { @@ -63,6 +68,27 @@ struct ConnectivityStatus { TimePoint next_service_time; }; +struct ServerPresenceState { + RxTimingConf desired{ + RxTimingConf::Every(std::chrono::milliseconds{AE_PING_INTERVAL_MS})}; + std::uint8_t rtt_reliability_percentile{kDefaultRttReliabilityPercentile}; + + bool has_confirmed_schedule{false}; + Duration confirmed_interval{}; + Duration confirmed_rx_window{}; + TimePoint confirmed_ping_send_time{}; + TimePoint confirmed_pong_receive_time{}; + TimePoint confirmed_window_open_local{}; + TimePoint confirmed_window_close_local{}; + + bool online{false}; + std::uint64_t current_attempt_id{0}; + PingAttemptKind current_attempt_kind{PingAttemptKind::kInitial}; + bool config_change_pending{false}; + bool selected_for_aggregate{true}; + bool has_user_rx_timing{false}; +}; + class ClientConnectivityPolicy : public Obj { AE_OBJECT(ClientConnectivityPolicy, Obj, 0) @@ -77,6 +103,7 @@ class ClientConnectivityPolicy : public Obj { RxTimingConfig& ForPriority(RxTimingConf conf) { static_assert(Priority < kMaxRxServerPriorities); policy_->rx_timings_[Priority].conf = conf; + policy_->ApplyDesiredToBoundServers(conf); return *this; } @@ -118,6 +145,13 @@ class ClientConnectivityPolicy : public Obj { RxTimingConfig ConfigureRxTimings( RequestPolicy::Variant targets = RequestPolicy::All{}); + // Per-server runtime config. Does not invent ONLINE until a confirming Pong. + void ConfigureServerRxTiming( + ServerId server_id, RxTimingConf conf, + std::uint8_t rtt_reliability_percentile = kDefaultRttReliabilityPercentile); + + void SetServerSelectedForAggregate(ServerId server_id, bool selected); + RequestPolicy::Variant const& rx_targets() const noexcept { return rx_targets_; } @@ -128,6 +162,9 @@ class ClientConnectivityPolicy : public Obj { Event::Subscriber suspend_allowed_event() noexcept { return EventSubscriber{suspend_allowed_event_}; } + Event::Subscriber server_rx_timing_changed_event() noexcept { + return EventSubscriber{server_rx_timing_changed_event_}; + } ConnectivityStatus GetStatus() const noexcept; void ResetRxTimings(); @@ -135,18 +172,43 @@ class ClientConnectivityPolicy : public Obj { SuspendBlocker AcquireSuspendBlock(); void ReportNextServiceTime(std::size_t priority, TimePoint next_service_time); + ServerPresenceState& EnsureServerPresence(ServerId server_id); + ServerPresenceState const* FindServerPresence(ServerId server_id) const noexcept; + ServerPresenceState* FindServerPresence(ServerId server_id) noexcept; + + // Confirm schedule from a successful Pong (measured RTT). + void ConfirmServerPong(ServerId server_id, TimePoint send_time, + TimePoint pong_time, Duration interval, + Duration rx_window); + + void MarkServerOffline(ServerId server_id, TimePoint now); + void ClearServerPresence(ServerId server_id); + // Drop confirmed schedule (quarantine / unusable) but keep desired timing. + void InvalidateConfirmedSchedule(ServerId server_id); + + // Read-only. No side effects. Aggregate: ONLINE iff any selected usable + // server has a confirmed schedule that has not expired. + bool IsLocallyOnline() const noexcept; + bool IsLocallyOnline(TimePoint now) const noexcept; + bool IsServerLocallyOnline(ServerId server_id, TimePoint now) const noexcept; + + void RefreshOnlineFlags(TimePoint now); + private: void ResetRuntimeState(); void IncrementSuspendBlock(); void DecrementSuspendBlock(); + void ApplyDesiredToBoundServers(RxTimingConf conf); RequestPolicy::Variant rx_targets_; std::array rx_timings_; + std::map server_presence_; bool can_suspend_{true}; std::uint8_t suspend_block_count_{}; Event suspend_allowed_event_; + Event server_rx_timing_changed_event_; }; } // namespace ae diff --git a/aether/cloud_connections/local_presence_schedule.h b/aether/cloud_connections/local_presence_schedule.h new file mode 100644 index 00000000..7a294a1e --- /dev/null +++ b/aether/cloud_connections/local_presence_schedule.h @@ -0,0 +1,157 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_CLOUD_CONNECTIONS_LOCAL_PRESENCE_SCHEDULE_H_ +#define AETHER_CLOUD_CONNECTIONS_LOCAL_PRESENCE_SCHEDULE_H_ + +#include +#include + +#include "aether/clock.h" + +namespace ae { + +// Fixed scheduler safety guard (not rx_window). +inline constexpr Duration kLocalPresenceGuard = + std::chrono::duration_cast(std::chrono::milliseconds{30}); + +inline constexpr std::uint8_t kDefaultRttReliabilityPercentile{99}; + +enum class PingAttemptKind : std::uint8_t { + kInitial = 0, + kPrefix1, + kPrefix2, + kRetry, + kRecovery, +}; + +// One-way RTT projection used consistently for schedule placement. +// Documented model: one_way = rtt / 2 (local monotonic timeline only). +inline Duration OneWayFromRtt(Duration rtt) noexcept { + return rtt / 2; +} + +struct ConfirmedReceiveSchedule { + Duration interval{}; + Duration rx_window{}; + TimePoint ping_send_time{}; + TimePoint pong_receive_time{}; + Duration measured_rtt{}; + TimePoint window_open_local{}; + TimePoint window_close_local{}; +}; + +// After successful Pong for Ping sent at send_time: +// R_server ≈ send_time + one_way +// window_open = R_server + interval +// window_close = window_open + rx_window +inline ConfirmedReceiveSchedule MakeConfirmedSchedule( + TimePoint send_time, TimePoint pong_time, Duration interval, + Duration rx_window) noexcept { + ConfirmedReceiveSchedule out{}; + out.interval = interval; + out.rx_window = rx_window; + out.ping_send_time = send_time; + out.pong_receive_time = pong_time; + if (pong_time > send_time) { + out.measured_rtt = + std::chrono::duration_cast(pong_time - send_time); + } + auto const one_way = OneWayFromRtt(out.measured_rtt); + out.window_open_local = send_time + one_way + interval; + out.window_close_local = out.window_open_local + rx_window; + return out; +} + +// prefix1 = O - 1.5*R - G +// prefix2 = O - 0.5*R - G +inline TimePoint ComputePrefix1Time(TimePoint window_open, Duration rtt, + Duration guard = kLocalPresenceGuard) noexcept { + return window_open - (rtt * 3) / 2 - guard; +} + +inline TimePoint ComputePrefix2Time(TimePoint window_open, Duration rtt, + Duration guard = kLocalPresenceGuard) noexcept { + return window_open - rtt / 2 - guard; +} + +inline bool IsConfirmedWindowOnline(bool has_confirmed, TimePoint now, + TimePoint window_close) noexcept { + if (!has_confirmed) { + return false; + } + return now <= window_close; +} + +inline bool IsCurrentPingAttempt(std::uint64_t active_attempt_id, + std::uint64_t result_attempt_id) noexcept { + return active_attempt_id == result_attempt_id; +} + +// Next Ping after a failed attempt. p99/selected RTT timeout is NOT OFFLINE. +struct PresenceAttemptPlan { + TimePoint when{}; + PingAttemptKind kind{PingAttemptKind::kRecovery}; + bool mark_offline{false}; +}; + +inline PresenceAttemptPlan PlanAfterFailedAttempt( + bool has_confirmed_schedule, TimePoint confirmed_window_open, + TimePoint confirmed_window_close, PingAttemptKind failed_kind, + TimePoint now, Duration rtt, + Duration guard = kLocalPresenceGuard) noexcept { + PresenceAttemptPlan plan{}; + if (has_confirmed_schedule && now <= confirmed_window_close) { + plan.mark_offline = false; + if (failed_kind == PingAttemptKind::kPrefix1) { + plan.kind = PingAttemptKind::kPrefix2; + auto const prefix2 = + ComputePrefix2Time(confirmed_window_open, rtt, guard); + plan.when = prefix2 > now ? prefix2 : now; + return plan; + } + plan.kind = PingAttemptKind::kRetry; + plan.when = now + rtt; + return plan; + } + plan.mark_offline = has_confirmed_schedule; + plan.kind = PingAttemptKind::kRecovery; + plan.when = now + rtt; + return plan; +} + +// After a confirming Pong: prefix1 of the new window, unless a newer desired +// interval/window still needs a Ping. +inline PresenceAttemptPlan PlanAfterSuccessfulPong( + TimePoint confirmed_window_open, TimePoint now, Duration rtt, + bool send_new_config_immediately, + Duration guard = kLocalPresenceGuard) noexcept { + PresenceAttemptPlan plan{}; + plan.mark_offline = false; + if (send_new_config_immediately) { + plan.kind = PingAttemptKind::kInitial; + plan.when = now; + return plan; + } + plan.kind = PingAttemptKind::kPrefix1; + auto const prefix1 = ComputePrefix1Time(confirmed_window_open, rtt, guard); + plan.when = prefix1 > now ? prefix1 : now; + return plan; +} + +} // namespace ae + +#endif // AETHER_CLOUD_CONNECTIONS_LOCAL_PRESENCE_SCHEDULE_H_ diff --git a/aether/cloud_connections/ping_cloud_servers.cpp b/aether/cloud_connections/ping_cloud_servers.cpp index b1128e14..d990fc65 100644 --- a/aether/cloud_connections/ping_cloud_servers.cpp +++ b/aether/cloud_connections/ping_cloud_servers.cpp @@ -17,18 +17,38 @@ #include "aether/cloud_connections/ping_cloud_servers.h" #include +#include #include #include #if AE_ENABLE_PING # include "aether/channels/channel.h" -# include "aether/executors/executors.h" - # include "aether/cloud_connections/cloud_connections_tele.h" +# include "aether/executors/executors.h" namespace ae { +namespace { + +char const* AttemptKindName(PingAttemptKind kind) { + switch (kind) { + case PingAttemptKind::kInitial: + return "INITIAL"; + case PingAttemptKind::kPrefix1: + return "PREFIX1"; + case PingAttemptKind::kPrefix2: + return "PREFIX2"; + case PingAttemptKind::kRetry: + return "RETRY"; + case PingAttemptKind::kRecovery: + return "RECOVERY"; + } + return "UNKNOWN"; +} + +} // namespace + PingCloudServers::ServerPing::ServerPing(AeContext const& ae_context, ClientConnectivityPolicy& policy, CloudServerConnection& cloud_sc, @@ -36,41 +56,90 @@ PingCloudServers::ServerPing::ServerPing(AeContext const& ae_context, : ae_context_{ae_context}, policy_{&policy}, cloud_sc_{&cloud_sc}, + server_id_{cloud_sc.server_id()}, priority_{priority} { assert(priority < policy_->rx_timings().size() && "Server ping priority should be in timings range"); - auto const& timings = policy_->rx_timings()[priority_]; - timing_conf_ = timings.conf; - - // if it's to early for next rx wait a bit - if ((timings.next_rx_point != TimePoint{}) && - (Now() < timings.next_rx_point)) { - AE_TELED_DEBUG("Wait a bit for next rx point till {}", - timings.next_rx_point); - start_sub_ = ae_context_.scheduler().DelayedTask([&]() { Start(); }, - timings.next_rx_point); - } else { - // acquire suspend for first ping - ping_blocker_ = policy_->AcquireSuspendBlock(); - start_sub_ = ae_context_.scheduler().Task([&]() { Start(); }); + auto& presence = policy_->EnsureServerPresence(server_id_); + if (!presence.has_user_rx_timing) { + presence.desired = policy_->rx_timings()[priority_].conf; } + active_conf_ = presence.desired; + policy_->SetServerSelectedForAggregate(server_id_, true); + + ping_blocker_ = policy_->AcquireSuspendBlock(); + start_sub_ = ae_context_.scheduler().Task( + [this]() { StartAttempt(PingAttemptKind::kInitial); }); } PingCloudServers::ServerPing::~ServerPing() = default; void PingCloudServers::ServerPing::Stop() { stop_ = true; - + AbandonInFlight(); waiter_.reset(); start_sub_.Reset(); + attempt_timeout_sub_.Reset(); rx_window_sub_.Reset(); restream_sub_.Reset(); link_state_sub_.Reset(); - ping_blocker_.Reset(); rx_window_blocker_.Reset(); restream_blocker_.Reset(); + policy_->SetServerSelectedForAggregate(server_id_, false); + policy_->InvalidateConfirmedSchedule(server_id_); + policy_->RefreshOnlineFlags(Now()); +} + +void PingCloudServers::ServerPing::NotifyConfigChanged() { + if (stop_) { + return; + } + auto* presence = policy_->FindServerPresence(server_id_); + if (presence == nullptr) { + return; + } + active_conf_ = presence->desired; + if (attempt_in_flight_) { + return; + } + if (presence->config_change_pending || !presence->has_confirmed_schedule) { + ScheduleNext(Now(), PingAttemptKind::kInitial); + return; + } + auto const rtt = SelectedRtt(); + auto const prefix1 = + ComputePrefix1Time(presence->confirmed_window_open_local, rtt); + ScheduleNext(prefix1 > Now() ? prefix1 : Now(), PingAttemptKind::kPrefix1); +} + +void PingCloudServers::ServerPing::ScheduleNext(TimePoint when, + PingAttemptKind kind) { + if (stop_) { + return; + } + next_ping_time_ = when; + policy_->ReportNextServiceTime(priority_, next_ping_time_); + auto* presence = policy_->FindServerPresence(server_id_); + if (presence != nullptr) { + presence->current_attempt_kind = kind; + } + start_sub_ = ae_context_.scheduler().DelayedTask( + [this, kind]() noexcept { StartAttempt(kind); }, when); +} + +void PingCloudServers::ServerPing::StartAttempt(PingAttemptKind kind) { + if (stop_ || attempt_in_flight_) { + return; + } + auto& presence = policy_->EnsureServerPresence(server_id_); + active_conf_ = presence.desired; + presence.current_attempt_kind = kind; + ++presence.current_attempt_id; + active_attempt_id_ = presence.current_attempt_id; + active_attempt_kind_ = kind; + Start(); } template @@ -102,45 +171,45 @@ auto PingCloudServers::ServerPing::EnsureLinked() { }); } -auto PingCloudServers::ServerPing::MakePing() { - return ex::let_value([&]() noexcept { - return ex::create( - [&](auto& ctx) noexcept { - // make ping action with timeout based on response statistics - // and timing properties for current server RxTimings - // during the ping and rx window setup suspend blocker - // and save expected next_ping_time_ - auto* cc = cloud_sc_->client_connection(); - assert(cc != nullptr && "Client connection should exists"); - - auto c = cc->server_connection().current_channel(); - if (c == nullptr) { - AE_TELED_ERROR("Current channel value invalid"); - return ex::set_error(std::move(ctx.receiver), 2); - } - - ping_.emplace(ae_context_, *cloud_sc_, timing_conf_.interval, - timing_conf_.rx_window, c->ResponseTimeout()); - - ping_blocker_ = policy_->AcquireSuspendBlock(); - ping_->result_event().Subscribe( - [this](Ping::PingResult const& res) noexcept { - OnPingResult(res); - ping_blocker_.Reset(); - }); - - // run ping request and open rx window - auto const current_time = Now(); - ping_->Start(current_time); - OpenRxWindow(current_time); - next_ping_time_ = current_time + timing_conf_.interval; - policy_->ReportNextServiceTime(priority_, next_ping_time_); - AE_TELED_DEBUG("Next ping time for priority {} at {} after {}", - priority_, next_ping_time_, timing_conf_.interval); +Duration PingCloudServers::ServerPing::SelectedRtt() const { + auto* cc = cloud_sc_->client_connection(); + if (cc == nullptr) { + return std::chrono::milliseconds{AE_DEFAULT_RESPONSE_TIMEOUT_MS}; + } + auto c = cc->server_connection().current_channel(); + if (!c) { + return std::chrono::milliseconds{AE_DEFAULT_RESPONSE_TIMEOUT_MS}; + } + auto const& stats = c->channel_statistics().response_time_statistics(); + auto const* presence = policy_->FindServerPresence(server_id_); + auto const pct = presence == nullptr ? kDefaultRttReliabilityPercentile + : presence->rtt_reliability_percentile; + if (stats.empty()) { + return std::chrono::milliseconds{AE_DEFAULT_RESPONSE_TIMEOUT_MS}; + } + return stats.PercentileValue(pct); +} - return ex::set_value(std::move(ctx.receiver)); - }); - }); +Duration PingCloudServers::ServerPing::AttemptTimeout(Duration rtt) const { + // Scheduler treats selected RTT as the attempt window; floor with guard. + if (rtt <= Duration{}) { + return kLocalPresenceGuard; + } + return rtt; +} + +void PingCloudServers::ServerPing::AbandonInFlight() { + attempt_timeout_sub_.Reset(); + if (ping_) { + ping_.reset(); + } + attempt_in_flight_ = false; + ping_blocker_.Reset(); + ++active_attempt_id_; + auto* presence = policy_->FindServerPresence(server_id_); + if (presence != nullptr) { + presence->current_attempt_id = active_attempt_id_; + } } void PingCloudServers::ServerPing::Start() { @@ -159,8 +228,64 @@ void PingCloudServers::ServerPing::Start() { } return ex::just(); }) | - MakePing() | - // track Stop command + ex::let_value([&]() noexcept { + return ex::create( + [&](auto& ctx) noexcept { + auto* cc = cloud_sc_->client_connection(); + assert(cc != nullptr && "Client connection should exists"); + auto c = cc->server_connection().current_channel(); + if (c == nullptr) { + AE_TELED_ERROR("Current channel value invalid"); + return ex::set_error(std::move(ctx.receiver), 2); + } + + auto const rtt = SelectedRtt(); + auto const timeout = AttemptTimeout(rtt); + auto& presence = policy_->EnsureServerPresence(server_id_); + active_conf_ = presence.desired; + auto const percentile = presence.rtt_reliability_percentile; + + active_sent_interval_ = active_conf_.interval; + active_sent_window_ = active_conf_.rx_window; + active_send_time_ = Now(); + attempt_in_flight_ = true; + + ping_.emplace(ae_context_, *cloud_sc_, active_sent_interval_, + active_sent_window_, timeout); + + ping_blocker_ = policy_->AcquireSuspendBlock(); + auto const attempt_id = active_attempt_id_; + auto const send_time = active_send_time_; + auto const sent_interval = active_sent_interval_; + auto const sent_window = active_sent_window_; + ping_->result_event().Subscribe( + [this, attempt_id, send_time, sent_interval, + sent_window](Ping::PingResult const& res) noexcept { + OnPingResult(attempt_id, send_time, sent_interval, + sent_window, res); + }); + + AE_TELED_DEBUG( + "PING_ATTEMPT server {} id {} kind {} send {}", + server_id_, attempt_id, + AttemptKindName(active_attempt_kind_), send_time); + AE_TELED_DEBUG( + "RTT server {} percentile {} selected_rtt {}", + server_id_, static_cast(percentile), rtt); + + ping_->Start(send_time); + + // Scheduler failure of this attempt at selected RTT — not + // OFFLINE. + attempt_timeout_sub_ = ae_context_.scheduler().DelayedTask( + [this, attempt_id]() noexcept { + OnAttemptTimeout(attempt_id); + }, + send_time + timeout); + + return ex::set_value(std::move(ctx.receiver)); + }); + }) | ex::let_value( [&]() noexcept -> ex::variant_sender(std::optional&& res) noexcept { - if (res && res->IsOk()) { - // repeat start on next_ping_time_ - start_sub_ = ae_context_.scheduler().DelayedTask( - [&]() noexcept { Start(); }, // ~['_']~ - next_ping_time_); - } else if (res && res->IsErr()) { + if (res && res->IsErr()) { AE_TELED_ERROR("Ping start error {}", std::move(res)->error()); - } else { + attempt_in_flight_ = false; + ScheduleRestream(); + ScheduleNext(Now() + SelectedRtt(), PingAttemptKind::kRecovery); + } else if (!(res && res->IsOk())) { AE_TELED_DEBUG("Server ping stopped"); } }); } -void PingCloudServers::ServerPing::OnPingResult(Ping::PingResult const& res) { +void PingCloudServers::ServerPing::OnPingResult(std::uint64_t attempt_id, + TimePoint send_time, + Duration sent_interval, + Duration sent_window, + Ping::PingResult const& res) { + if (stop_) { + return; + } + // Late / abandoned attempts must not roll confirmed schedule backwards. + if (!IsCurrentPingAttempt(active_attempt_id_, attempt_id)) { + AE_TELED_DEBUG("Ignoring stale ping result attempt {}", attempt_id); + return; + } + auto* cc = cloud_sc_->client_connection(); if (cc == nullptr) { AE_TELED_ERROR("Client connection is null"); return; } - auto c = cc->server_connection().current_channel(); if (!c) { AE_TELED_ERROR("Connection channel is null"); @@ -198,35 +333,105 @@ void PingCloudServers::ServerPing::OnPingResult(Ping::PingResult const& res) { } std::visit( - [this, c](auto const& value) { + [this, c, send_time, sent_interval, sent_window](auto const& value) { using T = std::decay_t; if constexpr (std::is_same_v>) { c->channel_statistics().AddResponseTime(value.value); + ApplyConfirmedPong(send_time, Now(), sent_interval, sent_window); } else if constexpr (std::is_same_v) { - AE_TELED_DEBUG("Got late ping duration"); + AE_TELED_DEBUG("Got late ping duration for active attempt"); c->channel_statistics().AddResponseTime(value.duration); + ApplyConfirmedPong(send_time, Now(), sent_interval, sent_window); } else { AE_TELED_ERROR("Ping error!"); + AbandonInFlight(); ScheduleRestream(); + AfterFailedAttempt(); } }, res); } -void PingCloudServers::ServerPing::OpenRxWindow(TimePoint sent_time) { - // keep rx window suspend block for timing_.rx_window time +void PingCloudServers::ServerPing::ApplyConfirmedPong(TimePoint send_time, + TimePoint pong_time, + Duration sent_interval, + Duration sent_window) { + attempt_timeout_sub_.Reset(); + attempt_in_flight_ = false; + ping_blocker_.Reset(); + + policy_->ConfirmServerPong(server_id_, send_time, pong_time, sent_interval, + sent_window); + auto* presence = policy_->FindServerPresence(server_id_); + assert(presence != nullptr); + AE_TELED_DEBUG( + "SCHEDULE confirmed server {} open {} close {} interval {} window {}", + server_id_, presence->confirmed_window_open_local, + presence->confirmed_window_close_local, sent_interval, sent_window); + AE_TELED_DEBUG("STATUS server {} ONLINE", server_id_); + + // Contractual window is a minimum listen guarantee, not a transport close. + HoldRxUntil(presence->confirmed_window_close_local); + + auto const now = Now(); + auto const rtt = SelectedRtt(); + auto const plan = PlanAfterSuccessfulPong( + presence->confirmed_window_open_local, now, rtt, + presence->config_change_pending); + ScheduleNext(plan.when, plan.kind); +} + +void PingCloudServers::ServerPing::OnAttemptTimeout(std::uint64_t attempt_id) { + if (stop_ || !IsCurrentPingAttempt(active_attempt_id_, attempt_id) || + !attempt_in_flight_) { + return; + } + AE_TELED_DEBUG("Ping attempt {} kind {} timed out for scheduler", attempt_id, + AttemptKindName(active_attempt_kind_)); + AbandonInFlight(); + AfterFailedAttempt(); +} + +void PingCloudServers::ServerPing::AfterFailedAttempt() { + if (stop_) { + return; + } + auto now = Now(); + policy_->RefreshOnlineFlags(now); + auto* presence = policy_->FindServerPresence(server_id_); + if (presence == nullptr) { + ScheduleNext(now + SelectedRtt(), PingAttemptKind::kRecovery); + return; + } + + auto const rtt = SelectedRtt(); + if (presence->config_change_pending && presence->has_confirmed_schedule) { + ScheduleNext(now, PingAttemptKind::kInitial); + return; + } + + auto const plan = PlanAfterFailedAttempt( + presence->has_confirmed_schedule, presence->confirmed_window_open_local, + presence->confirmed_window_close_local, active_attempt_kind_, now, rtt); + if (plan.mark_offline) { + policy_->MarkServerOffline(server_id_, now); + AE_TELED_DEBUG("STATUS server {} OFFLINE after window close {}", server_id_, + presence->confirmed_window_close_local); + } + ScheduleNext(plan.when, plan.kind); +} + +void PingCloudServers::ServerPing::HoldRxUntil(TimePoint until) { + // rx_window is a minimum contractual guarantee. Do not close transport. rx_window_blocker_ = policy_->AcquireSuspendBlock(); rx_window_sub_ = ae_context_.scheduler().DelayedTask( - [this]() { rx_window_blocker_.Reset(); }, - sent_time + timing_conf_.rx_window); + [this]() { rx_window_blocker_.Reset(); }, until); } void PingCloudServers::ServerPing::ScheduleRestream() { if (stop_) { return; } - - // TODO: should we block till restream? restream_blocker_ = policy_->AcquireSuspendBlock(); restream_sub_ = ae_context_.scheduler().Task([this]() { auto* cc = cloud_sc_->client_connection(); @@ -254,6 +459,9 @@ PingCloudServers::PingCloudServers( server_quarantine_released_sub_ = cloud_server_connections_->server_quarantine_release_event().Subscribe( MethodPtr<&PingCloudServers::ServerQuarantineReleased>{this}); + server_rx_timing_changed_sub_ = + policy_->server_rx_timing_changed_event().Subscribe( + MethodPtr<&PingCloudServers::OnServerRxTimingChanged>{this}); ServersUpdate(); } @@ -301,6 +509,7 @@ void PingCloudServers::ReconcileServer(CloudServerConnection& cloud_sc) { priority)); return; } + it->second->NotifyConfigChanged(); } void PingCloudServers::ServerQuarantined(CloudServerConnection* cloud_sc) { @@ -323,6 +532,13 @@ void PingCloudServers::ServerQuarantineReleased( server_pings_.erase(it); } } + +void PingCloudServers::OnServerRxTimingChanged(ServerId server_id) { + auto it = server_pings_.find(server_id); + if (it != server_pings_.end() && !it->second->stopped()) { + it->second->NotifyConfigChanged(); + } +} } // namespace ae #endif diff --git a/aether/cloud_connections/ping_cloud_servers.h b/aether/cloud_connections/ping_cloud_servers.h index dd55ef65..83fa8046 100644 --- a/aether/cloud_connections/ping_cloud_servers.h +++ b/aether/cloud_connections/ping_cloud_servers.h @@ -20,20 +20,21 @@ #include "aether/config.h" #if AE_ENABLE_PING +# include # include # include # include +# include "aether/ae_actions/ping.h" # include "aether/ae_context.h" +# include "aether/client_connectivity_policy.h" +# include "aether/cloud_connections/cloud_server_connections.h" +# include "aether/cloud_connections/local_presence_schedule.h" # include "aether/events/event_subscription.h" # include "aether/executors/executors.h" # include "aether/tasks/manual_task_scheduler.h" # include "aether/types/server_id.h" -# include "aether/ae_actions/ping.h" -# include "aether/client_connectivity_policy.h" -# include "aether/cloud_connections/cloud_server_connections.h" - namespace ae { class PingCloudServers { class ServerPing { @@ -45,37 +46,57 @@ class PingCloudServers { AE_CLASS_NO_COPY_MOVE(ServerPing) void Stop(); + void NotifyConfigChanged(); TimePoint next_service_time() const noexcept { return next_ping_time_; } std::size_t priority() const noexcept { return priority_; } - RxTimingConf const& timing() const noexcept { return timing_conf_; } + RxTimingConf const& timing() const noexcept { return active_conf_; } bool stopped() const noexcept { return stop_; } private: + void ScheduleNext(TimePoint when, PingAttemptKind kind); + void StartAttempt(PingAttemptKind kind); void Start(); template void WaitForLink(ClientServerConnection& cc, F&& f); auto EnsureLinked(); - auto MakePing(); - - void OnPingResult(Ping::PingResult const& res); - void OpenRxWindow(TimePoint sent_time); + Duration SelectedRtt() const; + Duration AttemptTimeout(Duration rtt) const; + + void OnPingResult(std::uint64_t attempt_id, TimePoint send_time, + Duration sent_interval, Duration sent_window, + Ping::PingResult const& res); + void ApplyConfirmedPong(TimePoint send_time, TimePoint pong_time, + Duration sent_interval, Duration sent_window); + void OnAttemptTimeout(std::uint64_t attempt_id); + void AfterFailedAttempt(); + void HoldRxUntil(TimePoint until); void ScheduleRestream(); + void AbandonInFlight(); AeContext ae_context_; ClientConnectivityPolicy* policy_; CloudServerConnection* cloud_sc_; - RxTimingConf timing_conf_{}; + ServerId server_id_{}; std::size_t priority_{}; + RxTimingConf active_conf_{}; std::optional> waiter_; std::optional ping_; bool stop_{false}; + bool attempt_in_flight_{false}; + std::uint64_t active_attempt_id_{0}; + PingAttemptKind active_attempt_kind_{PingAttemptKind::kInitial}; + TimePoint active_send_time_{}; + Duration active_sent_interval_{}; + Duration active_sent_window_{}; + Subscription link_state_sub_; TaskSubscription start_sub_; + TaskSubscription attempt_timeout_sub_; TaskSubscription rx_window_sub_; TaskSubscription restream_sub_; ClientConnectivityPolicy::SuspendBlocker ping_blocker_; @@ -96,6 +117,7 @@ class PingCloudServers { void ReconcileServer(CloudServerConnection& cloud_sc); void ServerQuarantined(CloudServerConnection* cloud_sc); void ServerQuarantineReleased(CloudServerConnection* cloud_sc); + void OnServerRxTimingChanged(ServerId server_id); AeContext ae_context_; CloudServerConnections* cloud_server_connections_; @@ -104,6 +126,7 @@ class PingCloudServers { Subscription servers_update_; Subscription server_quarantined_sub_; Subscription server_quarantine_released_sub_; + Subscription server_rx_timing_changed_sub_; TaskSubscription task_sub_; std::map> server_pings_; diff --git a/aether/types/statistic_counter.h b/aether/types/statistic_counter.h index 44078232..0c914852 100644 --- a/aether/types/statistic_counter.h +++ b/aether/types/statistic_counter.h @@ -81,21 +81,28 @@ class StatisticsCounter final { [[nodiscard]] TValue percentile() const { static_assert((Percentile >= 0) && (Percentile <= 100), "Percentile must be in [0,100]% range"); + return PercentileValue(Percentile); + } - if constexpr (Percentile == 0) { + /** + * \brief Runtime percentile accessor (0..100). Same semantics as the + * compile-time template overload. + */ + [[nodiscard]] TValue PercentileValue(std::size_t percentile) const { + assert(percentile <= 100); + assert(!value_buffer_.empty()); + if (percentile == 0) { return min(); - } else if constexpr (Percentile == 100) { + } + if (percentile == 100) { return max(); - } else { - assert(!value_buffer_.empty()); - auto sorted_list = value_buffer_; - std::sort(std::begin(sorted_list), std::end(sorted_list), Comparator{}); - - auto index = static_cast( // - std::ceil(static_cast(sorted_list.size() - 1) * Percentile / - 100.0)); - return sorted_list[index]; } + auto sorted_list = value_buffer_; + std::sort(std::begin(sorted_list), std::end(sorted_list), Comparator{}); + auto index = static_cast( // + std::ceil(static_cast(sorted_list.size() - 1) * percentile / + 100.0)); + return sorted_list[index]; } std::size_t size() const { return value_buffer_.size(); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 64f56845..aedd2dc6 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -46,4 +46,6 @@ add_subdirectory(test-serial-port) add_subdirectory(test-tasks) add_subdirectory(test-server-connection) +add_subdirectory(test-local-presence) + add_subdirectory(third_party_tests) diff --git a/tests/test-local-presence/CMakeLists.txt b/tests/test-local-presence/CMakeLists.txt new file mode 100644 index 00000000..cfd9e9e7 --- /dev/null +++ b/tests/test-local-presence/CMakeLists.txt @@ -0,0 +1,29 @@ +# Copyright 2026 Aethernet Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.16) + +if(NOT CM_PLATFORM) + project(test-local-presence LANGUAGES CXX) + + add_executable(${PROJECT_NAME} main.cpp) + target_include_directories(${PROJECT_NAME} PRIVATE ${ROOT_DIR}) + target_link_libraries(${PROJECT_NAME} PRIVATE unity aether) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(${PROJECT_NAME} PRIVATE /Zc:preprocessor) + endif() + add_test(NAME ${PROJECT_NAME} COMMAND $) +else() + message(WARNING "Not implemented for ${CM_PLATFORM}") +endif() diff --git a/tests/test-local-presence/main.cpp b/tests/test-local-presence/main.cpp new file mode 100644 index 00000000..b4d065df --- /dev/null +++ b/tests/test-local-presence/main.cpp @@ -0,0 +1,900 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include "aether/clock.h" +#include "aether/client_connectivity_policy.h" +#include "aether/cloud_connections/local_presence_schedule.h" +#include "aether/types/statistic_counter.h" + +namespace ae::test_local_presence { + +using Ms = std::chrono::milliseconds; + +TimePoint Tp(std::int64_t ms) { return TimePoint{Ms{ms}}; } + +Duration Dur(std::int64_t ms) { return std::chrono::duration_cast(Ms{ms}); } + +std::int64_t ToMs(TimePoint tp) { + return std::chrono::duration_cast(tp.time_since_epoch()).count(); +} + +std::int64_t ToMs(Duration d) { return std::chrono::duration_cast(d).count(); } + +void test_PrefixFormula() { + auto const R = Dur(100); + auto const G = kLocalPresenceGuard; + auto const O = Tp(1050); + auto const p1 = ComputePrefix1Time(O, R, G); + auto const p2 = ComputePrefix2Time(O, R, G); + TEST_ASSERT_EQUAL(870, ToMs(p1)); + TEST_ASSERT_EQUAL(970, ToMs(p2)); + TEST_ASSERT_EQUAL(100, std::chrono::duration_cast(p2 - p1).count()); + TEST_ASSERT_EQUAL(30, ToMs(G)); +} + +void test_ConfirmOnlyAfterPong() { + ClientConnectivityPolicy policy; + ServerId const sid{7}; + policy.ConfigureServerRxTiming(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), + 99); + auto* state = policy.FindServerPresence(sid); + TEST_ASSERT_NOT_NULL(state); + TEST_ASSERT_FALSE(state->has_confirmed_schedule); + TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(sid, Tp(0))); + TEST_ASSERT_FALSE(policy.IsLocallyOnline(Tp(0))); + + policy.ConfirmServerPong(sid, Tp(1000), Tp(1100), Dur(1000), Dur(300)); + state = policy.FindServerPresence(sid); + TEST_ASSERT_TRUE(state->has_confirmed_schedule); + TEST_ASSERT_EQUAL(2050, ToMs(state->confirmed_window_open_local)); + TEST_ASSERT_EQUAL(2350, ToMs(state->confirmed_window_close_local)); + TEST_ASSERT_TRUE(policy.IsServerLocallyOnline(sid, Tp(2050))); + TEST_ASSERT_TRUE(policy.IsServerLocallyOnline(sid, Tp(2350))); + TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(sid, Tp(2351))); +} + +void test_PerServerIndependence() { + ClientConnectivityPolicy policy; + ServerId const a{1}; + ServerId const b{2}; + policy.ConfigureServerRxTiming(a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), + 99); + policy.ConfigureServerRxTiming(b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), + 95); + policy.ConfirmServerPong(a, Tp(0), Tp(100), Dur(1000), Dur(300)); + TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(b, Tp(50))); + TEST_ASSERT_TRUE(policy.IsServerLocallyOnline(a, Tp(50))); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(50))); + + auto* sa = policy.FindServerPresence(a); + auto* sb = policy.FindServerPresence(b); + TEST_ASSERT_EQUAL(1000, ToMs(sa->desired.interval)); + TEST_ASSERT_EQUAL(3000, ToMs(sb->desired.interval)); + TEST_ASSERT_EQUAL(99, sa->rtt_reliability_percentile); + TEST_ASSERT_EQUAL(95, sb->rtt_reliability_percentile); + + auto const Ra = Dur(100); + auto const Rb = Dur(200); + TEST_ASSERT_EQUAL(870, ToMs(ComputePrefix1Time(Tp(1050), Ra))); + TEST_ASSERT_EQUAL(2720, ToMs(ComputePrefix1Time(Tp(3050), Rb))); +} + +void test_OfflineOnlyAfterWindowClose() { + ClientConnectivityPolicy policy; + ServerId const sid{3}; + policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(200)); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(1220))); + TEST_ASSERT_FALSE(policy.IsLocallyOnline(Tp(1221))); +} + +void test_RuntimeIntervalChangeKeepsOldConfirmed() { + ClientConnectivityPolicy policy; + ServerId const sid{4}; + policy.ConfigureServerRxTiming(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(200))); + policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(200)); + auto const close_before = policy.FindServerPresence(sid)->confirmed_window_close_local; + + policy.ConfigureServerRxTiming(sid, RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200))); + auto* state = policy.FindServerPresence(sid); + TEST_ASSERT_TRUE(state->config_change_pending); + TEST_ASSERT_EQUAL(10000, ToMs(state->desired.interval)); + TEST_ASSERT_TRUE(state->confirmed_window_close_local == close_before); + TEST_ASSERT_EQUAL(1000, ToMs(state->confirmed_interval)); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(close_before)); + + policy.ConfirmServerPong(sid, Tp(500), Tp(560), Dur(10000), Dur(200)); + state = policy.FindServerPresence(sid); + TEST_ASSERT_FALSE(state->config_change_pending); + TEST_ASSERT_EQUAL(10000, ToMs(state->confirmed_interval)); + + policy.ConfigureServerRxTiming(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(200))); + TEST_ASSERT_EQUAL(10000, ToMs(policy.FindServerPresence(sid)->confirmed_interval)); + policy.ConfirmServerPong(sid, Tp(20000), Tp(20040), Dur(1000), Dur(200)); + TEST_ASSERT_EQUAL(1000, ToMs(policy.FindServerPresence(sid)->confirmed_interval)); +} + +void test_RecoveryAfterOffline() { + ClientConnectivityPolicy policy; + ServerId const sid{5}; + policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(100)); + TEST_ASSERT_FALSE(policy.IsLocallyOnline(Tp(100000))); + policy.ConfirmServerPong(sid, Tp(100100), Tp(100140), Dur(1000), Dur(100)); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(100140))); +} + +void test_RuntimePercentile() { + StatisticsCounter stats; + for (int i = 1; i <= 20; ++i) { + stats.Add(i * 10); + } + auto const p95 = stats.PercentileValue(95); + auto const p99 = stats.PercentileValue(99); + TEST_ASSERT_TRUE(p99 >= p95); + TEST_ASSERT_EQUAL(stats.percentile<95>(), p95); + TEST_ASSERT_EQUAL(stats.percentile<99>(), p99); +} + +void test_ReliabilityP95VsP99PrefixTimes() { + StatisticsCounter stats; + for (int i = 1; i <= 20; ++i) { + stats.Add(Dur(i * 10)); + } + auto const p95 = stats.PercentileValue(95); + auto const p99 = stats.PercentileValue(99); + TEST_ASSERT_TRUE(p99 >= p95); + auto const O = Tp(5000); + auto const p1_95 = ComputePrefix1Time(O, p95); + auto const p1_99 = ComputePrefix1Time(O, p99); + auto const p2_95 = ComputePrefix2Time(O, p95); + auto const p2_99 = ComputePrefix2Time(O, p99); + TEST_ASSERT_TRUE(ToMs(p1_99) <= ToMs(p1_95)); + TEST_ASSERT_TRUE(ToMs(p2_99) <= ToMs(p2_95)); + if (p99 > p95) { + TEST_ASSERT_TRUE(ToMs(p1_99) < ToMs(p1_95)); + TEST_ASSERT_TRUE(ToMs(p2_99) < ToMs(p2_95)); + } +} + +void test_AggregateIgnoresDeselected() { + ClientConnectivityPolicy policy; + ServerId const a{10}; + ServerId const b{11}; + policy.ConfirmServerPong(a, Tp(0), Tp(40), Dur(1000), Dur(200)); + policy.ConfirmServerPong(b, Tp(0), Tp(40), Dur(1000), Dur(200)); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(50))); + policy.SetServerSelectedForAggregate(a, false); + policy.SetServerSelectedForAggregate(b, false); + TEST_ASSERT_FALSE(policy.IsLocallyOnline(Tp(50))); + policy.SetServerSelectedForAggregate(b, true); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(50))); +} + +void test_OneWayProjection() { + TEST_ASSERT_EQUAL(50, ToMs(OneWayFromRtt(Dur(100)))); +} + +void test_MakeConfirmedScheduleDeterministic() { + auto s = MakeConfirmedSchedule(Tp(1000), Tp(1200), Dur(500), Dur(100)); + TEST_ASSERT_EQUAL(1600, ToMs(s.window_open_local)); + TEST_ASSERT_EQUAL(1700, ToMs(s.window_close_local)); +} + +void test_PlanPrefix1FailSchedulesPrefix2() { + auto const plan = PlanAfterFailedAttempt(true, Tp(1050), Tp(1350), + PingAttemptKind::kPrefix1, Tp(870 + 100), + Dur(100)); + TEST_ASSERT_EQUAL(static_cast(PingAttemptKind::kPrefix2), + static_cast(plan.kind)); + TEST_ASSERT_EQUAL(970, ToMs(plan.when)); + TEST_ASSERT_FALSE(plan.mark_offline); +} + +void test_PlanPrefix2FailRetriesWhileOnline() { + auto const plan = PlanAfterFailedAttempt(true, Tp(1050), Tp(1350), + PingAttemptKind::kPrefix2, Tp(1070), + Dur(100)); + TEST_ASSERT_EQUAL(static_cast(PingAttemptKind::kRetry), + static_cast(plan.kind)); + TEST_ASSERT_EQUAL(1170, ToMs(plan.when)); + TEST_ASSERT_FALSE(plan.mark_offline); +} + +void test_PlanAfterCloseIsRecoveryOffline() { + auto const plan = PlanAfterFailedAttempt(true, Tp(1050), Tp(1350), + PingAttemptKind::kRetry, Tp(1351), + Dur(100)); + TEST_ASSERT_EQUAL(static_cast(PingAttemptKind::kRecovery), + static_cast(plan.kind)); + TEST_ASSERT_TRUE(plan.mark_offline); +} + +void test_StaleAttemptRejected() { + TEST_ASSERT_TRUE(IsCurrentPingAttempt(4, 4)); + TEST_ASSERT_FALSE(IsCurrentPingAttempt(5, 4)); +} + +void test_PlanSuccessSchedulesPrefix1() { + auto const plan = PlanAfterSuccessfulPong(Tp(1050), Tp(100), Dur(100), false); + TEST_ASSERT_EQUAL(static_cast(PingAttemptKind::kPrefix1), + static_cast(plan.kind)); + TEST_ASSERT_EQUAL(870, ToMs(plan.when)); +} + +struct AttemptLog { + ServerId server{}; + std::uint64_t attempt_id{}; + PingAttemptKind kind{PingAttemptKind::kInitial}; + TimePoint send_time{}; +}; + +struct PollStats { + int status_poll_count{}; + int online_samples{}; + int false_offline_samples{}; + int false_offline_transitions{}; + Duration max_false_offline_duration{}; + Duration current_false_offline_duration{}; + bool prev_online{false}; + bool have_prev{false}; +}; + +struct SimServer { + ServerId id{}; + Duration schedule_rtt{Dur(100)}; + Duration pong_delay{Dur(20)}; + bool connectivity_ok{true}; + bool stopped{false}; + bool transport_open{true}; + bool in_flight{false}; + int fail_next_attempts{0}; + std::uint64_t active_attempt_id{0}; + PingAttemptKind next_kind{PingAttemptKind::kInitial}; + TimePoint next_due{}; + TimePoint inflight_send{}; + TimePoint inflight_timeout_at{}; + TimePoint inflight_pong_at{}; + Duration inflight_interval{}; + Duration inflight_window{}; + PingAttemptKind inflight_kind{PingAttemptKind::kInitial}; + std::uint64_t inflight_id{0}; + bool inflight_should_fail{false}; +}; + +struct SimCounters { + int prefix1{}; + int prefix2{}; + int retry{}; + int recovery{}; + int initial{}; + int timeouts{}; + int confirmed_pongs{}; + int recoveries_to_online{}; +}; + +class LocalPresenceRuntime { + public: + explicit LocalPresenceRuntime(TimePoint start) : now_{start} {} + + ClientConnectivityPolicy& policy() { return policy_; } + TimePoint now() const { return now_; } + SimCounters const& counters() const { return counters_; } + PollStats const& poll_stats() const { return poll_stats_; } + std::vector const& attempts() const { return attempts_; } + SimServer& server(ServerId id) { return servers_.at(id); } + + bool IsLocallyOnline() const { return policy_.IsLocallyOnline(now_); } + + void AddServer(ServerId id, RxTimingConf conf, Duration rtt, + std::uint8_t percentile = kDefaultRttReliabilityPercentile) { + policy_.ConfigureServerRxTiming(id, conf, percentile); + SimServer s{}; + s.id = id; + s.schedule_rtt = rtt; + s.next_due = now_; + s.next_kind = PingAttemptKind::kInitial; + servers_.emplace(id, s); + } + + void Quarantine(ServerId id) { + auto& s = servers_.at(id); + s.stopped = true; + s.in_flight = false; + s.next_due = TimePoint::max(); + policy_.SetServerSelectedForAggregate(id, false); + policy_.InvalidateConfirmedSchedule(id); + } + + void Release(ServerId id) { + auto& s = servers_.at(id); + s.stopped = false; + s.fail_next_attempts = 0; + s.connectivity_ok = true; + policy_.SetServerSelectedForAggregate(id, true); + s.next_due = now_; + s.next_kind = PingAttemptKind::kInitial; + } + + void Poll(bool expected_connected) { + auto const online = IsLocallyOnline(); + ++poll_stats_.status_poll_count; + if (online) { + ++poll_stats_.online_samples; + poll_stats_.current_false_offline_duration = {}; + } + if (expected_connected && !online) { + ++poll_stats_.false_offline_samples; + poll_stats_.current_false_offline_duration = + poll_stats_.current_false_offline_duration + Dur(10); + if (poll_stats_.current_false_offline_duration > + poll_stats_.max_false_offline_duration) { + poll_stats_.max_false_offline_duration = + poll_stats_.current_false_offline_duration; + } + if (poll_stats_.have_prev && poll_stats_.prev_online) { + ++poll_stats_.false_offline_transitions; + } + } + poll_stats_.prev_online = online; + poll_stats_.have_prev = true; + } + + void ProcessDue() { + bool progress = true; + while (progress) { + progress = false; + for (auto& [id, s] : servers_) { + static_cast(id); + if (s.stopped || !s.in_flight) { + continue; + } + if (!s.inflight_should_fail && now_ >= s.inflight_pong_at) { + CompleteSuccess(s); + progress = true; + } + } + for (auto& [id, s] : servers_) { + static_cast(id); + if (s.stopped || !s.in_flight) { + continue; + } + if (now_ >= s.inflight_timeout_at) { + CompleteTimeout(s); + progress = true; + } + } + for (auto& [id, s] : servers_) { + static_cast(id); + if (s.stopped || s.in_flight) { + continue; + } + if (now_ >= s.next_due) { + StartSend(s); + progress = true; + } + } + } + } + + void AdvancePolling(Duration total, Duration step, bool expected_connected) { + auto const end = now_ + total; + ProcessDue(); + while (now_ < end) { + auto const next_poll = now_ + step; + for (;;) { + auto const ev = NextEventTime(); + if (ev == TimePoint::max() || ev > next_poll) { + break; + } + if (ev > now_) { + now_ = ev; + } + ProcessDue(); + if (NextEventTime() <= now_) { + break; + } + } + now_ = next_poll; + ProcessDue(); + Poll(expected_connected); + } + } + + void AdvanceTo(TimePoint t) { + if (t < now_) { + return; + } + for (;;) { + ProcessDue(); + if (now_ >= t) { + return; + } + auto const next = NextEventTime(); + if (next == TimePoint::max() || next > t) { + now_ = t; + ProcessDue(); + return; + } + if (next <= now_) { + now_ = t; + ProcessDue(); + return; + } + now_ = next; + } + } + + private: + TimePoint NextEventTime() const { + auto next = TimePoint::max(); + for (auto const& [id, s] : servers_) { + static_cast(id); + if (s.stopped) { + continue; + } + if (s.in_flight) { + if (!s.inflight_should_fail) { + next = std::min(next, s.inflight_pong_at); + } + next = std::min(next, s.inflight_timeout_at); + } else { + next = std::min(next, s.next_due); + } + } + return next; + } + void CountKind(PingAttemptKind kind) { + switch (kind) { + case PingAttemptKind::kPrefix1: + ++counters_.prefix1; + break; + case PingAttemptKind::kPrefix2: + ++counters_.prefix2; + break; + case PingAttemptKind::kRetry: + ++counters_.retry; + break; + case PingAttemptKind::kRecovery: + ++counters_.recovery; + break; + case PingAttemptKind::kInitial: + ++counters_.initial; + break; + } + } + + void StartSend(SimServer& s) { + auto& presence = policy_.EnsureServerPresence(s.id); + ++s.active_attempt_id; + presence.current_attempt_id = s.active_attempt_id; + presence.current_attempt_kind = s.next_kind; + s.in_flight = true; + s.inflight_id = s.active_attempt_id; + s.inflight_kind = s.next_kind; + s.inflight_send = now_; + s.inflight_interval = presence.desired.interval; + s.inflight_window = presence.desired.rx_window; + s.inflight_timeout_at = now_ + s.schedule_rtt; + s.inflight_pong_at = now_ + s.pong_delay; + s.inflight_should_fail = !s.connectivity_ok || (s.fail_next_attempts > 0); + if (s.fail_next_attempts > 0) { + --s.fail_next_attempts; + } + CountKind(s.next_kind); + attempts_.push_back(AttemptLog{s.id, s.inflight_id, s.inflight_kind, now_}); + } + + void CompleteSuccess(SimServer& s) { + auto const was_offline = !policy_.IsServerLocallyOnline(s.id, now_); + s.in_flight = false; + policy_.ConfirmServerPong(s.id, s.inflight_send, now_, s.inflight_interval, + s.inflight_window); + ++counters_.confirmed_pongs; + if (was_offline) { + ++counters_.recoveries_to_online; + } + auto* presence = policy_.FindServerPresence(s.id); + auto const plan = PlanAfterSuccessfulPong( + presence->confirmed_window_open_local, now_, s.schedule_rtt, + presence->config_change_pending); + s.next_due = plan.when; + s.next_kind = plan.kind; + } + + void CompleteTimeout(SimServer& s) { + s.in_flight = false; + ++s.active_attempt_id; + auto* presence = policy_.FindServerPresence(s.id); + if (presence != nullptr) { + presence->current_attempt_id = s.active_attempt_id; + } + ++counters_.timeouts; + auto const plan = PlanAfterFailedAttempt( + presence != nullptr && presence->has_confirmed_schedule, + presence != nullptr ? presence->confirmed_window_open_local : TimePoint{}, + presence != nullptr ? presence->confirmed_window_close_local : TimePoint{}, + s.inflight_kind, now_, s.schedule_rtt); + if (plan.mark_offline && presence != nullptr) { + policy_.MarkServerOffline(s.id, now_); + } + s.next_due = plan.when; + s.next_kind = plan.kind; + } + + ClientConnectivityPolicy policy_; + TimePoint now_{}; + std::map servers_; + std::vector attempts_; + SimCounters counters_{}; + PollStats poll_stats_{}; +}; + +void test_SendWithoutPongDoesNotConfirm() { + LocalPresenceRuntime rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100)); + rt.server(sid).connectivity_ok = false; + rt.AdvanceTo(Tp(0)); + TEST_ASSERT_EQUAL(1, rt.counters().initial); + TEST_ASSERT_FALSE(rt.IsLocallyOnline()); + TEST_ASSERT_FALSE(rt.policy().FindServerPresence(sid)->has_confirmed_schedule); +} + +void test_Prefix1SuccessCancelsPrefix2() { + LocalPresenceRuntime rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100)); + rt.server(sid).pong_delay = Dur(100); + rt.AdvanceTo(Tp(100)); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + auto* st = rt.policy().FindServerPresence(sid); + TEST_ASSERT_EQUAL(1050, ToMs(st->confirmed_window_open_local)); + auto const prefix1 = ComputePrefix1Time(st->confirmed_window_open_local, Dur(100)); + auto const prefix2 = ComputePrefix2Time(st->confirmed_window_open_local, Dur(100)); + TEST_ASSERT_EQUAL(870, ToMs(prefix1)); + TEST_ASSERT_EQUAL(970, ToMs(prefix2)); + + rt.AdvanceTo(prefix1); + TEST_ASSERT_EQUAL(1, rt.counters().prefix1); + rt.AdvanceTo(prefix1 + Dur(20)); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + TEST_ASSERT_EQUAL(0, rt.counters().prefix2); + rt.AdvanceTo(prefix2 + Dur(5)); + TEST_ASSERT_EQUAL(0, rt.counters().prefix2); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_Prefix1FailSendsPrefix2OnTarget() { + LocalPresenceRuntime rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100)); + rt.server(sid).pong_delay = Dur(100); + rt.AdvanceTo(Tp(100)); + auto* st = rt.policy().FindServerPresence(sid); + auto const prefix1 = ComputePrefix1Time(st->confirmed_window_open_local, Dur(100)); + auto const prefix2 = ComputePrefix2Time(st->confirmed_window_open_local, Dur(100)); + rt.server(sid).fail_next_attempts = 1; + rt.AdvanceTo(prefix1); + TEST_ASSERT_EQUAL(1, rt.counters().prefix1); + TEST_ASSERT_EQUAL(0, rt.counters().prefix2); + rt.AdvanceTo(prefix1 + Dur(100)); + TEST_ASSERT_EQUAL(1, rt.counters().prefix2); + TEST_ASSERT_EQUAL(970, ToMs(rt.attempts().back().send_time)); + TEST_ASSERT_EQUAL(prefix2.time_since_epoch().count(), + rt.attempts().back().send_time.time_since_epoch().count()); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_Prefix2SuccessNoOffline() { + LocalPresenceRuntime rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100)); + rt.server(sid).pong_delay = Dur(100); + rt.AdvanceTo(Tp(100)); + auto* st = rt.policy().FindServerPresence(sid); + auto const prefix1 = ComputePrefix1Time(st->confirmed_window_open_local, Dur(100)); + rt.server(sid).fail_next_attempts = 1; + rt.AdvanceTo(prefix1 + Dur(100)); + TEST_ASSERT_EQUAL(1, rt.counters().prefix2); + rt.AdvanceTo(prefix1 + Dur(120)); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + TEST_ASSERT_EQUAL(0, rt.poll_stats().false_offline_transitions); +} + +void test_PostPrefixRetriesStayOnline() { + LocalPresenceRuntime rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(400)), Dur(100)); + rt.server(sid).pong_delay = Dur(100); + rt.AdvanceTo(Tp(100)); + auto* st = rt.policy().FindServerPresence(sid); + auto const close = st->confirmed_window_close_local; + rt.server(sid).connectivity_ok = false; + auto const prefix1 = ComputePrefix1Time(st->confirmed_window_open_local, Dur(100)); + rt.AdvanceTo(prefix1); + rt.AdvanceTo(close); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + TEST_ASSERT_TRUE(rt.counters().prefix2 >= 1); + TEST_ASSERT_TRUE(rt.counters().retry >= 1); +} + +void test_RealOfflineAfterWindowClose() { + LocalPresenceRuntime rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100)); + rt.server(sid).pong_delay = Dur(100); + rt.AdvanceTo(Tp(100)); + auto* st = rt.policy().FindServerPresence(sid); + auto const close = st->confirmed_window_close_local; + rt.server(sid).connectivity_ok = false; + rt.AdvanceTo(close); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + rt.AdvanceTo(close + Dur(1)); + TEST_ASSERT_FALSE(rt.IsLocallyOnline()); +} + +void test_RecoveryPongRestoresOnline() { + LocalPresenceRuntime rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(200)), Dur(100)); + rt.server(sid).pong_delay = Dur(100); + rt.AdvanceTo(Tp(100)); + auto* st = rt.policy().FindServerPresence(sid); + rt.server(sid).connectivity_ok = false; + rt.AdvanceTo(st->confirmed_window_close_local + Dur(1)); + TEST_ASSERT_FALSE(rt.IsLocallyOnline()); + rt.server(sid).connectivity_ok = true; + rt.AdvanceTo(rt.now() + Dur(500)); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_RxWindowCloseDoesNotCloseTransport() { + LocalPresenceRuntime rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(200)), Dur(100)); + rt.server(sid).pong_delay = Dur(100); + rt.AdvanceTo(Tp(100)); + auto* st = rt.policy().FindServerPresence(sid); + rt.server(sid).connectivity_ok = false; + rt.AdvanceTo(st->confirmed_window_close_local + Dur(50)); + TEST_ASSERT_FALSE(rt.IsLocallyOnline()); + TEST_ASSERT_TRUE(rt.server(sid).transport_open); +} + +void test_QuarantineIndependentAndReleaseRestartsPing() { + LocalPresenceRuntime rt{Tp(0)}; + ServerId const a{1}; + ServerId const b{2}; + rt.AddServer(a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100), 99); + rt.AddServer(b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), Dur(200), 95); + rt.AdvanceTo(Tp(20)); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(a, rt.now())); + TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(b, rt.now())); + + rt.Quarantine(a); + TEST_ASSERT_FALSE(rt.policy().IsServerLocallyOnline(a, rt.now())); + TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(b, rt.now())); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + + auto const prefix1_before = rt.counters().prefix1; + rt.AdvanceTo(rt.now() + Dur(200)); + TEST_ASSERT_EQUAL(prefix1_before, rt.counters().prefix1); + + rt.Release(a); + rt.AdvanceTo(rt.now() + Dur(40)); + TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(a, rt.now())); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_RuntimeConfigChangeKeepsOldUntilPong() { + LocalPresenceRuntime rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(200)), Dur(100)); + rt.AdvanceTo(Tp(20)); + auto const close_old = rt.policy().FindServerPresence(sid)->confirmed_window_close_local; + rt.policy().ConfigureServerRxTiming( + sid, RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200))); + TEST_ASSERT_EQUAL(1000, ToMs(rt.policy().FindServerPresence(sid)->confirmed_interval)); + TEST_ASSERT_TRUE(rt.policy().FindServerPresence(sid)->confirmed_window_close_local == + close_old); + rt.server(sid).connectivity_ok = false; + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +struct RuntimeReport { + int confirmed_cycles{}; + Duration duration{}; + TimePoint fault_time{}; + TimePoint confirmed_window_close{}; + TimePoint detected_offline_time{}; + TimePoint recovered_time{}; + bool early_offline{false}; +}; + +RuntimeReport g_stat_report{}; +RuntimeReport g_fault_report{}; + +void test_StatisticalRuntimePollingIsLocallyOnline() { + LocalPresenceRuntime rt{Tp(0)}; + ServerId const sid{1}; + auto const interval = Dur(1000); + auto const window = Dur(400); + auto const rtt = Dur(100); + rt.AddServer(sid, RxTimingConf::Every(interval).WithWindow(window), rtt, 99); + rt.server(sid).pong_delay = Dur(20); + + rt.AdvanceTo(Tp(20)); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + auto const measure_start = rt.now(); + auto const min_duration = Dur(300000); + constexpr int kMinCycles = 300; + + while (true) { + rt.AdvancePolling(Dur(10), Dur(10), true); + auto const elapsed_ms = + std::chrono::duration_cast(rt.now() - measure_start).count(); + if (elapsed_ms >= 300000 && rt.counters().confirmed_pongs >= kMinCycles) { + break; + } + TEST_ASSERT_TRUE(elapsed_ms < 400000); + } + + g_stat_report.confirmed_cycles = rt.counters().confirmed_pongs; + g_stat_report.duration = + std::chrono::duration_cast(rt.now() - measure_start); + + std::printf( + "STATISTICAL runtime\n" + " duration_ms=%lld confirmed_pongs=%d status_polls=%d online_samples=%d\n" + " false_offline_samples=%d false_offline_transitions=%d " + "max_false_offline_ms=%lld\n" + " prefix1=%d prefix2=%d post_prefix_retry=%d timeouts=%d recoveries=%d\n" + " rtt_percentile=99 selected_rtt_ms=%lld guard_ms=30\n", + static_cast(ToMs(g_stat_report.duration)), + rt.counters().confirmed_pongs, rt.poll_stats().status_poll_count, + rt.poll_stats().online_samples, rt.poll_stats().false_offline_samples, + rt.poll_stats().false_offline_transitions, + static_cast(ToMs(rt.poll_stats().max_false_offline_duration)), + rt.counters().prefix1, rt.counters().prefix2, rt.counters().retry, + rt.counters().timeouts, rt.counters().recoveries_to_online, + static_cast(ToMs(rtt))); + + TEST_ASSERT_EQUAL(0, rt.poll_stats().false_offline_samples); + TEST_ASSERT_EQUAL(0, rt.poll_stats().false_offline_transitions); + TEST_ASSERT_TRUE(rt.counters().confirmed_pongs >= kMinCycles); + TEST_ASSERT_TRUE(rt.counters().prefix2 == 0); +} + +void test_FaultOfflineNotBeforeWindowCloseThenRecovery() { + LocalPresenceRuntime rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(400)), Dur(100)); + rt.server(sid).pong_delay = Dur(20); + rt.AdvanceTo(Tp(20)); + rt.AdvancePolling(Dur(2000), Dur(10), true); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + auto* st = rt.policy().FindServerPresence(sid); + auto const close = st->confirmed_window_close_local; + g_fault_report.fault_time = rt.now(); + g_fault_report.confirmed_window_close = close; + rt.server(sid).connectivity_ok = false; + + auto detected = TimePoint{}; + while (rt.now() < close + Dur(2000)) { + rt.AdvancePolling(Dur(10), Dur(10), false); + if (!rt.IsLocallyOnline()) { + detected = rt.now(); + g_fault_report.early_offline = !(detected > close); + break; + } + } + g_fault_report.detected_offline_time = detected; + TEST_ASSERT_FALSE(g_fault_report.early_offline); + TEST_ASSERT_TRUE(detected > close); + TEST_ASSERT_TRUE(ToMs(detected) >= ToMs(close)); + + rt.server(sid).connectivity_ok = true; + auto const recover_from = rt.now(); + while (rt.now() < recover_from + Dur(2000)) { + rt.AdvancePolling(Dur(10), Dur(10), false); + if (rt.IsLocallyOnline()) { + g_fault_report.recovered_time = rt.now(); + break; + } + } + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + auto const recovery_ms = ToMs(g_fault_report.recovered_time) - ToMs(recover_from); + std::printf( + "FAULT runtime\n" + " fault_ms=%lld window_close_ms=%lld detected_offline_ms=%lld\n" + " early_offline=%s recovery_latency_ms=%lld\n", + static_cast(ToMs(g_fault_report.fault_time)), + static_cast(ToMs(close)), + static_cast(ToMs(detected)), + g_fault_report.early_offline ? "YES" : "NO", + static_cast(recovery_ms)); +} + +void test_MultiServerRuntimeIndependence() { + LocalPresenceRuntime rt{Tp(0)}; + ServerId const a{1}; + ServerId const b{2}; + rt.AddServer(a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100), 99); + rt.AddServer(b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), Dur(200), 95); + rt.AdvanceTo(Tp(40)); + auto* sa = rt.policy().FindServerPresence(a); + auto* sb = rt.policy().FindServerPresence(b); + TEST_ASSERT_EQUAL(1000, ToMs(sa->confirmed_interval)); + TEST_ASSERT_EQUAL(3000, ToMs(sb->confirmed_interval)); + TEST_ASSERT_EQUAL(300, ToMs(sa->confirmed_rx_window)); + TEST_ASSERT_EQUAL(700, ToMs(sb->confirmed_rx_window)); + auto const p1a = ComputePrefix1Time(sa->confirmed_window_open_local, Dur(100)); + auto const p1b = ComputePrefix1Time(sb->confirmed_window_open_local, Dur(200)); + TEST_ASSERT_TRUE(ToMs(p1a) != ToMs(p1b)); + rt.server(a).connectivity_ok = false; + rt.AdvanceTo(sa->confirmed_window_close_local + Dur(1)); + TEST_ASSERT_FALSE(rt.policy().IsServerLocallyOnline(a, rt.now())); + TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(b, rt.now())); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +} // namespace ae::test_local_presence + +void setUp() {} +void tearDown() {} + +int main() { + UNITY_BEGIN(); + RUN_TEST(ae::test_local_presence::test_PrefixFormula); + RUN_TEST(ae::test_local_presence::test_ConfirmOnlyAfterPong); + RUN_TEST(ae::test_local_presence::test_PerServerIndependence); + RUN_TEST(ae::test_local_presence::test_OfflineOnlyAfterWindowClose); + RUN_TEST(ae::test_local_presence::test_RuntimeIntervalChangeKeepsOldConfirmed); + RUN_TEST(ae::test_local_presence::test_RecoveryAfterOffline); + RUN_TEST(ae::test_local_presence::test_RuntimePercentile); + RUN_TEST(ae::test_local_presence::test_ReliabilityP95VsP99PrefixTimes); + RUN_TEST(ae::test_local_presence::test_AggregateIgnoresDeselected); + RUN_TEST(ae::test_local_presence::test_OneWayProjection); + RUN_TEST(ae::test_local_presence::test_MakeConfirmedScheduleDeterministic); + RUN_TEST(ae::test_local_presence::test_PlanPrefix1FailSchedulesPrefix2); + RUN_TEST(ae::test_local_presence::test_PlanPrefix2FailRetriesWhileOnline); + RUN_TEST(ae::test_local_presence::test_PlanAfterCloseIsRecoveryOffline); + RUN_TEST(ae::test_local_presence::test_StaleAttemptRejected); + RUN_TEST(ae::test_local_presence::test_PlanSuccessSchedulesPrefix1); + RUN_TEST(ae::test_local_presence::test_SendWithoutPongDoesNotConfirm); + RUN_TEST(ae::test_local_presence::test_Prefix1SuccessCancelsPrefix2); + RUN_TEST(ae::test_local_presence::test_Prefix1FailSendsPrefix2OnTarget); + RUN_TEST(ae::test_local_presence::test_Prefix2SuccessNoOffline); + RUN_TEST(ae::test_local_presence::test_PostPrefixRetriesStayOnline); + RUN_TEST(ae::test_local_presence::test_RealOfflineAfterWindowClose); + RUN_TEST(ae::test_local_presence::test_RecoveryPongRestoresOnline); + RUN_TEST(ae::test_local_presence::test_RxWindowCloseDoesNotCloseTransport); + RUN_TEST(ae::test_local_presence::test_QuarantineIndependentAndReleaseRestartsPing); + RUN_TEST(ae::test_local_presence::test_RuntimeConfigChangeKeepsOldUntilPong); + RUN_TEST(ae::test_local_presence::test_StatisticalRuntimePollingIsLocallyOnline); + RUN_TEST(ae::test_local_presence::test_FaultOfflineNotBeforeWindowCloseThenRecovery); + RUN_TEST(ae::test_local_presence::test_MultiServerRuntimeIndependence); + return UNITY_END(); +} From 2a2f52341745a5e69e2eb42a9be54a529ac518ff Mon Sep 17 00:00:00 2001 From: aethernet-io Date: Wed, 2 Sep 2026 16:45:17 -0700 Subject: [PATCH 02/11] Add 1s/1s local presence timings and a Windows Firewall live test. Co-authored-by: Cursor --- tests/test-local-presence/CMakeLists.txt | 13 + tests/test-local-presence/firewall_live.cpp | 314 ++++++++++++++++++++ tests/test-local-presence/main.cpp | 4 +- 3 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 tests/test-local-presence/firewall_live.cpp diff --git a/tests/test-local-presence/CMakeLists.txt b/tests/test-local-presence/CMakeLists.txt index cfd9e9e7..df6cc3ad 100644 --- a/tests/test-local-presence/CMakeLists.txt +++ b/tests/test-local-presence/CMakeLists.txt @@ -24,6 +24,19 @@ if(NOT CM_PLATFORM) target_compile_options(${PROJECT_NAME} PRIVATE /Zc:preprocessor) endif() add_test(NAME ${PROJECT_NAME} COMMAND $) + + add_executable(test-local-presence-firewall firewall_live.cpp) + target_include_directories(test-local-presence-firewall PRIVATE ${ROOT_DIR}) + target_link_libraries(test-local-presence-firewall PRIVATE unity aether) + target_compile_definitions(test-local-presence-firewall PRIVATE + "AE_DISTILLATION=1" + ) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(test-local-presence-firewall PRIVATE /Zc:preprocessor) + endif() + add_test(NAME test-local-presence-firewall + COMMAND $) + set_tests_properties(test-local-presence-firewall PROPERTIES TIMEOUT 90) else() message(WARNING "Not implemented for ${CM_PLATFORM}") endif() diff --git a/tests/test-local-presence/firewall_live.cpp b/tests/test-local-presence/firewall_live.cpp new file mode 100644 index 00000000..f98ed422 --- /dev/null +++ b/tests/test-local-presence/firewall_live.cpp @@ -0,0 +1,314 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include + +#include "aether/adapters/ethernet.h" +#include "aether/aether_app.h" +#include "aether/all.h" +#include "aether/client.h" +#include "aether/client_connectivity_policy.h" +#include "aether/cloud_connections/cloud_server_connection.h" +#include "aether/global_ids.h" +#include "aether/types/uid.h" + +#if defined(_WIN32) +# include +#endif + +namespace ae::test_local_presence_firewall { + +using namespace std::chrono_literals; + +constexpr auto kInterval = 1s; +constexpr auto kWindow = 1s; +constexpr auto kPoll = 10ms; + +static constexpr auto kParentUid = + Uid::FromString("3ac93165-3d37-4970-87a6-fa4ee27744e4"); + +#if defined(_WIN32) + +std::wstring ThisExePath() { + wchar_t path[MAX_PATH]{}; + auto const n = GetModuleFileNameW(nullptr, path, MAX_PATH); + if (n == 0 || n >= MAX_PATH) { + return {}; + } + return std::wstring{path, static_cast(n)}; +} + +int RunHidden(std::wstring cmd) { + STARTUPINFOW si{}; + si.cb = sizeof(si); + si.dwFlags = STARTF_USESHOWWINDOW; + si.wShowWindow = SW_HIDE; + PROCESS_INFORMATION pi{}; + if (!CreateProcessW(nullptr, cmd.data(), nullptr, nullptr, FALSE, + CREATE_NO_WINDOW, nullptr, nullptr, &si, &pi)) { + return -1; + } + WaitForSingleObject(pi.hProcess, 20000); + DWORD code = 1; + GetExitCodeProcess(pi.hProcess, &code); + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + return static_cast(code); +} + +class WindowsExeFirewall { + public: + explicit WindowsExeFirewall(std::wstring exe_path) + : exe_path_{std::move(exe_path)}, + tag_{std::to_wstring(GetCurrentProcessId())} {} + + ~WindowsExeFirewall() { Unblock(); } + + WindowsExeFirewall(WindowsExeFirewall const&) = delete; + WindowsExeFirewall& operator=(WindowsExeFirewall const&) = delete; + + bool Block() { + Unblock(); + auto const quoted = L"\"" + exe_path_ + L"\""; + out_name_ = L"ae-lp-fw-out-" + tag_; + in_name_ = L"ae-lp-fw-in-" + tag_; + auto const out_cmd = + L"netsh advfirewall firewall add rule name=\"" + out_name_ + + L"\" dir=out action=block enable=yes profile=any program=" + quoted; + auto const in_cmd = + L"netsh advfirewall firewall add rule name=\"" + in_name_ + + L"\" dir=in action=block enable=yes profile=any program=" + quoted; + if (RunHidden(out_cmd) != 0 || RunHidden(in_cmd) != 0) { + Unblock(); + return false; + } + active_ = true; + return true; + } + + void Unblock() { + if (!out_name_.empty()) { + RunHidden(L"netsh advfirewall firewall delete rule name=\"" + out_name_ + + L"\""); + } + if (!in_name_.empty()) { + RunHidden(L"netsh advfirewall firewall delete rule name=\"" + in_name_ + + L"\""); + } + active_ = false; + } + + private: + std::wstring exe_path_; + std::wstring tag_; + std::wstring out_name_; + std::wstring in_name_; + bool active_{false}; +}; + +#endif + +std::unique_ptr MakeApp() { + return AetherApp::Construct(AetherAppContext{}.AddAdapterFactory( + [](AetherAppContext const& context) { + return EthernetAdapter::ptr::Create( + CreateWith{context.domain()}.with_id(GlobalId::kEthernetAdapter), + context.aether(), context.poller(), context.dns_resolver()); + })); +} + +void Pump(AetherApp& app, TimePoint until) { + while (!app.IsExited() && Now() < until) { + auto const next = app.Update(Now()); + auto const poll_at = Now() + kPoll; + app.WaitUntil(next < poll_at ? next : poll_at); + } +} + +TimePoint EarliestConfirmedClose(Client& client) { + auto close = TimePoint::max(); + auto policy = client.connectivity_policy(); + if (!policy) { + return close; + } + for (auto* server : client.cloud_connection().selected_servers()) { + if (server == nullptr) { + continue; + } + auto const* state = policy->FindServerPresence(server->server_id()); + if (state != nullptr && state->has_confirmed_schedule) { + close = std::min(close, state->confirmed_window_close_local); + } + } + return close; +} + +void ApplyOneSecondTimings(Client& client) { + auto policy = client.connectivity_policy(); + TEST_ASSERT_TRUE(static_cast(policy)); + policy->ResetRxTimings(); + policy->ConfigureRxTimings(RequestPolicy::All{}) + .ForAllPriorities(RxTimingConf::Every(kInterval).WithWindow(kWindow)); + for (auto* server : client.cloud_connection().selected_servers()) { + if (server == nullptr) { + continue; + } + policy->ConfigureServerRxTiming( + server->server_id(), + RxTimingConf::Every(kInterval).WithWindow(kWindow), 99); + } +} + +void test_WindowsFirewallOfflineAndRecovery() { +#if !defined(_WIN32) + TEST_IGNORE_MESSAGE("Windows Firewall test runs on Win32 only"); +#else + auto app = MakeApp(); + TEST_ASSERT_NOT_NULL(app.get()); + + Client::ptr client; + auto& select = app->aether()->SelectClient(kParentUid, "presence-fw"); + select.result_event().Subscribe([&](auto const& res) { + if (res) { + client = res.value(); + } + }); + Pump(*app, Now() + 45s); + if (!client) { + TEST_IGNORE_MESSAGE("SelectClient did not finish (no cloud / network)"); + } + + ApplyOneSecondTimings(*client.Load()); + (void)client->cloud_connection(); + + auto const online_deadline = Now() + 30s; + while (Now() < online_deadline && !app->IsExited()) { + Pump(*app, Now() + kPoll); + if (client->IsLocallyOnline()) { + break; + } + } + TEST_ASSERT_TRUE_MESSAGE(client->IsLocallyOnline(), + "did not become ONLINE before firewall"); + Pump(*app, Now() + 1500ms); + TEST_ASSERT_TRUE(client->IsLocallyOnline()); + + auto const close = EarliestConfirmedClose(*client.Load()); + TEST_ASSERT_TRUE_MESSAGE(close != TimePoint::max(), + "no confirmed receive window"); + + WindowsExeFirewall fw{ThisExePath()}; + auto const fault_time = Now(); + TEST_ASSERT_TRUE_MESSAGE( + fw.Block(), + "netsh advfirewall failed (run the test as Administrator)"); + + bool early_offline = false; + TimePoint detected_offline{}; + auto const detect_deadline = close + 3s; + while (Now() < detect_deadline && !app->IsExited()) { + Pump(*app, Now() + kPoll); + auto const online = client->IsLocallyOnline(); + auto const now = Now(); + if (now <= close) { + if (!online) { + early_offline = true; + detected_offline = now; + break; + } + } else if (!online) { + detected_offline = now; + break; + } + } + + auto const close_ms = + std::chrono::duration_cast(close - fault_time) + .count(); + auto const detected_ms = + detected_offline.time_since_epoch().count() == 0 + ? -1 + : std::chrono::duration_cast( + detected_offline - fault_time) + .count(); + std::printf( + "FIREWALL fault interval_ms=1000 rx_window_ms=1000 " + "fault_to_window_close_ms=%lld detected_offline_after_fault_ms=%lld " + "early_offline=%s\n", + static_cast(close_ms), static_cast(detected_ms), + early_offline ? "YES" : "NO"); + if (FILE* log = std::fopen("firewall_result.txt", "w")) { + std::fprintf( + log, + "interval_ms=1000\nrx_window_ms=1000\n" + "fault_to_window_close_ms=%lld\ndetected_offline_after_fault_ms=%lld\n" + "early_offline=%s\n", + static_cast(close_ms), static_cast(detected_ms), + early_offline ? "YES" : "NO"); + std::fclose(log); + } + + TEST_ASSERT_FALSE_MESSAGE(early_offline, + "OFFLINE appeared before confirmed_window_close"); + TEST_ASSERT_TRUE_MESSAGE(detected_offline.time_since_epoch().count() != 0, + "OFFLINE was not detected after firewall block"); + TEST_ASSERT_TRUE(detected_offline > close); + + fw.Unblock(); + auto const recover_from = Now(); + TimePoint recovered{}; + while (Now() < recover_from + 20s && !app->IsExited()) { + Pump(*app, Now() + kPoll); + if (client->IsLocallyOnline()) { + recovered = Now(); + break; + } + } + auto const recovery_ms = + recovered.time_since_epoch().count() == 0 + ? -1 + : std::chrono::duration_cast( + recovered - recover_from) + .count(); + std::printf("FIREWALL recovery_latency_ms=%lld\n", + static_cast(recovery_ms)); + if (FILE* log = std::fopen("firewall_result.txt", "a")) { + std::fprintf(log, "recovery_latency_ms=%lld\npass=1\n", + static_cast(recovery_ms)); + std::fclose(log); + } + TEST_ASSERT_TRUE_MESSAGE(client->IsLocallyOnline(), + "did not return ONLINE after firewall unblock"); +#endif +} + +} // namespace ae::test_local_presence_firewall + +void setUp() {} +void tearDown() {} + +int main() { + UNITY_BEGIN(); + RUN_TEST( + ae::test_local_presence_firewall::test_WindowsFirewallOfflineAndRecovery); + return UNITY_END(); +} diff --git a/tests/test-local-presence/main.cpp b/tests/test-local-presence/main.cpp index b4d065df..7e936571 100644 --- a/tests/test-local-presence/main.cpp +++ b/tests/test-local-presence/main.cpp @@ -740,7 +740,7 @@ void test_StatisticalRuntimePollingIsLocallyOnline() { LocalPresenceRuntime rt{Tp(0)}; ServerId const sid{1}; auto const interval = Dur(1000); - auto const window = Dur(400); + auto const window = Dur(1000); auto const rtt = Dur(100); rt.AddServer(sid, RxTimingConf::Every(interval).WithWindow(window), rtt, 99); rt.server(sid).pong_delay = Dur(20); @@ -790,7 +790,7 @@ void test_StatisticalRuntimePollingIsLocallyOnline() { void test_FaultOfflineNotBeforeWindowCloseThenRecovery() { LocalPresenceRuntime rt{Tp(0)}; ServerId const sid{1}; - rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(400)), Dur(100)); + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), Dur(100)); rt.server(sid).pong_delay = Dur(20); rt.AdvanceTo(Tp(20)); rt.AdvancePolling(Dur(2000), Dur(10), true); From 34c1b8b92c0b617ba20353c2f9135652af025f73 Mon Sep 17 00:00:00 2001 From: aethernet-io Date: Wed, 2 Sep 2026 18:02:29 -0700 Subject: [PATCH 03/11] Fix Local Presence current/next window, selected-RTT cadence, and late Pong handling. Keep RX blockers on the current promised window only, project openings from selected RTT, and bound outstanding Ping attempts so a late Pong still confirms the cycle. Privileged firewall tests are opt-in. Co-authored-by: Cursor --- CMakeLists.txt | 1 + aether/CMakeLists.txt | 1 + aether/ae_actions/ping.h | 2 + aether/client_connectivity_policy.cpp | 99 +- aether/client_connectivity_policy.h | 29 +- .../local_presence_machine.cpp | 505 ++++++++ .../local_presence_machine.h | 221 ++++ .../local_presence_schedule.h | 123 +- .../cloud_connections/ping_cloud_servers.cpp | 464 ++++---- aether/cloud_connections/ping_cloud_servers.h | 66 +- tests/test-local-presence/CMakeLists.txt | 27 +- tests/test-local-presence/firewall_live.cpp | 158 ++- tests/test-local-presence/main.cpp | 1011 ++++++++--------- 13 files changed, 1735 insertions(+), 972 deletions(-) create mode 100644 aether/cloud_connections/local_presence_machine.cpp create mode 100644 aether/cloud_connections/local_presence_machine.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 691ec1a5..d2a08193 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,7 @@ option(AE_BUILD_TOOLS "Build tools" ${AE_ROOT_PORJECT}) option(AE_BUILD_EXAMPLES "Build examples" ${AE_ROOT_PORJECT}) option(AE_BUILD_TESTS "Build tests" ${AE_ROOT_PORJECT}) option(AE_BUILD_ANDROID_SMOKE "Build Android NDK smoke shared library and runner" Off) +option(AE_ENABLE_PRIVILEGED_NETWORK_TESTS "Enable Administrator/network live tests" Off) option(AE_ADDRESS_SANITIZE "Enable address sanitizer" Off) option(AE_NO_STRIP_ALL "Do not apply --strip_all, useful for bloaty and similar tools " Off) diff --git a/aether/CMakeLists.txt b/aether/CMakeLists.txt index d0fdc096..52c9885a 100644 --- a/aether/CMakeLists.txt +++ b/aether/CMakeLists.txt @@ -193,6 +193,7 @@ list(APPEND aether_srcs "cloud_connections/cloud_server_connection.cpp" "cloud_connections/cloud_server_connections.cpp" "cloud_connections/ping_cloud_servers.cpp" + "cloud_connections/local_presence_machine.cpp" "cloud_connections/cloud_subscription.cpp" "cloud_connections/cloud_request.cpp") diff --git a/aether/ae_actions/ping.h b/aether/ae_actions/ping.h index 411deec6..8599cda9 100644 --- a/aether/ae_actions/ping.h +++ b/aether/ae_actions/ping.h @@ -48,6 +48,8 @@ class Ping { Ping(AeContext const& ae_context, CloudServerConnection& cloud_server_connection, Duration next_ping_hint, Duration rx_window, Duration timeout); + // `timeout` is the hard wait for a Pong (cleanup). Local Presence retry + // deadlines (pXX) are owned by LocalPresenceMachine, not this timer. AE_CLASS_NO_COPY_MOVE(Ping); diff --git a/aether/client_connectivity_policy.cpp b/aether/client_connectivity_policy.cpp index 69e33c7d..1814a308 100644 --- a/aether/client_connectivity_policy.cpp +++ b/aether/client_connectivity_policy.cpp @@ -42,10 +42,7 @@ ClientConnectivityPolicy::RxTimingConfig::RxTimingConfig( ClientConnectivityPolicy::RxTimingConfig& ClientConnectivityPolicy::RxTimingConfig::ForAllPriorities(RxTimingConf conf) { - for (auto& item : policy_->rx_timings_) { - item.conf = conf; - } - policy_->ApplyDesiredToBoundServers(conf); + policy_->ApplyDesiredForAllPriorities(conf); return *this; } @@ -119,6 +116,24 @@ void ClientConnectivityPolicy::SetServerSelectedForAggregate(ServerId server_id, EnsureServerPresence(server_id).selected_for_aggregate = selected; } +void ClientConnectivityPolicy::BindServerPriority(ServerId server_id, + std::size_t priority) { + auto& state = EnsureServerPresence(server_id); + state.bound_priority = priority; + if (!state.has_user_rx_timing && (priority < rx_timings_.size())) { + ApplyDesiredIfNoOverride(server_id, state, rx_timings_[priority].conf); + } +} + +void ClientConnectivityPolicy::SetServerQuarantined(ServerId server_id, + bool quarantined) { + EnsureServerPresence(server_id).quarantined = quarantined; +} + +void ClientConnectivityPolicy::RemoveServerFromCloud(ServerId server_id) { + ClearServerPresence(server_id); +} + ClientConnectivityPolicy::SuspendBlocker ClientConnectivityPolicy::AcquireSuspendBlock() { return SuspendBlocker{*this}; @@ -189,10 +204,11 @@ void ClientConnectivityPolicy::ConfirmServerPong(ServerId server_id, TimePoint send_time, TimePoint pong_time, Duration interval, - Duration rx_window) { + Duration rx_window, + Duration selected_rtt) { auto& state = EnsureServerPresence(server_id); - auto const schedule = - MakeConfirmedSchedule(send_time, pong_time, interval, rx_window); + auto const schedule = MakeConfirmedSchedule(send_time, pong_time, interval, + rx_window, selected_rtt); state.has_confirmed_schedule = true; state.confirmed_interval = schedule.interval; state.confirmed_rx_window = schedule.rx_window; @@ -200,37 +216,14 @@ void ClientConnectivityPolicy::ConfirmServerPong(ServerId server_id, state.confirmed_pong_receive_time = schedule.pong_receive_time; state.confirmed_window_open_local = schedule.window_open_local; state.confirmed_window_close_local = schedule.window_close_local; - state.online = true; state.config_change_pending = (state.desired.interval != interval) || (state.desired.rx_window != rx_window); } -void ClientConnectivityPolicy::MarkServerOffline(ServerId server_id, - TimePoint now) { - auto* state = FindServerPresence(server_id); - if (state == nullptr) { - return; - } - if (state->has_confirmed_schedule && now > state->confirmed_window_close_local) { - state->online = false; - } -} - void ClientConnectivityPolicy::ClearServerPresence(ServerId server_id) { server_presence_.erase(server_id); } -void ClientConnectivityPolicy::InvalidateConfirmedSchedule(ServerId server_id) { - auto* state = FindServerPresence(server_id); - if (state == nullptr) { - return; - } - state->has_confirmed_schedule = false; - state->online = false; - state->confirmed_window_open_local = {}; - state->confirmed_window_close_local = {}; -} - bool ClientConnectivityPolicy::IsLocallyOnline() const noexcept { return IsLocallyOnline(Now()); } @@ -259,14 +252,6 @@ bool ClientConnectivityPolicy::IsServerLocallyOnline( state->confirmed_window_close_local); } -void ClientConnectivityPolicy::RefreshOnlineFlags(TimePoint now) { - for (auto& [id, state] : server_presence_) { - static_cast(id); - state.online = IsConfirmedWindowOnline(state.has_confirmed_schedule, now, - state.confirmed_window_close_local); - } -} - void ClientConnectivityPolicy::ResetRuntimeState() { auto current_time = Now(); for (auto& t : rx_timings_) { @@ -279,22 +264,44 @@ void ClientConnectivityPolicy::ResetRuntimeState() { static_cast(id); if (current_time < state.confirmed_pong_receive_time) { state.has_confirmed_schedule = false; - state.online = false; state.confirmed_window_open_local = {}; state.confirmed_window_close_local = {}; } } } -void ClientConnectivityPolicy::ApplyDesiredToBoundServers(RxTimingConf conf) { +void ClientConnectivityPolicy::ApplyDesiredIfNoOverride( + ServerId server_id, ServerPresenceState& state, RxTimingConf conf) { + if (state.has_user_rx_timing) { + return; + } + auto const timing_changed = (state.desired.interval != conf.interval) || + (state.desired.rx_window != conf.rx_window); + state.desired = conf; + if (timing_changed) { + state.config_change_pending = true; + server_rx_timing_changed_event_.Emit(server_id); + } +} + +void ClientConnectivityPolicy::ApplyDesiredForAllPriorities(RxTimingConf conf) { + for (auto& item : rx_timings_) { + item.conf = conf; + } for (auto& [id, state] : server_presence_) { - static_cast(id); - auto const timing_changed = (state.desired.interval != conf.interval) || - (state.desired.rx_window != conf.rx_window); - state.desired = conf; - if (timing_changed) { - state.config_change_pending = true; + ApplyDesiredIfNoOverride(id, state, conf); + } +} + +void ClientConnectivityPolicy::ApplyDesiredForPriority(std::size_t priority, + RxTimingConf conf) { + assert(priority < rx_timings_.size()); + rx_timings_[priority].conf = conf; + for (auto& [id, state] : server_presence_) { + if (state.bound_priority != priority) { + continue; } + ApplyDesiredIfNoOverride(id, state, conf); } } diff --git a/aether/client_connectivity_policy.h b/aether/client_connectivity_policy.h index 88413ca4..5c6f3522 100644 --- a/aether/client_connectivity_policy.h +++ b/aether/client_connectivity_policy.h @@ -81,12 +81,11 @@ struct ServerPresenceState { TimePoint confirmed_window_open_local{}; TimePoint confirmed_window_close_local{}; - bool online{false}; - std::uint64_t current_attempt_id{0}; - PingAttemptKind current_attempt_kind{PingAttemptKind::kInitial}; bool config_change_pending{false}; bool selected_for_aggregate{true}; bool has_user_rx_timing{false}; + bool quarantined{false}; + std::size_t bound_priority{static_cast(-1)}; }; class ClientConnectivityPolicy : public Obj { @@ -102,8 +101,7 @@ class ClientConnectivityPolicy : public Obj { template RxTimingConfig& ForPriority(RxTimingConf conf) { static_assert(Priority < kMaxRxServerPriorities); - policy_->rx_timings_[Priority].conf = conf; - policy_->ApplyDesiredToBoundServers(conf); + policy_->ApplyDesiredForPriority(Priority, conf); return *this; } @@ -151,6 +149,9 @@ class ClientConnectivityPolicy : public Obj { std::uint8_t rtt_reliability_percentile = kDefaultRttReliabilityPercentile); void SetServerSelectedForAggregate(ServerId server_id, bool selected); + void BindServerPriority(ServerId server_id, std::size_t priority); + void SetServerQuarantined(ServerId server_id, bool quarantined); + void RemoveServerFromCloud(ServerId server_id); RequestPolicy::Variant const& rx_targets() const noexcept { return rx_targets_; @@ -176,29 +177,27 @@ class ClientConnectivityPolicy : public Obj { ServerPresenceState const* FindServerPresence(ServerId server_id) const noexcept; ServerPresenceState* FindServerPresence(ServerId server_id) noexcept; - // Confirm schedule from a successful Pong (measured RTT). + // Confirm schedule from a successful Pong using selected_rtt projection. void ConfirmServerPong(ServerId server_id, TimePoint send_time, TimePoint pong_time, Duration interval, - Duration rx_window); + Duration rx_window, Duration selected_rtt); - void MarkServerOffline(ServerId server_id, TimePoint now); void ClearServerPresence(ServerId server_id); - // Drop confirmed schedule (quarantine / unusable) but keep desired timing. - void InvalidateConfirmedSchedule(ServerId server_id); - // Read-only. No side effects. Aggregate: ONLINE iff any selected usable - // server has a confirmed schedule that has not expired. + // Read-only. No side effects. Aggregate: ONLINE iff any selected server + // still in Personal Cloud has a confirmed schedule that has not expired. bool IsLocallyOnline() const noexcept; bool IsLocallyOnline(TimePoint now) const noexcept; bool IsServerLocallyOnline(ServerId server_id, TimePoint now) const noexcept; - void RefreshOnlineFlags(TimePoint now); - private: void ResetRuntimeState(); void IncrementSuspendBlock(); void DecrementSuspendBlock(); - void ApplyDesiredToBoundServers(RxTimingConf conf); + void ApplyDesiredForAllPriorities(RxTimingConf conf); + void ApplyDesiredForPriority(std::size_t priority, RxTimingConf conf); + void ApplyDesiredIfNoOverride(ServerId server_id, ServerPresenceState& state, + RxTimingConf conf); RequestPolicy::Variant rx_targets_; std::array rx_timings_; diff --git a/aether/cloud_connections/local_presence_machine.cpp b/aether/cloud_connections/local_presence_machine.cpp new file mode 100644 index 00000000..8ddee26f --- /dev/null +++ b/aether/cloud_connections/local_presence_machine.cpp @@ -0,0 +1,505 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "aether/cloud_connections/local_presence_machine.h" + +#include +#include + +namespace ae { + +namespace { + +void CountKind(LocalPresenceMachine::Counters& counters, + PingAttemptKind kind) noexcept { + switch (kind) { + case PingAttemptKind::kPrefix1: + ++counters.prefix1; + break; + case PingAttemptKind::kPrefix2: + ++counters.prefix2; + break; + case PingAttemptKind::kRetry: + ++counters.retry; + break; + case PingAttemptKind::kRecovery: + ++counters.recovery; + break; + case PingAttemptKind::kInitial: + ++counters.initial; + break; + } +} + +} // namespace + +LocalPresenceMachine::LocalPresenceMachine() = default; + +void LocalPresenceMachine::SetDesired(TimePoint now, RxTimingConf conf, + std::uint8_t percentile) { + if (percentile == 0) { + percentile = kDefaultRttReliabilityPercentile; + } + if (percentile > 100) { + percentile = 100; + } + auto const changed = (desired_.interval != conf.interval) || + (desired_.rx_window != conf.rx_window); + desired_ = conf; + percentile_ = percentile; + if (!changed) { + return; + } + config_pending_ = true; + if (!removed_ && !quarantined_ && !HasActiveSchedulerAttempt()) { + ArmSend(PingAttemptKind::kInitial, now); + } +} + +void LocalPresenceMachine::ArmInitial(TimePoint now) { + if (removed_ || quarantined_) { + return; + } + ArmSend(PingAttemptKind::kInitial, now); +} + +void LocalPresenceMachine::RestoreConfirmed(TimePoint open, TimePoint close, + Duration interval, Duration window, + TimePoint now, + Duration selected_rtt) { + if (removed_) { + return; + } + has_confirmed_ = true; + confirmed_open_ = open; + confirmed_close_ = close; + confirmed_interval_ = interval; + confirmed_window_ = window; + cycle_has_target_ = false; + cycle_confirmed_ = true; + PlanAfterConfirm(now, selected_rtt); +} + +LocalPresenceMachine::Tick LocalPresenceMachine::TickNow( + TimePoint now, Duration selected_rtt) { + Tick out{}; + if (removed_) { + request_blocker_held_ = false; + current_window_blocker_held_ = false; + return out; + } + + ReleaseCurrentWindowIfDue(now); + CleanupExpired(now); + MarkSchedulerTimeouts(now, selected_rtt); + RecalcRequestBlocker(); + + if (restream_pending_) { + out.restream = true; + out.restream_reason = restream_reason_; + restream_pending_ = false; + restream_reason_ = PresenceRestreamReason::kNone; + } + + if (!quarantined_ && send_armed_ && (now >= next_send_time_) && + !HasActiveSchedulerAttempt()) { + out.want_send = true; + out.send = BuildSendSpec(now, selected_rtt); + send_in_progress_ = true; + send_armed_ = false; + RecalcRequestBlocker(); + } + + out.next_wake = NextWake(now); + return out; +} + +void LocalPresenceMachine::OnSendStarting() { + send_in_progress_ = true; + RecalcRequestBlocker(); +} + +void LocalPresenceMachine::OnAttemptSent(SendSpec spec, TimePoint send_time) { + send_in_progress_ = false; + + Attempt attempt{}; + attempt.attempt_id = spec.attempt_id; + attempt.cycle_id = spec.cycle_id; + attempt.kind = spec.kind; + attempt.send_time = send_time; + attempt.selected_rtt = spec.selected_rtt; + attempt.sent_interval = spec.wire_interval; + attempt.desired_interval = spec.desired_interval; + attempt.sent_window = spec.rx_window; + attempt.following_open_target = spec.following_open_target; + attempt.retry_deadline = send_time + spec.selected_rtt; + attempt.cleanup_deadline = MakeCleanupDeadline(send_time, spec.selected_rtt); + attempt.scheduler_timed_out = false; + attempt.awaiting_response = true; + attempts_.push_back(attempt); + BoundAttempts(); + CountKind(counters_, spec.kind); + + if (spec.opens_current_window && has_confirmed_) { + if ((spec.kind == PingAttemptKind::kPrefix1) || + !current_window_blocker_held_) { + current_promised_close_ = confirmed_close_; + } + current_window_blocker_held_ = true; + } + RecalcRequestBlocker(); +} + +void LocalPresenceMachine::OnStartFailed(TimePoint now, Duration selected_rtt, + PresenceRestreamReason reason) { + send_in_progress_ = false; + RecalcRequestBlocker(); + if (reason != PresenceRestreamReason::kNone) { + restream_pending_ = true; + restream_reason_ = reason; + ++counters_.restreams; + } + ArmSend(PingAttemptKind::kRecovery, now + selected_rtt); +} + +LocalPresenceMachine::PongOutcome LocalPresenceMachine::OnPong( + std::uint64_t attempt_id, std::uint64_t cycle_id, TimePoint send_time, + TimePoint pong_time, Duration sent_interval, + Duration sent_desired_interval, Duration sent_window, + TimePoint following_open_target, Duration selected_rtt_after_sample) { + PongOutcome out{}; + auto* attempt = FindAttempt(attempt_id); + if (attempt != nullptr) { + cycle_id = attempt->cycle_id; + sent_interval = attempt->sent_interval; + sent_desired_interval = attempt->desired_interval; + sent_window = attempt->sent_window; + following_open_target = attempt->following_open_target; + attempt->awaiting_response = false; + EraseAttempt(attempt_id); + } + if (sent_desired_interval <= Duration{}) { + sent_desired_interval = sent_interval; + } + RecalcRequestBlocker(); + + out.schedule = MakeConfirmedSchedule(send_time, pong_time, sent_interval, + sent_window, selected_rtt_after_sample); + + auto const same_cycle = (cycle_id == active_cycle_id_); + if (!same_cycle || (cycle_confirmed_ && same_cycle)) { + out.disposition = PongDisposition::kStatsOnly; + ++counters_.late_pongs; + if (same_cycle) { + last_following_target_ = following_open_target; + } + return out; + } + + auto const was_online = IsOnline(pong_time); + has_confirmed_ = true; + confirmed_open_ = out.schedule.window_open_local; + confirmed_close_ = out.schedule.window_close_local; + confirmed_interval_ = sent_desired_interval; + confirmed_window_ = sent_window; + config_pending_ = (desired_.interval != sent_desired_interval) || + (desired_.rx_window != sent_window); + cycle_confirmed_ = true; + active_cycle_id_ = cycle_id; + last_following_target_ = following_open_target; + ++counters_.confirmed_pongs; + if (!was_online) { + ++counters_.recoveries_to_online; + } + out.disposition = PongDisposition::kConfirmedSchedule; + PlanAfterConfirm(pong_time, selected_rtt_after_sample); + return out; +} + +void LocalPresenceMachine::OnHardFailure(std::uint64_t attempt_id, + TimePoint now, Duration selected_rtt, + PresenceRestreamReason reason) { + auto* attempt = FindAttempt(attempt_id); + auto kind = PingAttemptKind::kRecovery; + if (attempt != nullptr) { + kind = attempt->kind; + attempt->awaiting_response = false; + EraseAttempt(attempt_id); + } + send_in_progress_ = false; + RecalcRequestBlocker(); + restream_pending_ = true; + restream_reason_ = reason; + ++counters_.restreams; + Attempt timed_out{}; + timed_out.kind = kind; + if (has_confirmed_ && (now <= confirmed_close_) && + AttemptOpensPromisedWindow(kind)) { + PlanAfterTimeout(timed_out, now, selected_rtt); + } else { + ArmSend(PingAttemptKind::kRecovery, now + selected_rtt); + } +} + +void LocalPresenceMachine::OnHardWaitExpired(std::uint64_t attempt_id, + TimePoint now) { + static_cast(now); + EraseAttempt(attempt_id); + RecalcRequestBlocker(); +} + +void LocalPresenceMachine::OnQuarantine(TimePoint now) { + quarantined_ = true; + send_in_progress_ = false; + send_armed_ = false; + attempts_.clear(); + RecalcRequestBlocker(); + ReleaseCurrentWindowIfDue(now); +} + +void LocalPresenceMachine::OnQuarantineReleased(TimePoint now, + Duration selected_rtt) { + quarantined_ = false; + ArmSend(PingAttemptKind::kRecovery, now + selected_rtt); +} + +void LocalPresenceMachine::OnRemoved() { + removed_ = true; + quarantined_ = false; + send_in_progress_ = false; + send_armed_ = false; + has_confirmed_ = false; + current_window_blocker_held_ = false; + request_blocker_held_ = false; + attempts_.clear(); +} + +bool LocalPresenceMachine::IsOnline(TimePoint now) const noexcept { + if (removed_) { + return false; + } + return IsConfirmedWindowOnline(has_confirmed_, now, confirmed_close_); +} + +LocalPresenceMachine::Attempt* LocalPresenceMachine::FindAttempt( + std::uint64_t attempt_id) noexcept { + for (auto& attempt : attempts_) { + if (attempt.attempt_id == attempt_id) { + return &attempt; + } + } + return nullptr; +} + +void LocalPresenceMachine::EraseAttempt(std::uint64_t attempt_id) { + attempts_.erase(std::remove_if(attempts_.begin(), attempts_.end(), + [attempt_id](Attempt const& attempt) { + return attempt.attempt_id == attempt_id; + }), + attempts_.end()); +} + +void LocalPresenceMachine::BoundAttempts() { + while (attempts_.size() > kMaxOutstandingPresenceAttempts) { + auto it = std::find_if(attempts_.begin(), attempts_.end(), + [](Attempt const& attempt) { + return attempt.scheduler_timed_out || + !attempt.awaiting_response; + }); + if (it == attempts_.end()) { + it = attempts_.begin(); + } + attempts_.erase(it); + } +} + +void LocalPresenceMachine::CleanupExpired(TimePoint now) { + attempts_.erase(std::remove_if(attempts_.begin(), attempts_.end(), + [now](Attempt const& attempt) { + return now >= attempt.cleanup_deadline; + }), + attempts_.end()); +} + +void LocalPresenceMachine::RecalcRequestBlocker() { + if (removed_) { + request_blocker_held_ = false; + return; + } + if (send_in_progress_ && !current_window_blocker_held_) { + request_blocker_held_ = true; + return; + } + for (auto const& attempt : attempts_) { + if (attempt.awaiting_response && + !AttemptOpensPromisedWindow(attempt.kind)) { + request_blocker_held_ = true; + return; + } + } + request_blocker_held_ = false; +} + +void LocalPresenceMachine::ReleaseCurrentWindowIfDue(TimePoint now) { + if (current_window_blocker_held_ && (now > current_promised_close_)) { + current_window_blocker_held_ = false; + } +} + +void LocalPresenceMachine::MarkSchedulerTimeouts(TimePoint now, + Duration selected_rtt) { + for (auto& attempt : attempts_) { + if (!attempt.awaiting_response || attempt.scheduler_timed_out) { + continue; + } + if (now < attempt.retry_deadline) { + continue; + } + attempt.scheduler_timed_out = true; + ++counters_.scheduler_timeouts; + if (!send_armed_ && !send_in_progress_ && !quarantined_) { + PlanAfterTimeout(attempt, now, selected_rtt); + } + } +} + +void LocalPresenceMachine::PlanAfterTimeout(Attempt const& timed_out, + TimePoint now, + Duration selected_rtt) { + if (has_confirmed_ && (now <= confirmed_close_)) { + if (timed_out.kind == PingAttemptKind::kPrefix1) { + auto prefix2 = + ComputePrefix2Time(confirmed_open_, selected_rtt); + if (prefix2 < now) { + prefix2 = now; + } + ArmSend(PingAttemptKind::kPrefix2, prefix2); + return; + } + ArmSend(PingAttemptKind::kRetry, now + selected_rtt); + return; + } + ArmSend(PingAttemptKind::kRecovery, now + selected_rtt); +} + +void LocalPresenceMachine::PlanAfterConfirm(TimePoint now, + Duration selected_rtt) { + cycle_has_target_ = false; + if (config_pending_) { + ArmSend(PingAttemptKind::kInitial, now); + return; + } + auto prefix1 = ComputePrefix1Time(confirmed_open_, selected_rtt); + if (prefix1 < now) { + prefix1 = now; + } + ArmSend(PingAttemptKind::kPrefix1, prefix1); +} + +void LocalPresenceMachine::ArmSend(PingAttemptKind kind, TimePoint when) { + next_kind_ = kind; + next_send_time_ = when; + send_armed_ = true; +} + +LocalPresenceMachine::SendSpec LocalPresenceMachine::BuildSendSpec( + TimePoint now, Duration selected_rtt) { + if (selected_rtt <= Duration{}) { + selected_rtt = kLocalPresenceGuard; + } + + SendSpec spec{}; + spec.attempt_id = ++next_attempt_id_; + spec.kind = next_kind_; + spec.selected_rtt = selected_rtt; + spec.rx_window = desired_.rx_window; + + auto const start_new_cycle = + (spec.kind == PingAttemptKind::kInitial) || + (spec.kind == PingAttemptKind::kRecovery) || + (spec.kind == PingAttemptKind::kPrefix1) || !cycle_has_target_; + + if (start_new_cycle) { + active_cycle_id_ = ++next_cycle_id_; + cycle_confirmed_ = false; + cycle_has_target_ = true; + if (has_confirmed_ && AttemptOpensPromisedWindow(spec.kind)) { + cycle_following_target_ = confirmed_open_ + desired_.interval; + } else { + auto const a_estimated = now + OneWayFromRtt(selected_rtt); + cycle_following_target_ = a_estimated + desired_.interval; + } + } + + spec.cycle_id = active_cycle_id_; + auto plan = PlanWireInterval(now, selected_rtt, cycle_following_target_, + desired_.interval); + cycle_following_target_ = plan.following_open_target; + spec.following_open_target = plan.following_open_target; + spec.wire_interval = plan.wire_interval; + spec.desired_interval = desired_.interval; + spec.hard_wait = + PresenceHardWait(selected_rtt, desired_.interval, desired_.rx_window); + spec.retry_deadline = now + selected_rtt; + spec.cleanup_deadline = MakeCleanupDeadline(now, selected_rtt); + spec.opens_current_window = + has_confirmed_ && AttemptOpensPromisedWindow(spec.kind); + return spec; +} + +bool LocalPresenceMachine::HasActiveSchedulerAttempt() const noexcept { + if (send_in_progress_) { + return true; + } + for (auto const& attempt : attempts_) { + if (attempt.awaiting_response && !attempt.scheduler_timed_out) { + return true; + } + } + return false; +} + +TimePoint LocalPresenceMachine::NextWake(TimePoint now) const noexcept { + static_cast(now); + auto next = TimePoint::max(); + if (send_armed_) { + next = std::min(next, next_send_time_); + } + if (current_window_blocker_held_) { + next = std::min(next, current_promised_close_); + } + for (auto const& attempt : attempts_) { + if (attempt.awaiting_response && !attempt.scheduler_timed_out) { + next = std::min(next, attempt.retry_deadline); + } + next = std::min(next, attempt.cleanup_deadline); + } + return next; +} + +TimePoint LocalPresenceMachine::MakeCleanupDeadline(TimePoint send_time, + Duration rtt) const { + auto deadline = send_time + (rtt * 8); + if (has_confirmed_) { + auto const until_close = confirmed_close_ + rtt; + if (until_close > deadline) { + deadline = until_close; + } + } + return deadline; +} + +} // namespace ae diff --git a/aether/cloud_connections/local_presence_machine.h b/aether/cloud_connections/local_presence_machine.h new file mode 100644 index 00000000..c48d5fd4 --- /dev/null +++ b/aether/cloud_connections/local_presence_machine.h @@ -0,0 +1,221 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_CLOUD_CONNECTIONS_LOCAL_PRESENCE_MACHINE_H_ +#define AETHER_CLOUD_CONNECTIONS_LOCAL_PRESENCE_MACHINE_H_ + +#include +#include +#include + +#include "aether/client_connectivity_policy.h" +#include "aether/cloud_connections/local_presence_schedule.h" +#include "aether/clock.h" + +namespace ae { + +// Production Local Presence orchestration used by PingCloudServers and tests. +// +// Client attempt_id / cycle_id are local only. The current cloud Ping API does +// not carry a schedule generation. If PREFIX1 is delayed on one adapter and +// PREFIX2 is applied first, a later PREFIX1 Pong is treated as same-cycle +// stats-only locally, but the server may still overwrite the listen schedule +// with PREFIX1's wire interval. Correct cross-adapter ordering would need a +// wire/server generation; that protocol extension is intentionally not made +// here. +class LocalPresenceMachine { + public: + struct Attempt { + std::uint64_t attempt_id{}; + std::uint64_t cycle_id{}; + PingAttemptKind kind{PingAttemptKind::kInitial}; + TimePoint send_time{}; + Duration selected_rtt{}; + Duration sent_interval{}; + Duration desired_interval{}; + Duration sent_window{}; + TimePoint following_open_target{}; + TimePoint retry_deadline{}; + TimePoint cleanup_deadline{}; + bool scheduler_timed_out{false}; + bool awaiting_response{true}; + }; + + struct SendSpec { + std::uint64_t attempt_id{}; + std::uint64_t cycle_id{}; + PingAttemptKind kind{PingAttemptKind::kInitial}; + Duration wire_interval{}; + Duration desired_interval{}; + Duration rx_window{}; + Duration selected_rtt{}; + Duration hard_wait{}; + TimePoint following_open_target{}; + TimePoint retry_deadline{}; + TimePoint cleanup_deadline{}; + bool opens_current_window{false}; + }; + + struct Tick { + TimePoint next_wake{TimePoint::max()}; + bool want_send{false}; + SendSpec send{}; + bool restream{false}; + PresenceRestreamReason restream_reason{PresenceRestreamReason::kNone}; + }; + + enum class PongDisposition : std::uint8_t { + kUnknownAttempt = 0, + kStatsOnly, + kConfirmedSchedule, + }; + + struct PongOutcome { + PongDisposition disposition{PongDisposition::kUnknownAttempt}; + ConfirmedReceiveSchedule schedule{}; + }; + + struct Counters { + int initial{}; + int prefix1{}; + int prefix2{}; + int retry{}; + int recovery{}; + int scheduler_timeouts{}; + int confirmed_pongs{}; + int late_pongs{}; + int restreams{}; + int recoveries_to_online{}; + }; + + LocalPresenceMachine(); + + void SetDesired(TimePoint now, RxTimingConf conf, std::uint8_t percentile); + RxTimingConf const& desired() const noexcept { return desired_; } + std::uint8_t percentile() const noexcept { return percentile_; } + + void RestoreConfirmed(TimePoint open, TimePoint close, Duration interval, + Duration window, TimePoint now, + Duration selected_rtt); + void ArmInitial(TimePoint now); + + Tick TickNow(TimePoint now, Duration selected_rtt); + + void OnSendStarting(); + void OnAttemptSent(SendSpec spec, TimePoint send_time); + void OnStartFailed(TimePoint now, Duration selected_rtt, + PresenceRestreamReason reason); + + PongOutcome OnPong(std::uint64_t attempt_id, std::uint64_t cycle_id, + TimePoint send_time, TimePoint pong_time, + Duration sent_interval, Duration sent_desired_interval, + Duration sent_window, TimePoint following_open_target, + Duration selected_rtt_after_sample); + + void OnHardFailure(std::uint64_t attempt_id, TimePoint now, + Duration selected_rtt, PresenceRestreamReason reason); + void OnHardWaitExpired(std::uint64_t attempt_id, TimePoint now); + + void OnQuarantine(TimePoint now); + void OnQuarantineReleased(TimePoint now, Duration selected_rtt); + void OnRemoved(); + + bool IsOnline(TimePoint now) const noexcept; + bool has_confirmed_schedule() const noexcept { return has_confirmed_; } + TimePoint confirmed_window_open() const noexcept { return confirmed_open_; } + TimePoint confirmed_window_close() const noexcept { return confirmed_close_; } + Duration confirmed_interval() const noexcept { return confirmed_interval_; } + Duration confirmed_rx_window() const noexcept { return confirmed_window_; } + TimePoint current_promised_close() const noexcept { + return current_promised_close_; + } + bool current_window_blocker_held() const noexcept { + return current_window_blocker_held_; + } + bool request_blocker_held() const noexcept { return request_blocker_held_; } + bool CanSuspend() const noexcept { + return !current_window_blocker_held_ && !request_blocker_held_; + } + std::size_t outstanding_attempt_count() const noexcept { + return attempts_.size(); + } + std::vector const& attempts() const noexcept { return attempts_; } + Counters const& counters() const noexcept { return counters_; } + bool quarantined() const noexcept { return quarantined_; } + bool removed() const noexcept { return removed_; } + bool config_change_pending() const noexcept { return config_pending_; } + TimePoint last_following_target() const noexcept { + return last_following_target_; + } + TimePoint PeekNextWake() const noexcept { return NextWake(TimePoint{}); } + + private: + Attempt* FindAttempt(std::uint64_t attempt_id) noexcept; + void EraseAttempt(std::uint64_t attempt_id); + void BoundAttempts(); + void CleanupExpired(TimePoint now); + void RecalcRequestBlocker(); + void ReleaseCurrentWindowIfDue(TimePoint now); + void MarkSchedulerTimeouts(TimePoint now, Duration selected_rtt); + void PlanAfterTimeout(Attempt const& timed_out, TimePoint now, + Duration selected_rtt); + void PlanAfterConfirm(TimePoint now, Duration selected_rtt); + void ArmSend(PingAttemptKind kind, TimePoint when); + SendSpec BuildSendSpec(TimePoint now, Duration selected_rtt); + bool HasActiveSchedulerAttempt() const noexcept; + TimePoint NextWake(TimePoint now) const noexcept; + TimePoint MakeCleanupDeadline(TimePoint send_time, Duration rtt) const; + + RxTimingConf desired_{RxTimingConf::Every( + std::chrono::milliseconds{AE_PING_INTERVAL_MS})}; + std::uint8_t percentile_{kDefaultRttReliabilityPercentile}; + + bool has_confirmed_{false}; + TimePoint confirmed_open_{}; + TimePoint confirmed_close_{}; + Duration confirmed_interval_{}; + Duration confirmed_window_{}; + bool config_pending_{false}; + + bool current_window_blocker_held_{false}; + TimePoint current_promised_close_{}; + + bool quarantined_{false}; + bool removed_{false}; + bool send_in_progress_{false}; + bool send_armed_{false}; + bool request_blocker_held_{false}; + bool restream_pending_{false}; + PresenceRestreamReason restream_reason_{PresenceRestreamReason::kNone}; + + PingAttemptKind next_kind_{PingAttemptKind::kInitial}; + TimePoint next_send_time_{}; + + std::uint64_t next_attempt_id_{0}; + std::uint64_t next_cycle_id_{0}; + std::uint64_t active_cycle_id_{0}; + TimePoint cycle_following_target_{}; + bool cycle_has_target_{false}; + bool cycle_confirmed_{false}; + TimePoint last_following_target_{}; + + std::vector attempts_{}; + Counters counters_{}; +}; + +} // namespace ae + +#endif // AETHER_CLOUD_CONNECTIONS_LOCAL_PRESENCE_MACHINE_H_ diff --git a/aether/cloud_connections/local_presence_schedule.h b/aether/cloud_connections/local_presence_schedule.h index 7a294a1e..cb58d9f7 100644 --- a/aether/cloud_connections/local_presence_schedule.h +++ b/aether/cloud_connections/local_presence_schedule.h @@ -30,6 +30,8 @@ inline constexpr Duration kLocalPresenceGuard = inline constexpr std::uint8_t kDefaultRttReliabilityPercentile{99}; +inline constexpr std::size_t kMaxOutstandingPresenceAttempts{8}; + enum class PingAttemptKind : std::uint8_t { kInitial = 0, kPrefix1, @@ -38,10 +40,25 @@ enum class PingAttemptKind : std::uint8_t { kRecovery, }; +enum class PresenceRestreamReason : std::uint8_t { + kNone = 0, + kHardWriteFailure, + kHardLinkFailure, + kPingApiError, + kConnectionUnavailable, +}; + // One-way RTT projection used consistently for schedule placement. -// Documented model: one_way = rtt / 2 (local monotonic timeline only). -inline Duration OneWayFromRtt(Duration rtt) noexcept { - return rtt / 2; +// Documented model: one_way = selected_rtt / 2 (local monotonic timeline only). +inline Duration OneWayFromRtt(Duration rtt) noexcept { return rtt / 2; } + +inline Duration MaxDuration(Duration a, Duration b) noexcept { + return a > b ? a : b; +} + +inline Duration PresenceHardWait(Duration selected_rtt, Duration interval, + Duration window) noexcept { + return MaxDuration(selected_rtt * 8, interval + window); } struct ConfirmedReceiveSchedule { @@ -50,41 +67,66 @@ struct ConfirmedReceiveSchedule { TimePoint ping_send_time{}; TimePoint pong_receive_time{}; Duration measured_rtt{}; + Duration selected_rtt{}; TimePoint window_open_local{}; TimePoint window_close_local{}; }; // After successful Pong for Ping sent at send_time: -// R_server ≈ send_time + one_way -// window_open = R_server + interval -// window_close = window_open + rx_window +// estimated_server_receive = send_time + selected_rtt / 2 +// window_open = estimated_server_receive + sent_interval +// window_close = window_open + sent_window +// measured RTT is diagnostics/statistics only and must not move the projection. inline ConfirmedReceiveSchedule MakeConfirmedSchedule( TimePoint send_time, TimePoint pong_time, Duration interval, - Duration rx_window) noexcept { + Duration rx_window, Duration selected_rtt) noexcept { ConfirmedReceiveSchedule out{}; out.interval = interval; out.rx_window = rx_window; out.ping_send_time = send_time; out.pong_receive_time = pong_time; + out.selected_rtt = selected_rtt; if (pong_time > send_time) { out.measured_rtt = std::chrono::duration_cast(pong_time - send_time); } - auto const one_way = OneWayFromRtt(out.measured_rtt); + auto const one_way = OneWayFromRtt(selected_rtt); out.window_open_local = send_time + one_way + interval; out.window_close_local = out.window_open_local + rx_window; return out; } +struct CadencePlan { + TimePoint following_open_target{}; + Duration wire_interval{}; +}; + +// configured interval is the distance between planned opening targets, not +// between early prefix sends. +inline CadencePlan PlanWireInterval(TimePoint send_time, Duration selected_rtt, + TimePoint following_open_target, + Duration desired_interval) noexcept { + auto const a_estimated = send_time + OneWayFromRtt(selected_rtt); + if (following_open_target <= a_estimated) { + return CadencePlan{a_estimated + desired_interval, desired_interval}; + } + return CadencePlan{ + following_open_target, + std::chrono::duration_cast(following_open_target - + a_estimated)}; +} + // prefix1 = O - 1.5*R - G // prefix2 = O - 0.5*R - G -inline TimePoint ComputePrefix1Time(TimePoint window_open, Duration rtt, - Duration guard = kLocalPresenceGuard) noexcept { +inline TimePoint ComputePrefix1Time( + TimePoint window_open, Duration rtt, + Duration guard = kLocalPresenceGuard) noexcept { return window_open - (rtt * 3) / 2 - guard; } -inline TimePoint ComputePrefix2Time(TimePoint window_open, Duration rtt, - Duration guard = kLocalPresenceGuard) noexcept { +inline TimePoint ComputePrefix2Time( + TimePoint window_open, Duration rtt, + Duration guard = kLocalPresenceGuard) noexcept { return window_open - rtt / 2 - guard; } @@ -96,60 +138,9 @@ inline bool IsConfirmedWindowOnline(bool has_confirmed, TimePoint now, return now <= window_close; } -inline bool IsCurrentPingAttempt(std::uint64_t active_attempt_id, - std::uint64_t result_attempt_id) noexcept { - return active_attempt_id == result_attempt_id; -} - -// Next Ping after a failed attempt. p99/selected RTT timeout is NOT OFFLINE. -struct PresenceAttemptPlan { - TimePoint when{}; - PingAttemptKind kind{PingAttemptKind::kRecovery}; - bool mark_offline{false}; -}; - -inline PresenceAttemptPlan PlanAfterFailedAttempt( - bool has_confirmed_schedule, TimePoint confirmed_window_open, - TimePoint confirmed_window_close, PingAttemptKind failed_kind, - TimePoint now, Duration rtt, - Duration guard = kLocalPresenceGuard) noexcept { - PresenceAttemptPlan plan{}; - if (has_confirmed_schedule && now <= confirmed_window_close) { - plan.mark_offline = false; - if (failed_kind == PingAttemptKind::kPrefix1) { - plan.kind = PingAttemptKind::kPrefix2; - auto const prefix2 = - ComputePrefix2Time(confirmed_window_open, rtt, guard); - plan.when = prefix2 > now ? prefix2 : now; - return plan; - } - plan.kind = PingAttemptKind::kRetry; - plan.when = now + rtt; - return plan; - } - plan.mark_offline = has_confirmed_schedule; - plan.kind = PingAttemptKind::kRecovery; - plan.when = now + rtt; - return plan; -} - -// After a confirming Pong: prefix1 of the new window, unless a newer desired -// interval/window still needs a Ping. -inline PresenceAttemptPlan PlanAfterSuccessfulPong( - TimePoint confirmed_window_open, TimePoint now, Duration rtt, - bool send_new_config_immediately, - Duration guard = kLocalPresenceGuard) noexcept { - PresenceAttemptPlan plan{}; - plan.mark_offline = false; - if (send_new_config_immediately) { - plan.kind = PingAttemptKind::kInitial; - plan.when = now; - return plan; - } - plan.kind = PingAttemptKind::kPrefix1; - auto const prefix1 = ComputePrefix1Time(confirmed_window_open, rtt, guard); - plan.when = prefix1 > now ? prefix1 : now; - return plan; +inline bool AttemptOpensPromisedWindow(PingAttemptKind kind) noexcept { + return kind == PingAttemptKind::kPrefix1 || + kind == PingAttemptKind::kPrefix2 || kind == PingAttemptKind::kRetry; } } // namespace ae diff --git a/aether/cloud_connections/ping_cloud_servers.cpp b/aether/cloud_connections/ping_cloud_servers.cpp index d990fc65..3bb37354 100644 --- a/aether/cloud_connections/ping_cloud_servers.cpp +++ b/aether/cloud_connections/ping_cloud_servers.cpp @@ -18,7 +18,9 @@ #include #include +#include #include +#include #include #if AE_ENABLE_PING @@ -26,29 +28,10 @@ # include "aether/channels/channel.h" # include "aether/cloud_connections/cloud_connections_tele.h" # include "aether/executors/executors.h" +# include namespace ae { -namespace { - -char const* AttemptKindName(PingAttemptKind kind) { - switch (kind) { - case PingAttemptKind::kInitial: - return "INITIAL"; - case PingAttemptKind::kPrefix1: - return "PREFIX1"; - case PingAttemptKind::kPrefix2: - return "PREFIX2"; - case PingAttemptKind::kRetry: - return "RETRY"; - case PingAttemptKind::kRecovery: - return "RECOVERY"; - } - return "UNKNOWN"; -} - -} // namespace - PingCloudServers::ServerPing::ServerPing(AeContext const& ae_context, ClientConnectivityPolicy& policy, CloudServerConnection& cloud_sc, @@ -61,35 +44,48 @@ PingCloudServers::ServerPing::ServerPing(AeContext const& ae_context, assert(priority < policy_->rx_timings().size() && "Server ping priority should be in timings range"); + policy_->BindServerPriority(server_id_, priority_); auto& presence = policy_->EnsureServerPresence(server_id_); - if (!presence.has_user_rx_timing) { - presence.desired = policy_->rx_timings()[priority_].conf; - } - active_conf_ = presence.desired; policy_->SetServerSelectedForAggregate(server_id_, true); - - ping_blocker_ = policy_->AcquireSuspendBlock(); - start_sub_ = ae_context_.scheduler().Task( - [this]() { StartAttempt(PingAttemptKind::kInitial); }); + machine_.SetDesired(Now(), presence.desired, + presence.rtt_reliability_percentile); + if (presence.has_confirmed_schedule) { + machine_.RestoreConfirmed( + presence.confirmed_window_open_local, + presence.confirmed_window_close_local, presence.confirmed_interval, + presence.confirmed_rx_window, Now(), SelectedRtt()); + } else { + machine_.ArmInitial(Now()); + } + Pump(); } -PingCloudServers::ServerPing::~ServerPing() = default; - -void PingCloudServers::ServerPing::Stop() { +PingCloudServers::ServerPing::~ServerPing() { stop_ = true; - AbandonInFlight(); waiter_.reset(); - start_sub_.Reset(); - attempt_timeout_sub_.Reset(); - rx_window_sub_.Reset(); + live_.clear(); + wake_sub_.Reset(); + current_window_sub_.Reset(); restream_sub_.Reset(); link_state_sub_.Reset(); - ping_blocker_.Reset(); - rx_window_blocker_.Reset(); + request_blocker_.Reset(); + current_window_blocker_.Reset(); restream_blocker_.Reset(); - policy_->SetServerSelectedForAggregate(server_id_, false); - policy_->InvalidateConfirmedSchedule(server_id_); - policy_->RefreshOnlineFlags(Now()); +} + +void PingCloudServers::ServerPing::PauseForQuarantine() { + machine_.OnQuarantine(Now()); + waiter_.reset(); + live_.clear(); + policy_->SetServerQuarantined(server_id_, true); + SyncBlockers(); + Pump(); +} + +void PingCloudServers::ServerPing::ResumeFromQuarantine() { + policy_->SetServerQuarantined(server_id_, false); + machine_.OnQuarantineReleased(Now(), SelectedRtt()); + Pump(); } void PingCloudServers::ServerPing::NotifyConfigChanged() { @@ -100,46 +96,92 @@ void PingCloudServers::ServerPing::NotifyConfigChanged() { if (presence == nullptr) { return; } - active_conf_ = presence->desired; - if (attempt_in_flight_) { + machine_.SetDesired(Now(), presence->desired, + presence->rtt_reliability_percentile); + Pump(); +} + +void PingCloudServers::ServerPing::Pump() { + if (stop_) { return; } - if (presence->config_change_pending || !presence->has_confirmed_schedule) { - ScheduleNext(Now(), PingAttemptKind::kInitial); + auto const now = Now(); + auto const rtt = SelectedRtt(); + auto tick = machine_.TickNow(now, rtt); + SyncBlockers(); + DropFinishedPings(); + if (tick.restream) { + ScheduleRestream(); + } + if (tick.want_send) { + StartSend(tick.send); return; } - auto const rtt = SelectedRtt(); - auto const prefix1 = - ComputePrefix1Time(presence->confirmed_window_open_local, rtt); - ScheduleNext(prefix1 > Now() ? prefix1 : Now(), PingAttemptKind::kPrefix1); + ScheduleWake(tick.next_wake); } -void PingCloudServers::ServerPing::ScheduleNext(TimePoint when, - PingAttemptKind kind) { - if (stop_) { +void PingCloudServers::ServerPing::ScheduleWake(TimePoint when) { + next_wake_ = when; + policy_->ReportNextServiceTime(priority_, next_wake_); + if (when == TimePoint::max()) { + wake_sub_.Reset(); return; } - next_ping_time_ = when; - policy_->ReportNextServiceTime(priority_, next_ping_time_); - auto* presence = policy_->FindServerPresence(server_id_); - if (presence != nullptr) { - presence->current_attempt_kind = kind; + wake_sub_ = ae_context_.scheduler().DelayedTask([this]() noexcept { Pump(); }, + when); +} + +void PingCloudServers::ServerPing::SyncBlockers() { + if (machine_.current_window_blocker_held()) { + if (!holding_current_) { + current_window_blocker_ = policy_->AcquireSuspendBlock(); + holding_current_ = true; + } + auto const close = machine_.current_promised_close(); + if (scheduled_current_close_ != close) { + scheduled_current_close_ = close; + current_window_sub_ = ae_context_.scheduler().DelayedTask( + [this]() noexcept { + holding_current_ = false; + scheduled_current_close_ = {}; + current_window_blocker_.Reset(); + Pump(); + }, + close); + } + } else { + current_window_sub_.Reset(); + current_window_blocker_.Reset(); + holding_current_ = false; + scheduled_current_close_ = {}; + } + + if (machine_.request_blocker_held()) { + if (!holding_request_) { + request_blocker_ = policy_->AcquireSuspendBlock(); + holding_request_ = true; + } + } else { + request_blocker_.Reset(); + holding_request_ = false; } - start_sub_ = ae_context_.scheduler().DelayedTask( - [this, kind]() noexcept { StartAttempt(kind); }, when); } -void PingCloudServers::ServerPing::StartAttempt(PingAttemptKind kind) { - if (stop_ || attempt_in_flight_) { - return; +void PingCloudServers::ServerPing::DropFinishedPings() { + for (auto it = live_.begin(); it != live_.end();) { + bool found = false; + for (auto const& attempt : machine_.attempts()) { + if (attempt.attempt_id == it->first) { + found = true; + break; + } + } + if (!found) { + it = live_.erase(it); + } else { + ++it; + } } - auto& presence = policy_->EnsureServerPresence(server_id_); - active_conf_ = presence.desired; - presence.current_attempt_kind = kind; - ++presence.current_attempt_id; - active_attempt_id_ = presence.current_attempt_id; - active_attempt_kind_ = kind; - Start(); } template @@ -190,32 +232,13 @@ Duration PingCloudServers::ServerPing::SelectedRtt() const { return stats.PercentileValue(pct); } -Duration PingCloudServers::ServerPing::AttemptTimeout(Duration rtt) const { - // Scheduler treats selected RTT as the attempt window; floor with guard. - if (rtt <= Duration{}) { - return kLocalPresenceGuard; - } - return rtt; -} - -void PingCloudServers::ServerPing::AbandonInFlight() { - attempt_timeout_sub_.Reset(); - if (ping_) { - ping_.reset(); - } - attempt_in_flight_ = false; - ping_blocker_.Reset(); - ++active_attempt_id_; - auto* presence = policy_->FindServerPresence(server_id_); - if (presence != nullptr) { - presence->current_attempt_id = active_attempt_id_; - } -} - -void PingCloudServers::ServerPing::Start() { +void PingCloudServers::ServerPing::StartSend( + LocalPresenceMachine::SendSpec spec) { if (stop_) { return; } + machine_.OnSendStarting(); + SyncBlockers(); waiter_.emplace( ae_context_, EnsureLinked() | @@ -232,57 +255,35 @@ void PingCloudServers::ServerPing::Start() { return ex::create( [&](auto& ctx) noexcept { auto* cc = cloud_sc_->client_connection(); - assert(cc != nullptr && "Client connection should exists"); + if (cc == nullptr) { + return ex::set_error(std::move(ctx.receiver), 1); + } auto c = cc->server_connection().current_channel(); if (c == nullptr) { AE_TELED_ERROR("Current channel value invalid"); return ex::set_error(std::move(ctx.receiver), 2); } - auto const rtt = SelectedRtt(); - auto const timeout = AttemptTimeout(rtt); - auto& presence = policy_->EnsureServerPresence(server_id_); - active_conf_ = presence.desired; - auto const percentile = presence.rtt_reliability_percentile; - - active_sent_interval_ = active_conf_.interval; - active_sent_window_ = active_conf_.rx_window; - active_send_time_ = Now(); - attempt_in_flight_ = true; - - ping_.emplace(ae_context_, *cloud_sc_, active_sent_interval_, - active_sent_window_, timeout); - - ping_blocker_ = policy_->AcquireSuspendBlock(); - auto const attempt_id = active_attempt_id_; - auto const send_time = active_send_time_; - auto const sent_interval = active_sent_interval_; - auto const sent_window = active_sent_window_; - ping_->result_event().Subscribe( - [this, attempt_id, send_time, sent_interval, - sent_window](Ping::PingResult const& res) noexcept { - OnPingResult(attempt_id, send_time, sent_interval, - sent_window, res); + auto const send_time = Now(); + auto ping = std::make_unique( + ae_context_, *cloud_sc_, spec.wire_interval, + spec.rx_window, spec.hard_wait); + auto const attempt_id = spec.attempt_id; + ping->result_event().Subscribe( + [this, attempt_id](Ping::PingResult const& res) noexcept { + OnPingResult(attempt_id, res); }); - + ping->Start(send_time); + machine_.OnAttemptSent(spec, send_time); + LiveAttempt live{}; + live.spec = spec; + live.send_time = send_time; + live.ping = std::move(ping); + live_[attempt_id] = std::move(live); AE_TELED_DEBUG( - "PING_ATTEMPT server {} id {} kind {} send {}", - server_id_, attempt_id, - AttemptKindName(active_attempt_kind_), send_time); - AE_TELED_DEBUG( - "RTT server {} percentile {} selected_rtt {}", - server_id_, static_cast(percentile), rtt); - - ping_->Start(send_time); - - // Scheduler failure of this attempt at selected RTT — not - // OFFLINE. - attempt_timeout_sub_ = ae_context_.scheduler().DelayedTask( - [this, attempt_id]() noexcept { - OnAttemptTimeout(attempt_id); - }, - send_time + timeout); - + "PING_ATTEMPT server {} id {} kind {} send {} wire {}", + server_id_, attempt_id, static_cast(spec.kind), + send_time, spec.wire_interval); return ex::set_value(std::move(ctx.receiver)); }); }) | @@ -298,134 +299,104 @@ void PingCloudServers::ServerPing::Start() { [this](std::optional&& res) noexcept { if (res && res->IsErr()) { AE_TELED_ERROR("Ping start error {}", std::move(res)->error()); - attempt_in_flight_ = false; - ScheduleRestream(); - ScheduleNext(Now() + SelectedRtt(), PingAttemptKind::kRecovery); + machine_.OnStartFailed(Now(), SelectedRtt(), + PresenceRestreamReason::kConnectionUnavailable); + SyncBlockers(); + Pump(); } else if (!(res && res->IsOk())) { AE_TELED_DEBUG("Server ping stopped"); + machine_.OnStartFailed(Now(), SelectedRtt(), + PresenceRestreamReason::kNone); + SyncBlockers(); + } else { + Pump(); } }); } void PingCloudServers::ServerPing::OnPingResult(std::uint64_t attempt_id, - TimePoint send_time, - Duration sent_interval, - Duration sent_window, Ping::PingResult const& res) { if (stop_) { return; } - // Late / abandoned attempts must not roll confirmed schedule backwards. - if (!IsCurrentPingAttempt(active_attempt_id_, attempt_id)) { - AE_TELED_DEBUG("Ignoring stale ping result attempt {}", attempt_id); - return; - } - - auto* cc = cloud_sc_->client_connection(); - if (cc == nullptr) { - AE_TELED_ERROR("Client connection is null"); - return; - } - auto c = cc->server_connection().current_channel(); - if (!c) { - AE_TELED_ERROR("Connection channel is null"); + auto it = live_.find(attempt_id); + if (it == live_.end()) { return; } + auto spec = it->second.spec; + auto send_time = it->second.send_time; std::visit( - [this, c, send_time, sent_interval, sent_window](auto const& value) { + [this, attempt_id, spec, send_time](auto const& value) { using T = std::decay_t; - if constexpr (std::is_same_v>) { - c->channel_statistics().AddResponseTime(value.value); - ApplyConfirmedPong(send_time, Now(), sent_interval, sent_window); - } else if constexpr (std::is_same_v) { - AE_TELED_DEBUG("Got late ping duration for active attempt"); - c->channel_statistics().AddResponseTime(value.duration); - ApplyConfirmedPong(send_time, Now(), sent_interval, sent_window); + if constexpr (std::is_same_v> || + std::is_same_v) { + Duration measured{}; + if constexpr (std::is_same_v>) { + measured = value.value; + } else { + measured = value.duration; + } + AddRttSample(measured); + auto const selected = SelectedRtt(); + auto outcome = machine_.OnPong( + attempt_id, spec.cycle_id, send_time, Now(), spec.wire_interval, + spec.desired_interval, spec.rx_window, + spec.following_open_target, selected); + ApplyConfirmed(outcome); + live_.erase(attempt_id); + SyncBlockers(); + Pump(); } else { - AE_TELED_ERROR("Ping error!"); - AbandonInFlight(); - ScheduleRestream(); - AfterFailedAttempt(); + auto const code = value.error; + if (code == 2) { + machine_.OnHardWaitExpired(attempt_id, Now()); + live_.erase(attempt_id); + SyncBlockers(); + Pump(); + return; + } + auto reason = PresenceRestreamReason::kPingApiError; + if (code == 1) { + reason = PresenceRestreamReason::kHardWriteFailure; + } + machine_.OnHardFailure(attempt_id, Now(), SelectedRtt(), reason); + live_.erase(attempt_id); + SyncBlockers(); + Pump(); } }, res); } -void PingCloudServers::ServerPing::ApplyConfirmedPong(TimePoint send_time, - TimePoint pong_time, - Duration sent_interval, - Duration sent_window) { - attempt_timeout_sub_.Reset(); - attempt_in_flight_ = false; - ping_blocker_.Reset(); - - policy_->ConfirmServerPong(server_id_, send_time, pong_time, sent_interval, - sent_window); - auto* presence = policy_->FindServerPresence(server_id_); - assert(presence != nullptr); - AE_TELED_DEBUG( - "SCHEDULE confirmed server {} open {} close {} interval {} window {}", - server_id_, presence->confirmed_window_open_local, - presence->confirmed_window_close_local, sent_interval, sent_window); - AE_TELED_DEBUG("STATUS server {} ONLINE", server_id_); - - // Contractual window is a minimum listen guarantee, not a transport close. - HoldRxUntil(presence->confirmed_window_close_local); - - auto const now = Now(); - auto const rtt = SelectedRtt(); - auto const plan = PlanAfterSuccessfulPong( - presence->confirmed_window_open_local, now, rtt, - presence->config_change_pending); - ScheduleNext(plan.when, plan.kind); -} - -void PingCloudServers::ServerPing::OnAttemptTimeout(std::uint64_t attempt_id) { - if (stop_ || !IsCurrentPingAttempt(active_attempt_id_, attempt_id) || - !attempt_in_flight_) { - return; - } - AE_TELED_DEBUG("Ping attempt {} kind {} timed out for scheduler", attempt_id, - AttemptKindName(active_attempt_kind_)); - AbandonInFlight(); - AfterFailedAttempt(); -} - -void PingCloudServers::ServerPing::AfterFailedAttempt() { - if (stop_) { +void PingCloudServers::ServerPing::AddRttSample(Duration measured) { + auto* cc = cloud_sc_->client_connection(); + if (cc == nullptr) { return; } - auto now = Now(); - policy_->RefreshOnlineFlags(now); - auto* presence = policy_->FindServerPresence(server_id_); - if (presence == nullptr) { - ScheduleNext(now + SelectedRtt(), PingAttemptKind::kRecovery); + auto c = cc->server_connection().current_channel(); + if (!c) { return; } + c->channel_statistics().AddResponseTime(measured); +} - auto const rtt = SelectedRtt(); - if (presence->config_change_pending && presence->has_confirmed_schedule) { - ScheduleNext(now, PingAttemptKind::kInitial); +void PingCloudServers::ServerPing::ApplyConfirmed( + LocalPresenceMachine::PongOutcome const& outcome) { + if (outcome.disposition != + LocalPresenceMachine::PongDisposition::kConfirmedSchedule) { return; } - - auto const plan = PlanAfterFailedAttempt( - presence->has_confirmed_schedule, presence->confirmed_window_open_local, - presence->confirmed_window_close_local, active_attempt_kind_, now, rtt); - if (plan.mark_offline) { - policy_->MarkServerOffline(server_id_, now); - AE_TELED_DEBUG("STATUS server {} OFFLINE after window close {}", server_id_, - presence->confirmed_window_close_local); - } - ScheduleNext(plan.when, plan.kind); -} - -void PingCloudServers::ServerPing::HoldRxUntil(TimePoint until) { - // rx_window is a minimum contractual guarantee. Do not close transport. - rx_window_blocker_ = policy_->AcquireSuspendBlock(); - rx_window_sub_ = ae_context_.scheduler().DelayedTask( - [this]() { rx_window_blocker_.Reset(); }, until); + policy_->ConfirmServerPong( + server_id_, outcome.schedule.ping_send_time, + outcome.schedule.pong_receive_time, outcome.schedule.interval, + outcome.schedule.rx_window, outcome.schedule.selected_rtt); + auto& state = policy_->EnsureServerPresence(server_id_); + state.confirmed_interval = machine_.confirmed_interval(); + state.config_change_pending = machine_.config_change_pending(); + AE_TELED_DEBUG("SCHEDULE confirmed server {} open {} close {}", server_id_, + outcome.schedule.window_open_local, + outcome.schedule.window_close_local); } void PingCloudServers::ServerPing::ScheduleRestream() { @@ -492,6 +463,7 @@ void PingCloudServers::DispatchToServers() { } }, policy_->rx_targets()); + RemoveMissingServers(); } void PingCloudServers::ReconcileServer(CloudServerConnection& cloud_sc) { @@ -499,8 +471,7 @@ void PingCloudServers::ReconcileServer(CloudServerConnection& cloud_sc) { auto const priority = cloud_sc.priority(); auto it = server_pings_.find(server_id); - if ((it == server_pings_.end()) || (it->second->priority() != priority) || - it->second->stopped()) { + if ((it == server_pings_.end()) || (it->second->priority() != priority)) { if (it != server_pings_.end()) { it->second.reset(); } @@ -518,7 +489,7 @@ void PingCloudServers::ServerQuarantined(CloudServerConnection* cloud_sc) { } auto it = server_pings_.find(cloud_sc->server_id()); if (it != server_pings_.end()) { - it->second->Stop(); + it->second->PauseForQuarantine(); } } @@ -529,16 +500,33 @@ void PingCloudServers::ServerQuarantineReleased( } auto it = server_pings_.find(cloud_sc->server_id()); if (it != server_pings_.end()) { - server_pings_.erase(it); + it->second->ResumeFromQuarantine(); } } void PingCloudServers::OnServerRxTimingChanged(ServerId server_id) { auto it = server_pings_.find(server_id); - if (it != server_pings_.end() && !it->second->stopped()) { + if (it != server_pings_.end() && !it->second->quarantined()) { it->second->NotifyConfigChanged(); } } + +void PingCloudServers::RemoveMissingServers() { + std::set in_cloud; + for (auto* sc : cloud_server_connections_->servers()) { + if (sc != nullptr) { + in_cloud.insert(sc->server_id()); + } + } + for (auto it = server_pings_.begin(); it != server_pings_.end();) { + if (in_cloud.find(it->first) == in_cloud.end()) { + policy_->RemoveServerFromCloud(it->first); + it = server_pings_.erase(it); + } else { + ++it; + } + } +} } // namespace ae #endif diff --git a/aether/cloud_connections/ping_cloud_servers.h b/aether/cloud_connections/ping_cloud_servers.h index 83fa8046..6ba5f2fc 100644 --- a/aether/cloud_connections/ping_cloud_servers.h +++ b/aether/cloud_connections/ping_cloud_servers.h @@ -29,7 +29,7 @@ # include "aether/ae_context.h" # include "aether/client_connectivity_policy.h" # include "aether/cloud_connections/cloud_server_connections.h" -# include "aether/cloud_connections/local_presence_schedule.h" +# include "aether/cloud_connections/local_presence_machine.h" # include "aether/events/event_subscription.h" # include "aether/executors/executors.h" # include "aether/tasks/manual_task_scheduler.h" @@ -45,64 +45,61 @@ class PingCloudServers { AE_CLASS_NO_COPY_MOVE(ServerPing) - void Stop(); + void PauseForQuarantine(); + void ResumeFromQuarantine(); void NotifyConfigChanged(); - TimePoint next_service_time() const noexcept { return next_ping_time_; } + TimePoint next_service_time() const noexcept { return next_wake_; } std::size_t priority() const noexcept { return priority_; } - RxTimingConf const& timing() const noexcept { return active_conf_; } - bool stopped() const noexcept { return stop_; } + bool quarantined() const noexcept { return machine_.quarantined(); } private: - void ScheduleNext(TimePoint when, PingAttemptKind kind); - void StartAttempt(PingAttemptKind kind); - void Start(); + struct LiveAttempt { + LocalPresenceMachine::SendSpec spec{}; + TimePoint send_time{}; + std::unique_ptr ping; + Subscription result_sub; + }; + + void Pump(); + void ScheduleWake(TimePoint when); + void SyncBlockers(); + void DropFinishedPings(); template void WaitForLink(ClientServerConnection& cc, F&& f); auto EnsureLinked(); Duration SelectedRtt() const; - Duration AttemptTimeout(Duration rtt) const; - - void OnPingResult(std::uint64_t attempt_id, TimePoint send_time, - Duration sent_interval, Duration sent_window, - Ping::PingResult const& res); - void ApplyConfirmedPong(TimePoint send_time, TimePoint pong_time, - Duration sent_interval, Duration sent_window); - void OnAttemptTimeout(std::uint64_t attempt_id); - void AfterFailedAttempt(); - void HoldRxUntil(TimePoint until); + void StartSend(LocalPresenceMachine::SendSpec spec); + void OnPingResult(std::uint64_t attempt_id, Ping::PingResult const& res); + void AddRttSample(Duration measured); void ScheduleRestream(); - void AbandonInFlight(); + void ApplyConfirmed(LocalPresenceMachine::PongOutcome const& outcome); AeContext ae_context_; ClientConnectivityPolicy* policy_; CloudServerConnection* cloud_sc_; ServerId server_id_{}; std::size_t priority_{}; - RxTimingConf active_conf_{}; + LocalPresenceMachine machine_; std::optional> waiter_; - std::optional ping_; - bool stop_{false}; - bool attempt_in_flight_{false}; - std::uint64_t active_attempt_id_{0}; - PingAttemptKind active_attempt_kind_{PingAttemptKind::kInitial}; - TimePoint active_send_time_{}; - Duration active_sent_interval_{}; - Duration active_sent_window_{}; + std::map live_; Subscription link_state_sub_; - TaskSubscription start_sub_; - TaskSubscription attempt_timeout_sub_; - TaskSubscription rx_window_sub_; + TaskSubscription wake_sub_; + TaskSubscription current_window_sub_; TaskSubscription restream_sub_; - ClientConnectivityPolicy::SuspendBlocker ping_blocker_; - ClientConnectivityPolicy::SuspendBlocker rx_window_blocker_; + ClientConnectivityPolicy::SuspendBlocker request_blocker_; + ClientConnectivityPolicy::SuspendBlocker current_window_blocker_; ClientConnectivityPolicy::SuspendBlocker restream_blocker_; - TimePoint next_ping_time_; + TimePoint next_wake_{TimePoint::max()}; + bool stop_{false}; + bool holding_request_{false}; + bool holding_current_{false}; + TimePoint scheduled_current_close_{}; }; public: @@ -118,6 +115,7 @@ class PingCloudServers { void ServerQuarantined(CloudServerConnection* cloud_sc); void ServerQuarantineReleased(CloudServerConnection* cloud_sc); void OnServerRxTimingChanged(ServerId server_id); + void RemoveMissingServers(); AeContext ae_context_; CloudServerConnections* cloud_server_connections_; diff --git a/tests/test-local-presence/CMakeLists.txt b/tests/test-local-presence/CMakeLists.txt index df6cc3ad..9574b556 100644 --- a/tests/test-local-presence/CMakeLists.txt +++ b/tests/test-local-presence/CMakeLists.txt @@ -25,18 +25,23 @@ if(NOT CM_PLATFORM) endif() add_test(NAME ${PROJECT_NAME} COMMAND $) - add_executable(test-local-presence-firewall firewall_live.cpp) - target_include_directories(test-local-presence-firewall PRIVATE ${ROOT_DIR}) - target_link_libraries(test-local-presence-firewall PRIVATE unity aether) - target_compile_definitions(test-local-presence-firewall PRIVATE - "AE_DISTILLATION=1" - ) - if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - target_compile_options(test-local-presence-firewall PRIVATE /Zc:preprocessor) + if(AE_ENABLE_PRIVILEGED_NETWORK_TESTS) + add_executable(test-local-presence-firewall firewall_live.cpp) + target_include_directories(test-local-presence-firewall PRIVATE ${ROOT_DIR}) + target_link_libraries(test-local-presence-firewall PRIVATE unity aether) + target_compile_definitions(test-local-presence-firewall PRIVATE + "AE_DISTILLATION=1" + ) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(test-local-presence-firewall PRIVATE /Zc:preprocessor) + endif() + add_test(NAME test-local-presence-firewall + COMMAND $) + set_tests_properties(test-local-presence-firewall PROPERTIES + TIMEOUT 90 + LABELS "manual;privileged;network" + ) endif() - add_test(NAME test-local-presence-firewall - COMMAND $) - set_tests_properties(test-local-presence-firewall PROPERTIES TIMEOUT 90) else() message(WARNING "Not implemented for ${CM_PLATFORM}") endif() diff --git a/tests/test-local-presence/firewall_live.cpp b/tests/test-local-presence/firewall_live.cpp index f98ed422..4d17066f 100644 --- a/tests/test-local-presence/firewall_live.cpp +++ b/tests/test-local-presence/firewall_live.cpp @@ -144,23 +144,38 @@ void Pump(AetherApp& app, TimePoint until) { } } -TimePoint EarliestConfirmedClose(Client& client) { - auto close = TimePoint::max(); - auto policy = client.connectivity_policy(); - if (!policy) { - return close; +#if defined(_WIN32) +bool IsProcessElevated() { + HANDLE token = nullptr; + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &token)) { + return false; } - for (auto* server : client.cloud_connection().selected_servers()) { - if (server == nullptr) { - continue; - } - auto const* state = policy->FindServerPresence(server->server_id()); - if (state != nullptr && state->has_confirmed_schedule) { - close = std::min(close, state->confirmed_window_close_local); + TOKEN_ELEVATION elevation{}; + DWORD size = 0; + auto const ok = GetTokenInformation(token, TokenElevation, &elevation, + sizeof(elevation), &size); + CloseHandle(token); + return ok && (elevation.TokenIsElevated != 0); +} + +CloudServerConnection* FirstSelectedServer(Client& client) { + auto const& selected = client.cloud_connection().selected_servers(); + for (auto* server : selected) { + if (server != nullptr) { + return server; } } - return close; + return nullptr; +} + +TimePoint ServerConfirmedClose(ClientConnectivityPolicy& policy, ServerId id) { + auto const* state = policy.FindServerPresence(id); + if (state == nullptr || !state->has_confirmed_schedule) { + return TimePoint::max(); + } + return state->confirmed_window_close_local; } +#endif void ApplyOneSecondTimings(Client& client) { auto policy = client.connectivity_policy(); @@ -182,6 +197,12 @@ void test_WindowsFirewallOfflineAndRecovery() { #if !defined(_WIN32) TEST_IGNORE_MESSAGE("Windows Firewall test runs on Win32 only"); #else + if (!IsProcessElevated()) { + TEST_IGNORE_MESSAGE( + "SKIP: privileged firewall test requires Administrator " + "(AE_ENABLE_PRIVILEGED_NETWORK_TESTS=ON)"); + } + auto app = MakeApp(); TEST_ASSERT_NOT_NULL(app.get()); @@ -212,24 +233,38 @@ void test_WindowsFirewallOfflineAndRecovery() { Pump(*app, Now() + 1500ms); TEST_ASSERT_TRUE(client->IsLocallyOnline()); - auto const close = EarliestConfirmedClose(*client.Load()); - TEST_ASSERT_TRUE_MESSAGE(close != TimePoint::max(), + auto* target = FirstSelectedServer(*client.Load()); + TEST_ASSERT_NOT_NULL(target); + auto const sid = target->server_id(); + auto policy = client->connectivity_policy(); + TEST_ASSERT_TRUE(static_cast(policy)); + auto const initial_close = ServerConfirmedClose(*policy.Load(), sid); + TEST_ASSERT_TRUE_MESSAGE(initial_close != TimePoint::max(), "no confirmed receive window"); WindowsExeFirewall fw{ThisExePath()}; auto const fault_time = Now(); - TEST_ASSERT_TRUE_MESSAGE( - fw.Block(), - "netsh advfirewall failed (run the test as Administrator)"); + if (!fw.Block()) { + TEST_IGNORE_MESSAGE( + "SKIP: netsh advfirewall failed (need Administrator)"); + } bool early_offline = false; TimePoint detected_offline{}; - auto const detect_deadline = close + 3s; + TimePoint final_close = initial_close; + int pongs_after_block = 0; + auto detect_deadline = final_close + 5s; while (Now() < detect_deadline && !app->IsExited()) { Pump(*app, Now() + kPoll); - auto const online = client->IsLocallyOnline(); auto const now = Now(); - if (now <= close) { + auto const close_now = ServerConfirmedClose(*policy.Load(), sid); + if (close_now != TimePoint::max() && close_now > final_close) { + ++pongs_after_block; + final_close = close_now; + detect_deadline = final_close + 5s; + } + auto const online = policy->IsServerLocallyOnline(sid, now); + if (now <= final_close) { if (!online) { early_offline = true; detected_offline = now; @@ -241,62 +276,83 @@ void test_WindowsFirewallOfflineAndRecovery() { } } - auto const close_ms = - std::chrono::duration_cast(close - fault_time) - .count(); - auto const detected_ms = + auto const to_ms = [](TimePoint a, TimePoint b) { + return std::chrono::duration_cast(a - b) + .count(); + }; + auto const fault_to_initial = to_ms(initial_close, fault_time); + auto const fault_to_final = to_ms(final_close, fault_time); + auto const fault_to_offline = detected_offline.time_since_epoch().count() == 0 ? -1 - : std::chrono::duration_cast( - detected_offline - fault_time) - .count(); + : to_ms(detected_offline, fault_time); std::printf( - "FIREWALL fault interval_ms=1000 rx_window_ms=1000 " - "fault_to_window_close_ms=%lld detected_offline_after_fault_ms=%lld " + "FIREWALL interval_ms=1000 rx_window_ms=1000 " + "pongs_after_block=%d fault_to_initial_close_ms=%lld " + "final_effective_close_ms=%lld fault_to_offline_ms=%lld " "early_offline=%s\n", - static_cast(close_ms), static_cast(detected_ms), + pongs_after_block, static_cast(fault_to_initial), + static_cast(fault_to_final), + static_cast(fault_to_offline), early_offline ? "YES" : "NO"); if (FILE* log = std::fopen("firewall_result.txt", "w")) { - std::fprintf( - log, - "interval_ms=1000\nrx_window_ms=1000\n" - "fault_to_window_close_ms=%lld\ndetected_offline_after_fault_ms=%lld\n" - "early_offline=%s\n", - static_cast(close_ms), static_cast(detected_ms), - early_offline ? "YES" : "NO"); + std::fprintf(log, + "interval_ms=1000\nrx_window_ms=1000\n" + "pongs_after_block=%d\nfault_to_initial_close_ms=%lld\n" + "final_effective_close_ms=%lld\nfault_to_offline_ms=%lld\n" + "early_offline=%s\n", + pongs_after_block, static_cast(fault_to_initial), + static_cast(fault_to_final), + static_cast(fault_to_offline), + early_offline ? "YES" : "NO"); std::fclose(log); } - TEST_ASSERT_FALSE_MESSAGE(early_offline, - "OFFLINE appeared before confirmed_window_close"); + TEST_ASSERT_FALSE_MESSAGE( + early_offline, "OFFLINE appeared before current_effective_close"); TEST_ASSERT_TRUE_MESSAGE(detected_offline.time_since_epoch().count() != 0, "OFFLINE was not detected after firewall block"); - TEST_ASSERT_TRUE(detected_offline > close); + TEST_ASSERT_TRUE(detected_offline > final_close); fw.Unblock(); auto const recover_from = Now(); + TimePoint first_pong{}; TimePoint recovered{}; + auto last_close = final_close; while (Now() < recover_from + 20s && !app->IsExited()) { Pump(*app, Now() + kPoll); - if (client->IsLocallyOnline()) { + auto const close_now = ServerConfirmedClose(*policy.Load(), sid); + if (first_pong.time_since_epoch().count() == 0 && + close_now != TimePoint::max() && close_now > last_close) { + first_pong = Now(); + } + if (policy->IsServerLocallyOnline(sid, Now())) { recovered = Now(); break; } } - auto const recovery_ms = + auto const unblock_to_pong = + first_pong.time_since_epoch().count() == 0 + ? -1 + : to_ms(first_pong, recover_from); + auto const unblock_to_online = recovered.time_since_epoch().count() == 0 ? -1 - : std::chrono::duration_cast( - recovered - recover_from) - .count(); - std::printf("FIREWALL recovery_latency_ms=%lld\n", - static_cast(recovery_ms)); + : to_ms(recovered, recover_from); + std::printf( + "FIREWALL unblock_to_first_successful_pong_ms=%lld " + "unblock_to_online_ms=%lld\n", + static_cast(unblock_to_pong), + static_cast(unblock_to_online)); if (FILE* log = std::fopen("firewall_result.txt", "a")) { - std::fprintf(log, "recovery_latency_ms=%lld\npass=1\n", - static_cast(recovery_ms)); + std::fprintf(log, + "unblock_to_first_successful_pong_ms=%lld\n" + "unblock_to_online_ms=%lld\npass=1\n", + static_cast(unblock_to_pong), + static_cast(unblock_to_online)); std::fclose(log); } - TEST_ASSERT_TRUE_MESSAGE(client->IsLocallyOnline(), + TEST_ASSERT_TRUE_MESSAGE(policy->IsServerLocallyOnline(sid, Now()), "did not return ONLINE after firewall unblock"); #endif } diff --git a/tests/test-local-presence/main.cpp b/tests/test-local-presence/main.cpp index 7e936571..e0e0b0e3 100644 --- a/tests/test-local-presence/main.cpp +++ b/tests/test-local-presence/main.cpp @@ -18,13 +18,14 @@ #include #include #include +#include #include #include #include -#include "aether/clock.h" #include "aether/client_connectivity_policy.h" +#include "aether/cloud_connections/local_presence_machine.h" #include "aether/cloud_connections/local_presence_schedule.h" #include "aether/types/statistic_counter.h" @@ -34,13 +35,17 @@ using Ms = std::chrono::milliseconds; TimePoint Tp(std::int64_t ms) { return TimePoint{Ms{ms}}; } -Duration Dur(std::int64_t ms) { return std::chrono::duration_cast(Ms{ms}); } +Duration Dur(std::int64_t ms) { + return std::chrono::duration_cast(Ms{ms}); +} std::int64_t ToMs(TimePoint tp) { return std::chrono::duration_cast(tp.time_since_epoch()).count(); } -std::int64_t ToMs(Duration d) { return std::chrono::duration_cast(d).count(); } +std::int64_t ToMs(Duration d) { + return std::chrono::duration_cast(d).count(); +} void test_PrefixFormula() { auto const R = Dur(100); @@ -57,15 +62,16 @@ void test_PrefixFormula() { void test_ConfirmOnlyAfterPong() { ClientConnectivityPolicy policy; ServerId const sid{7}; - policy.ConfigureServerRxTiming(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), - 99); + policy.ConfigureServerRxTiming( + sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), 99); auto* state = policy.FindServerPresence(sid); TEST_ASSERT_NOT_NULL(state); TEST_ASSERT_FALSE(state->has_confirmed_schedule); TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(sid, Tp(0))); TEST_ASSERT_FALSE(policy.IsLocallyOnline(Tp(0))); - policy.ConfirmServerPong(sid, Tp(1000), Tp(1100), Dur(1000), Dur(300)); + policy.ConfirmServerPong(sid, Tp(1000), Tp(1100), Dur(1000), Dur(300), + Dur(100)); state = policy.FindServerPresence(sid); TEST_ASSERT_TRUE(state->has_confirmed_schedule); TEST_ASSERT_EQUAL(2050, ToMs(state->confirmed_window_open_local)); @@ -75,36 +81,37 @@ void test_ConfirmOnlyAfterPong() { TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(sid, Tp(2351))); } +void test_SelectedRttProjectionIgnoresMeasuredPong() { + auto const selected = Dur(100); + auto fast = MakeConfirmedSchedule(Tp(1000), Tp(1020), Dur(1000), Dur(1000), + selected); + auto slow = MakeConfirmedSchedule(Tp(1000), Tp(1400), Dur(1000), Dur(1000), + selected); + TEST_ASSERT_EQUAL(ToMs(fast.window_open_local), ToMs(slow.window_open_local)); + TEST_ASSERT_EQUAL(ToMs(fast.window_close_local), + ToMs(slow.window_close_local)); + TEST_ASSERT_EQUAL(2050, ToMs(fast.window_open_local)); + TEST_ASSERT_TRUE(ToMs(fast.measured_rtt) != ToMs(slow.measured_rtt)); +} + void test_PerServerIndependence() { ClientConnectivityPolicy policy; ServerId const a{1}; ServerId const b{2}; - policy.ConfigureServerRxTiming(a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), - 99); - policy.ConfigureServerRxTiming(b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), - 95); - policy.ConfirmServerPong(a, Tp(0), Tp(100), Dur(1000), Dur(300)); + policy.ConfigureServerRxTiming( + a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), 99); + policy.ConfigureServerRxTiming( + b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), 95); + policy.ConfirmServerPong(a, Tp(0), Tp(100), Dur(1000), Dur(300), Dur(100)); TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(b, Tp(50))); TEST_ASSERT_TRUE(policy.IsServerLocallyOnline(a, Tp(50))); TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(50))); - - auto* sa = policy.FindServerPresence(a); - auto* sb = policy.FindServerPresence(b); - TEST_ASSERT_EQUAL(1000, ToMs(sa->desired.interval)); - TEST_ASSERT_EQUAL(3000, ToMs(sb->desired.interval)); - TEST_ASSERT_EQUAL(99, sa->rtt_reliability_percentile); - TEST_ASSERT_EQUAL(95, sb->rtt_reliability_percentile); - - auto const Ra = Dur(100); - auto const Rb = Dur(200); - TEST_ASSERT_EQUAL(870, ToMs(ComputePrefix1Time(Tp(1050), Ra))); - TEST_ASSERT_EQUAL(2720, ToMs(ComputePrefix1Time(Tp(3050), Rb))); } void test_OfflineOnlyAfterWindowClose() { ClientConnectivityPolicy policy; ServerId const sid{3}; - policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(200)); + policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(200), Dur(40)); TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(1220))); TEST_ASSERT_FALSE(policy.IsLocallyOnline(Tp(1221))); } @@ -112,11 +119,14 @@ void test_OfflineOnlyAfterWindowClose() { void test_RuntimeIntervalChangeKeepsOldConfirmed() { ClientConnectivityPolicy policy; ServerId const sid{4}; - policy.ConfigureServerRxTiming(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(200))); - policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(200)); - auto const close_before = policy.FindServerPresence(sid)->confirmed_window_close_local; + policy.ConfigureServerRxTiming( + sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(200))); + policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(200), Dur(40)); + auto const close_before = + policy.FindServerPresence(sid)->confirmed_window_close_local; - policy.ConfigureServerRxTiming(sid, RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200))); + policy.ConfigureServerRxTiming( + sid, RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200))); auto* state = policy.FindServerPresence(sid); TEST_ASSERT_TRUE(state->config_change_pending); TEST_ASSERT_EQUAL(10000, ToMs(state->desired.interval)); @@ -124,24 +134,11 @@ void test_RuntimeIntervalChangeKeepsOldConfirmed() { TEST_ASSERT_EQUAL(1000, ToMs(state->confirmed_interval)); TEST_ASSERT_TRUE(policy.IsLocallyOnline(close_before)); - policy.ConfirmServerPong(sid, Tp(500), Tp(560), Dur(10000), Dur(200)); + policy.ConfirmServerPong(sid, Tp(500), Tp(560), Dur(10000), Dur(200), + Dur(60)); state = policy.FindServerPresence(sid); TEST_ASSERT_FALSE(state->config_change_pending); TEST_ASSERT_EQUAL(10000, ToMs(state->confirmed_interval)); - - policy.ConfigureServerRxTiming(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(200))); - TEST_ASSERT_EQUAL(10000, ToMs(policy.FindServerPresence(sid)->confirmed_interval)); - policy.ConfirmServerPong(sid, Tp(20000), Tp(20040), Dur(1000), Dur(200)); - TEST_ASSERT_EQUAL(1000, ToMs(policy.FindServerPresence(sid)->confirmed_interval)); -} - -void test_RecoveryAfterOffline() { - ClientConnectivityPolicy policy; - ServerId const sid{5}; - policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(100)); - TEST_ASSERT_FALSE(policy.IsLocallyOnline(Tp(100000))); - policy.ConfirmServerPong(sid, Tp(100100), Tp(100140), Dur(1000), Dur(100)); - TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(100140))); } void test_RuntimePercentile() { @@ -181,8 +178,8 @@ void test_AggregateIgnoresDeselected() { ClientConnectivityPolicy policy; ServerId const a{10}; ServerId const b{11}; - policy.ConfirmServerPong(a, Tp(0), Tp(40), Dur(1000), Dur(200)); - policy.ConfirmServerPong(b, Tp(0), Tp(40), Dur(1000), Dur(200)); + policy.ConfirmServerPong(a, Tp(0), Tp(40), Dur(1000), Dur(200), Dur(40)); + policy.ConfirmServerPong(b, Tp(0), Tp(40), Dur(1000), Dur(200), Dur(40)); TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(50))); policy.SetServerSelectedForAggregate(a, false); policy.SetServerSelectedForAggregate(b, false); @@ -191,149 +188,143 @@ void test_AggregateIgnoresDeselected() { TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(50))); } -void test_OneWayProjection() { - TEST_ASSERT_EQUAL(50, ToMs(OneWayFromRtt(Dur(100)))); -} +void test_OneWayProjection() { TEST_ASSERT_EQUAL(50, ToMs(OneWayFromRtt(Dur(100)))); } void test_MakeConfirmedScheduleDeterministic() { - auto s = MakeConfirmedSchedule(Tp(1000), Tp(1200), Dur(500), Dur(100)); + auto s = + MakeConfirmedSchedule(Tp(1000), Tp(1200), Dur(500), Dur(100), Dur(200)); TEST_ASSERT_EQUAL(1600, ToMs(s.window_open_local)); TEST_ASSERT_EQUAL(1700, ToMs(s.window_close_local)); } -void test_PlanPrefix1FailSchedulesPrefix2() { - auto const plan = PlanAfterFailedAttempt(true, Tp(1050), Tp(1350), - PingAttemptKind::kPrefix1, Tp(870 + 100), - Dur(100)); - TEST_ASSERT_EQUAL(static_cast(PingAttemptKind::kPrefix2), - static_cast(plan.kind)); - TEST_ASSERT_EQUAL(970, ToMs(plan.when)); - TEST_ASSERT_FALSE(plan.mark_offline); -} - -void test_PlanPrefix2FailRetriesWhileOnline() { - auto const plan = PlanAfterFailedAttempt(true, Tp(1050), Tp(1350), - PingAttemptKind::kPrefix2, Tp(1070), - Dur(100)); - TEST_ASSERT_EQUAL(static_cast(PingAttemptKind::kRetry), - static_cast(plan.kind)); - TEST_ASSERT_EQUAL(1170, ToMs(plan.when)); - TEST_ASSERT_FALSE(plan.mark_offline); -} - -void test_PlanAfterCloseIsRecoveryOffline() { - auto const plan = PlanAfterFailedAttempt(true, Tp(1050), Tp(1350), - PingAttemptKind::kRetry, Tp(1351), - Dur(100)); - TEST_ASSERT_EQUAL(static_cast(PingAttemptKind::kRecovery), - static_cast(plan.kind)); - TEST_ASSERT_TRUE(plan.mark_offline); -} - -void test_StaleAttemptRejected() { - TEST_ASSERT_TRUE(IsCurrentPingAttempt(4, 4)); - TEST_ASSERT_FALSE(IsCurrentPingAttempt(5, 4)); -} - -void test_PlanSuccessSchedulesPrefix1() { - auto const plan = PlanAfterSuccessfulPong(Tp(1050), Tp(100), Dur(100), false); - TEST_ASSERT_EQUAL(static_cast(PingAttemptKind::kPrefix1), - static_cast(plan.kind)); - TEST_ASSERT_EQUAL(870, ToMs(plan.when)); +void test_ConfigScopeOverrideAndPriority() { + ClientConnectivityPolicy policy; + ServerId const a{1}; + ServerId const b{2}; + ServerId const c{3}; + policy.BindServerPriority(a, 0); + policy.BindServerPriority(b, 1); + policy.ConfigureServerRxTiming( + a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), 99); + TEST_ASSERT_EQUAL(1000, ToMs(policy.FindServerPresence(a)->desired.interval)); + TEST_ASSERT_EQUAL(AE_PING_INTERVAL_MS, + ToMs(policy.FindServerPresence(b)->desired.interval)); + + policy.ConfigureRxTimings().ForAllPriorities( + RxTimingConf::Every(Dur(3000)).WithWindow(Dur(3000))); + TEST_ASSERT_EQUAL(1000, ToMs(policy.FindServerPresence(a)->desired.interval)); + TEST_ASSERT_EQUAL(3000, ToMs(policy.FindServerPresence(b)->desired.interval)); + + policy.ConfigureRxTimings().ForPriority<0>( + RxTimingConf::Every(Dur(4000)).WithWindow(Dur(4000))); + TEST_ASSERT_EQUAL(1000, ToMs(policy.FindServerPresence(a)->desired.interval)); + TEST_ASSERT_EQUAL(3000, ToMs(policy.FindServerPresence(b)->desired.interval)); + + policy.BindServerPriority(c, 0); + TEST_ASSERT_EQUAL(4000, ToMs(policy.FindServerPresence(c)->desired.interval)); + TEST_ASSERT_EQUAL(3000, ToMs(policy.FindServerPresence(b)->desired.interval)); } -struct AttemptLog { - ServerId server{}; - std::uint64_t attempt_id{}; - PingAttemptKind kind{PingAttemptKind::kInitial}; - TimePoint send_time{}; -}; - struct PollStats { int status_poll_count{}; int online_samples{}; int false_offline_samples{}; int false_offline_transitions{}; - Duration max_false_offline_duration{}; - Duration current_false_offline_duration{}; bool prev_online{false}; bool have_prev{false}; }; -struct SimServer { - ServerId id{}; - Duration schedule_rtt{Dur(100)}; - Duration pong_delay{Dur(20)}; - bool connectivity_ok{true}; - bool stopped{false}; - bool transport_open{true}; - bool in_flight{false}; - int fail_next_attempts{0}; - std::uint64_t active_attempt_id{0}; - PingAttemptKind next_kind{PingAttemptKind::kInitial}; - TimePoint next_due{}; - TimePoint inflight_send{}; - TimePoint inflight_timeout_at{}; - TimePoint inflight_pong_at{}; - Duration inflight_interval{}; - Duration inflight_window{}; - PingAttemptKind inflight_kind{PingAttemptKind::kInitial}; - std::uint64_t inflight_id{0}; - bool inflight_should_fail{false}; -}; - -struct SimCounters { - int prefix1{}; - int prefix2{}; - int retry{}; - int recovery{}; - int initial{}; - int timeouts{}; - int confirmed_pongs{}; - int recoveries_to_online{}; +struct PendingPong { + LocalPresenceMachine::SendSpec spec{}; + TimePoint send_time{}; + TimePoint deliver_at{}; + bool drop{false}; + bool hard_fail{false}; }; -class LocalPresenceRuntime { +class PresenceHarness { public: - explicit LocalPresenceRuntime(TimePoint start) : now_{start} {} + using DelayFn = std::function; + + explicit PresenceHarness(TimePoint start) : now_{start} {} ClientConnectivityPolicy& policy() { return policy_; } TimePoint now() const { return now_; } - SimCounters const& counters() const { return counters_; } PollStats const& poll_stats() const { return poll_stats_; } - std::vector const& attempts() const { return attempts_; } - SimServer& server(ServerId id) { return servers_.at(id); } - bool IsLocallyOnline() const { return policy_.IsLocallyOnline(now_); } + LocalPresenceMachine& machine(ServerId id) { return servers_.at(id).machine; } - void AddServer(ServerId id, RxTimingConf conf, Duration rtt, - std::uint8_t percentile = kDefaultRttReliabilityPercentile) { + LocalPresenceMachine::Counters const& counters(ServerId id) { + return servers_.at(id).machine.counters(); + } + + void AddServer(ServerId id, RxTimingConf conf, Duration seed_rtt, + std::uint8_t percentile = kDefaultRttReliabilityPercentile, + std::size_t priority = 0) { + policy_.BindServerPriority(id, priority); policy_.ConfigureServerRxTiming(id, conf, percentile); - SimServer s{}; + Server s{}; s.id = id; - s.schedule_rtt = rtt; - s.next_due = now_; - s.next_kind = PingAttemptKind::kInitial; - servers_.emplace(id, s); + s.seed_rtt = seed_rtt; + s.percentile = percentile; + for (int i = 0; i < 20; ++i) { + s.stats.Add(seed_rtt); + } + s.machine.SetDesired(now_, conf, percentile); + s.machine.ArmInitial(now_); + servers_.emplace(id, std::move(s)); } - void Quarantine(ServerId id) { - auto& s = servers_.at(id); - s.stopped = true; - s.in_flight = false; - s.next_due = TimePoint::max(); - policy_.SetServerSelectedForAggregate(id, false); - policy_.InvalidateConfirmedSchedule(id); + void SetFixedDelay(ServerId id, Duration delay) { + servers_.at(id).fixed_delay = delay; + } + + void SetDelayFn(ServerId id, DelayFn fn) { servers_.at(id).delay_fn = std::move(fn); } + + void SetDropKind(ServerId id, PingAttemptKind kind, bool drop) { + servers_.at(id).drop_kind[static_cast(kind)] = drop; + } + + void SetConnectivity(ServerId id, bool ok) { + servers_.at(id).connectivity_ok = ok; + } + + bool IsLocallyOnline() const { return policy_.IsLocallyOnline(now_); } + + void SyncBlockers() { + bool need_current = false; + bool need_request = false; + for (auto& [id, s] : servers_) { + static_cast(id); + need_current = need_current || s.machine.current_window_blocker_held(); + need_request = need_request || s.machine.request_blocker_held(); + } + if (need_current) { + if (!have_current_) { + current_block_ = policy_.AcquireSuspendBlock(); + have_current_ = true; + } + } else { + current_block_.Reset(); + have_current_ = false; + } + if (need_request) { + if (!have_request_) { + request_block_ = policy_.AcquireSuspendBlock(); + have_request_ = true; + } + } else { + request_block_.Reset(); + have_request_ = false; + } } - void Release(ServerId id) { + Duration SelectedRtt(ServerId id) { auto& s = servers_.at(id); - s.stopped = false; - s.fail_next_attempts = 0; - s.connectivity_ok = true; - policy_.SetServerSelectedForAggregate(id, true); - s.next_due = now_; - s.next_kind = PingAttemptKind::kInitial; + if (s.stats.empty()) { + return s.seed_rtt; + } + return s.stats.PercentileValue(s.percentile); } void Poll(bool expected_connected) { @@ -341,17 +332,9 @@ class LocalPresenceRuntime { ++poll_stats_.status_poll_count; if (online) { ++poll_stats_.online_samples; - poll_stats_.current_false_offline_duration = {}; } if (expected_connected && !online) { ++poll_stats_.false_offline_samples; - poll_stats_.current_false_offline_duration = - poll_stats_.current_false_offline_duration + Dur(10); - if (poll_stats_.current_false_offline_duration > - poll_stats_.max_false_offline_duration) { - poll_stats_.max_false_offline_duration = - poll_stats_.current_false_offline_duration; - } if (poll_stats_.have_prev && poll_stats_.prev_online) { ++poll_stats_.false_offline_transitions; } @@ -360,46 +343,50 @@ class LocalPresenceRuntime { poll_stats_.have_prev = true; } - void ProcessDue() { + void Process() { bool progress = true; while (progress) { progress = false; for (auto& [id, s] : servers_) { static_cast(id); - if (s.stopped || !s.in_flight) { - continue; - } - if (!s.inflight_should_fail && now_ >= s.inflight_pong_at) { - CompleteSuccess(s); + if (DeliverDue(s)) { progress = true; } } for (auto& [id, s] : servers_) { static_cast(id); - if (s.stopped || !s.in_flight) { - continue; - } - if (now_ >= s.inflight_timeout_at) { - CompleteTimeout(s); + if (TickServer(s)) { progress = true; } } - for (auto& [id, s] : servers_) { - static_cast(id); - if (s.stopped || s.in_flight) { - continue; - } - if (now_ >= s.next_due) { - StartSend(s); - progress = true; - } + } + SyncBlockers(); + } + + void AdvanceTo(TimePoint t) { + if (t < now_) { + return; + } + while (now_ < t) { + Process(); + auto const next = NextEventTime(); + if (next == TimePoint::max() || next > t) { + now_ = t; + Process(); + return; + } + if (next > now_) { + now_ = next; + } else { + now_ = now_ + Dur(1); } } + Process(); } void AdvancePolling(Duration total, Duration step, bool expected_connected) { auto const end = now_ + total; - ProcessDue(); + Process(); while (now_ < end) { auto const next_poll = now_ + step; for (;;) { @@ -410,454 +397,459 @@ class LocalPresenceRuntime { if (ev > now_) { now_ = ev; } - ProcessDue(); + Process(); if (NextEventTime() <= now_) { break; } } now_ = next_poll; - ProcessDue(); + Process(); Poll(expected_connected); } } - void AdvanceTo(TimePoint t) { - if (t < now_) { - return; - } - for (;;) { - ProcessDue(); - if (now_ >= t) { - return; - } - auto const next = NextEventTime(); - if (next == TimePoint::max() || next > t) { - now_ = t; - ProcessDue(); - return; - } - if (next <= now_) { - now_ = t; - ProcessDue(); - return; - } - now_ = next; - } - } - private: + struct Server { + ServerId id{}; + LocalPresenceMachine machine{}; + StatisticsCounter stats{}; + Duration seed_rtt{Dur(100)}; + std::uint8_t percentile{kDefaultRttReliabilityPercentile}; + Duration fixed_delay{Dur(20)}; + DelayFn delay_fn{}; + bool drop_kind[5]{}; + bool connectivity_ok{true}; + int send_count{}; + std::vector pending{}; + }; + TimePoint NextEventTime() const { auto next = TimePoint::max(); for (auto const& [id, s] : servers_) { static_cast(id); - if (s.stopped) { - continue; - } - if (s.in_flight) { - if (!s.inflight_should_fail) { - next = std::min(next, s.inflight_pong_at); - } - next = std::min(next, s.inflight_timeout_at); - } else { - next = std::min(next, s.next_due); + next = std::min(next, s.machine.PeekNextWake()); + for (auto const& p : s.pending) { + next = std::min(next, p.deliver_at); } } return next; } - void CountKind(PingAttemptKind kind) { - switch (kind) { - case PingAttemptKind::kPrefix1: - ++counters_.prefix1; - break; - case PingAttemptKind::kPrefix2: - ++counters_.prefix2; - break; - case PingAttemptKind::kRetry: - ++counters_.retry; - break; - case PingAttemptKind::kRecovery: - ++counters_.recovery; - break; - case PingAttemptKind::kInitial: - ++counters_.initial; - break; - } - } - void StartSend(SimServer& s) { - auto& presence = policy_.EnsureServerPresence(s.id); - ++s.active_attempt_id; - presence.current_attempt_id = s.active_attempt_id; - presence.current_attempt_kind = s.next_kind; - s.in_flight = true; - s.inflight_id = s.active_attempt_id; - s.inflight_kind = s.next_kind; - s.inflight_send = now_; - s.inflight_interval = presence.desired.interval; - s.inflight_window = presence.desired.rx_window; - s.inflight_timeout_at = now_ + s.schedule_rtt; - s.inflight_pong_at = now_ + s.pong_delay; - s.inflight_should_fail = !s.connectivity_ok || (s.fail_next_attempts > 0); - if (s.fail_next_attempts > 0) { - --s.fail_next_attempts; + bool DeliverDue(Server& s) { + bool any = false; + for (auto it = s.pending.begin(); it != s.pending.end();) { + if (it->drop) { + ++it; + continue; + } + if (now_ < it->deliver_at) { + ++it; + continue; + } + if (it->hard_fail) { + s.machine.OnHardFailure(it->spec.attempt_id, now_, SelectedRtt(s.id), + PresenceRestreamReason::kHardWriteFailure); + } else { + auto const measured = + std::chrono::duration_cast(now_ - it->send_time); + s.stats.Add(measured); + auto const selected = SelectedRtt(s.id); + auto outcome = s.machine.OnPong( + it->spec.attempt_id, it->spec.cycle_id, it->send_time, now_, + it->spec.wire_interval, it->spec.desired_interval, + it->spec.rx_window, it->spec.following_open_target, selected); + if (outcome.disposition == + LocalPresenceMachine::PongDisposition::kConfirmedSchedule) { + policy_.ConfirmServerPong( + s.id, outcome.schedule.ping_send_time, + outcome.schedule.pong_receive_time, outcome.schedule.interval, + outcome.schedule.rx_window, outcome.schedule.selected_rtt); + auto& st = policy_.EnsureServerPresence(s.id); + st.confirmed_interval = s.machine.confirmed_interval(); + st.config_change_pending = s.machine.config_change_pending(); + } + } + it = s.pending.erase(it); + any = true; } - CountKind(s.next_kind); - attempts_.push_back(AttemptLog{s.id, s.inflight_id, s.inflight_kind, now_}); + return any; } - void CompleteSuccess(SimServer& s) { - auto const was_offline = !policy_.IsServerLocallyOnline(s.id, now_); - s.in_flight = false; - policy_.ConfirmServerPong(s.id, s.inflight_send, now_, s.inflight_interval, - s.inflight_window); - ++counters_.confirmed_pongs; - if (was_offline) { - ++counters_.recoveries_to_online; + bool TickServer(Server& s) { + auto tick = s.machine.TickNow(now_, SelectedRtt(s.id)); + if (!tick.want_send) { + return false; } - auto* presence = policy_.FindServerPresence(s.id); - auto const plan = PlanAfterSuccessfulPong( - presence->confirmed_window_open_local, now_, s.schedule_rtt, - presence->config_change_pending); - s.next_due = plan.when; - s.next_kind = plan.kind; - } - - void CompleteTimeout(SimServer& s) { - s.in_flight = false; - ++s.active_attempt_id; - auto* presence = policy_.FindServerPresence(s.id); - if (presence != nullptr) { - presence->current_attempt_id = s.active_attempt_id; + s.machine.OnSendStarting(); + s.machine.OnAttemptSent(tick.send, now_); + ++s.send_count; + PendingPong p{}; + p.spec = tick.send; + p.send_time = now_; + auto delay = s.fixed_delay; + if (s.delay_fn) { + delay = s.delay_fn(tick.send.kind, s.send_count); } - ++counters_.timeouts; - auto const plan = PlanAfterFailedAttempt( - presence != nullptr && presence->has_confirmed_schedule, - presence != nullptr ? presence->confirmed_window_open_local : TimePoint{}, - presence != nullptr ? presence->confirmed_window_close_local : TimePoint{}, - s.inflight_kind, now_, s.schedule_rtt); - if (plan.mark_offline && presence != nullptr) { - policy_.MarkServerOffline(s.id, now_); + auto const drop = + !s.connectivity_ok || (ToMs(delay) < 0) || (ToMs(delay) > 5000); + if (!drop) { + p.deliver_at = now_ + delay; + s.pending.push_back(p); } - s.next_due = plan.when; - s.next_kind = plan.kind; + return true; } - ClientConnectivityPolicy policy_; + ClientConnectivityPolicy policy_{}; TimePoint now_{}; - std::map servers_; - std::vector attempts_; - SimCounters counters_{}; + std::map servers_{}; PollStats poll_stats_{}; + ClientConnectivityPolicy::SuspendBlocker current_block_{}; + ClientConnectivityPolicy::SuspendBlocker request_block_{}; + bool have_current_{false}; + bool have_request_{false}; }; void test_SendWithoutPongDoesNotConfirm() { - LocalPresenceRuntime rt{Tp(0)}; + PresenceHarness rt{Tp(0)}; ServerId const sid{1}; - rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100)); - rt.server(sid).connectivity_ok = false; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), + Dur(100)); + rt.SetConnectivity(sid, false); rt.AdvanceTo(Tp(0)); - TEST_ASSERT_EQUAL(1, rt.counters().initial); + TEST_ASSERT_EQUAL(1, rt.counters(sid).initial); TEST_ASSERT_FALSE(rt.IsLocallyOnline()); - TEST_ASSERT_FALSE(rt.policy().FindServerPresence(sid)->has_confirmed_schedule); + TEST_ASSERT_FALSE(rt.machine(sid).has_confirmed_schedule()); } -void test_Prefix1SuccessCancelsPrefix2() { - LocalPresenceRuntime rt{Tp(0)}; +void test_LongSleepSuspendBetweenPongAndPrefix1() { + PresenceHarness rt{Tp(0)}; ServerId const sid{1}; - rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100)); - rt.server(sid).pong_delay = Dur(100); - rt.AdvanceTo(Tp(100)); + auto const interval = Dur(10 * 60 * 1000); + rt.AddServer(sid, RxTimingConf::Every(interval).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); - auto* st = rt.policy().FindServerPresence(sid); - TEST_ASSERT_EQUAL(1050, ToMs(st->confirmed_window_open_local)); - auto const prefix1 = ComputePrefix1Time(st->confirmed_window_open_local, Dur(100)); - auto const prefix2 = ComputePrefix2Time(st->confirmed_window_open_local, Dur(100)); - TEST_ASSERT_EQUAL(870, ToMs(prefix1)); - TEST_ASSERT_EQUAL(970, ToMs(prefix2)); + rt.SyncBlockers(); + TEST_ASSERT_TRUE(rt.policy().GetStatus().can_suspend); + TEST_ASSERT_TRUE(rt.machine(sid).CanSuspend()); + + auto const open = rt.machine(sid).confirmed_window_open(); + auto const prefix1 = ComputePrefix1Time(open, Dur(100)); + rt.AdvanceTo(prefix1 - Dur(1)); + TEST_ASSERT_TRUE(rt.machine(sid).CanSuspend()); + TEST_ASSERT_TRUE(rt.policy().GetStatus().can_suspend); rt.AdvanceTo(prefix1); - TEST_ASSERT_EQUAL(1, rt.counters().prefix1); - rt.AdvanceTo(prefix1 + Dur(20)); - TEST_ASSERT_TRUE(rt.IsLocallyOnline()); - TEST_ASSERT_EQUAL(0, rt.counters().prefix2); - rt.AdvanceTo(prefix2 + Dur(5)); - TEST_ASSERT_EQUAL(0, rt.counters().prefix2); + TEST_ASSERT_FALSE(rt.machine(sid).CanSuspend()); + TEST_ASSERT_FALSE(rt.policy().GetStatus().can_suspend); + + auto const current_c = rt.machine(sid).current_promised_close(); + TEST_ASSERT_TRUE(current_c == rt.machine(sid).confirmed_window_close() || + ToMs(current_c) <= ToMs(rt.machine(sid).confirmed_window_close())); + rt.AdvanceTo(current_c); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + rt.AdvanceTo(current_c + Dur(1)); + TEST_ASSERT_FALSE(rt.machine(sid).current_window_blocker_held()); } -void test_Prefix1FailSendsPrefix2OnTarget() { - LocalPresenceRuntime rt{Tp(0)}; +void test_CurrentVsNextWindowBlocker() { + PresenceHarness rt{Tp(0)}; ServerId const sid{1}; - rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100)); - rt.server(sid).pong_delay = Dur(100); - rt.AdvanceTo(Tp(100)); - auto* st = rt.policy().FindServerPresence(sid); - auto const prefix1 = ComputePrefix1Time(st->confirmed_window_open_local, Dur(100)); - auto const prefix2 = ComputePrefix2Time(st->confirmed_window_open_local, Dur(100)); - rt.server(sid).fail_next_attempts = 1; + rt.AddServer(sid, RxTimingConf::Every(Dur(10000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + auto const c0 = rt.machine(sid).confirmed_window_close(); + TEST_ASSERT_TRUE(rt.machine(sid).CanSuspend()); + auto const prefix1 = + ComputePrefix1Time(rt.machine(sid).confirmed_window_open(), Dur(100)); rt.AdvanceTo(prefix1); - TEST_ASSERT_EQUAL(1, rt.counters().prefix1); - TEST_ASSERT_EQUAL(0, rt.counters().prefix2); - rt.AdvanceTo(prefix1 + Dur(100)); - TEST_ASSERT_EQUAL(1, rt.counters().prefix2); - TEST_ASSERT_EQUAL(970, ToMs(rt.attempts().back().send_time)); - TEST_ASSERT_EQUAL(prefix2.time_since_epoch().count(), - rt.attempts().back().send_time.time_since_epoch().count()); - TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + TEST_ASSERT_TRUE(rt.machine(sid).current_window_blocker_held()); + TEST_ASSERT_EQUAL(ToMs(c0), ToMs(rt.machine(sid).current_promised_close())); + rt.AdvanceTo(prefix1 + Dur(20)); + auto const c1 = rt.machine(sid).confirmed_window_close(); + TEST_ASSERT_TRUE(ToMs(c1) > ToMs(c0)); + TEST_ASSERT_EQUAL(ToMs(c0), ToMs(rt.machine(sid).current_promised_close())); } -void test_Prefix2SuccessNoOffline() { - LocalPresenceRuntime rt{Tp(0)}; +void test_Prefix1LatePongAfterPrefix2() { + PresenceHarness rt{Tp(0)}; ServerId const sid{1}; - rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100)); - rt.server(sid).pong_delay = Dur(100); - rt.AdvanceTo(Tp(100)); - auto* st = rt.policy().FindServerPresence(sid); - auto const prefix1 = ComputePrefix1Time(st->confirmed_window_open_local, Dur(100)); - rt.server(sid).fail_next_attempts = 1; - rt.AdvanceTo(prefix1 + Dur(100)); - TEST_ASSERT_EQUAL(1, rt.counters().prefix2); - rt.AdvanceTo(prefix1 + Dur(120)); + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + auto const open = rt.machine(sid).confirmed_window_open(); + auto const target_before = rt.machine(sid).last_following_target(); + static_cast(target_before); + rt.SetDelayFn(sid, [](PingAttemptKind kind, int) { + if (kind == PingAttemptKind::kPrefix1) { + return Dur(250); + } + return Dur(20); + }); + auto const prefix1 = ComputePrefix1Time(open, Dur(100)); + rt.AdvanceTo(prefix1 + Dur(260)); + TEST_ASSERT_TRUE(rt.counters(sid).prefix2 >= 1); + TEST_ASSERT_EQUAL(0, rt.poll_stats().false_offline_samples); + TEST_ASSERT_TRUE(rt.machine(sid).outstanding_attempt_count() == 0); + TEST_ASSERT_TRUE(rt.counters(sid).late_pongs >= 1); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); - TEST_ASSERT_EQUAL(0, rt.poll_stats().false_offline_transitions); } -void test_PostPrefixRetriesStayOnline() { - LocalPresenceRuntime rt{Tp(0)}; +void test_RttTailEntersStatistics() { + StatisticsCounter stats; + for (int i = 0; i < 100; ++i) { + stats.Add(Dur(100)); + } + TEST_ASSERT_EQUAL(100, ToMs(stats.PercentileValue(99))); + stats.Add(Dur(300)); + TEST_ASSERT_TRUE(ToMs(stats.PercentileValue(99)) >= 300); + + PresenceHarness rt{Tp(0)}; ServerId const sid{1}; - rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(400)), Dur(100)); - rt.server(sid).pong_delay = Dur(100); - rt.AdvanceTo(Tp(100)); - auto* st = rt.policy().FindServerPresence(sid); - auto const close = st->confirmed_window_close_local; - rt.server(sid).connectivity_ok = false; - auto const prefix1 = ComputePrefix1Time(st->confirmed_window_open_local, Dur(100)); - rt.AdvanceTo(prefix1); - rt.AdvanceTo(close); - TEST_ASSERT_TRUE(rt.IsLocallyOnline()); - TEST_ASSERT_TRUE(rt.counters().prefix2 >= 1); - TEST_ASSERT_TRUE(rt.counters().retry >= 1); + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + rt.SetDelayFn(sid, [](PingAttemptKind kind, int) { + if (kind == PingAttemptKind::kPrefix1) { + return Dur(300); + } + return Dur(20); + }); + auto const prefix1 = + ComputePrefix1Time(rt.machine(sid).confirmed_window_open(), Dur(100)); + rt.AdvanceTo(prefix1 + Dur(310)); + TEST_ASSERT_TRUE(ToMs(rt.SelectedRtt(sid)) >= 300); } -void test_RealOfflineAfterWindowClose() { - LocalPresenceRuntime rt{Tp(0)}; +void test_PxxMissDoesNotRestream() { + PresenceHarness rt{Tp(0)}; ServerId const sid{1}; - rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100)); - rt.server(sid).pong_delay = Dur(100); - rt.AdvanceTo(Tp(100)); - auto* st = rt.policy().FindServerPresence(sid); - auto const close = st->confirmed_window_close_local; - rt.server(sid).connectivity_ok = false; - rt.AdvanceTo(close); - TEST_ASSERT_TRUE(rt.IsLocallyOnline()); - rt.AdvanceTo(close + Dur(1)); - TEST_ASSERT_FALSE(rt.IsLocallyOnline()); + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + TEST_ASSERT_EQUAL(0, rt.counters(sid).restreams); + rt.SetDelayFn(sid, [](PingAttemptKind kind, int) { + if (kind == PingAttemptKind::kPrefix1) { + return Dur(250); + } + return Dur(20); + }); + auto const prefix1 = + ComputePrefix1Time(rt.machine(sid).confirmed_window_open(), Dur(100)); + rt.AdvanceTo(prefix1 + Dur(110)); + TEST_ASSERT_TRUE(rt.counters(sid).prefix2 >= 1); + TEST_ASSERT_EQUAL(0, rt.counters(sid).restreams); + TEST_ASSERT_TRUE(rt.machine(sid).outstanding_attempt_count() >= 1); } -void test_RecoveryPongRestoresOnline() { - LocalPresenceRuntime rt{Tp(0)}; - ServerId const sid{1}; - rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(200)), Dur(100)); - rt.server(sid).pong_delay = Dur(100); - rt.AdvanceTo(Tp(100)); - auto* st = rt.policy().FindServerPresence(sid); - rt.server(sid).connectivity_ok = false; - rt.AdvanceTo(st->confirmed_window_close_local + Dur(1)); - TEST_ASSERT_FALSE(rt.IsLocallyOnline()); - rt.server(sid).connectivity_ok = true; - rt.AdvanceTo(rt.now() + Dur(500)); - TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +void test_EnsureLinkedErrorReleasesBlocker() { + LocalPresenceMachine machine; + machine.ArmInitial(Tp(0)); + auto tick = machine.TickNow(Tp(0), Dur(100)); + TEST_ASSERT_TRUE(tick.want_send); + machine.OnSendStarting(); + TEST_ASSERT_TRUE(machine.request_blocker_held()); + machine.OnStartFailed(Tp(0), Dur(100), + PresenceRestreamReason::kConnectionUnavailable); + TEST_ASSERT_FALSE(machine.request_blocker_held()); + TEST_ASSERT_TRUE(machine.CanSuspend()); + TEST_ASSERT_EQUAL(1, machine.counters().restreams); } -void test_RxWindowCloseDoesNotCloseTransport() { - LocalPresenceRuntime rt{Tp(0)}; +void test_QuarantineKeepsConfirmedUntilClose() { + PresenceHarness rt{Tp(0)}; ServerId const sid{1}; - rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(200)), Dur(100)); - rt.server(sid).pong_delay = Dur(100); - rt.AdvanceTo(Tp(100)); - auto* st = rt.policy().FindServerPresence(sid); - rt.server(sid).connectivity_ok = false; - rt.AdvanceTo(st->confirmed_window_close_local + Dur(50)); - TEST_ASSERT_FALSE(rt.IsLocallyOnline()); - TEST_ASSERT_TRUE(rt.server(sid).transport_open); + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + auto const close = rt.machine(sid).confirmed_window_close(); + TEST_ASSERT_TRUE(ToMs(close) >= 2000); + rt.machine(sid).OnQuarantine(Tp(1000)); + TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(sid, Tp(1500))); + TEST_ASSERT_TRUE(rt.policy().IsLocallyOnline(Tp(1500))); + TEST_ASSERT_TRUE(rt.policy().IsLocallyOnline(close)); + TEST_ASSERT_FALSE(rt.policy().IsLocallyOnline(close + Dur(1))); } -void test_QuarantineIndependentAndReleaseRestartsPing() { - LocalPresenceRuntime rt{Tp(0)}; +void test_HardRemovalDropsAggregateImmediately() { + ClientConnectivityPolicy policy; ServerId const a{1}; ServerId const b{2}; - rt.AddServer(a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100), 99); - rt.AddServer(b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), Dur(200), 95); - rt.AdvanceTo(Tp(20)); - TEST_ASSERT_TRUE(rt.IsLocallyOnline()); - TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(a, rt.now())); - TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(b, rt.now())); - - rt.Quarantine(a); - TEST_ASSERT_FALSE(rt.policy().IsServerLocallyOnline(a, rt.now())); - TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(b, rt.now())); - TEST_ASSERT_TRUE(rt.IsLocallyOnline()); - - auto const prefix1_before = rt.counters().prefix1; - rt.AdvanceTo(rt.now() + Dur(200)); - TEST_ASSERT_EQUAL(prefix1_before, rt.counters().prefix1); - - rt.Release(a); - rt.AdvanceTo(rt.now() + Dur(40)); - TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(a, rt.now())); - TEST_ASSERT_TRUE(rt.IsLocallyOnline()); + policy.ConfirmServerPong(a, Tp(0), Tp(40), Dur(1000), Dur(200), Dur(40)); + policy.ConfirmServerPong(b, Tp(0), Tp(40), Dur(1000), Dur(200), Dur(40)); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(50))); + policy.RemoveServerFromCloud(a); + TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(a, Tp(50))); + TEST_ASSERT_TRUE(policy.IsServerLocallyOnline(b, Tp(50))); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(50))); + policy.RemoveServerFromCloud(b); + TEST_ASSERT_FALSE(policy.IsLocallyOnline(Tp(50))); } void test_RuntimeConfigChangeKeepsOldUntilPong() { - LocalPresenceRuntime rt{Tp(0)}; + PresenceHarness rt{Tp(0)}; ServerId const sid{1}; - rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(200)), Dur(100)); + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(200)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); rt.AdvanceTo(Tp(20)); - auto const close_old = rt.policy().FindServerPresence(sid)->confirmed_window_close_local; + auto const close_old = rt.machine(sid).confirmed_window_close(); rt.policy().ConfigureServerRxTiming( sid, RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200))); - TEST_ASSERT_EQUAL(1000, ToMs(rt.policy().FindServerPresence(sid)->confirmed_interval)); - TEST_ASSERT_TRUE(rt.policy().FindServerPresence(sid)->confirmed_window_close_local == - close_old); - rt.server(sid).connectivity_ok = false; + rt.machine(sid).SetDesired( + rt.now(), RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200)), 99); + TEST_ASSERT_EQUAL(1000, ToMs(rt.machine(sid).confirmed_interval())); + TEST_ASSERT_TRUE(rt.machine(sid).confirmed_window_close() == close_old); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_Prefix1SuccessNoPrefix2() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + auto const prefix1 = + ComputePrefix1Time(rt.machine(sid).confirmed_window_open(), Dur(100)); + rt.AdvanceTo(prefix1 + Dur(20)); + TEST_ASSERT_EQUAL(1, rt.counters(sid).prefix1); + TEST_ASSERT_EQUAL(0, rt.counters(sid).prefix2); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_MultiServerIndependentSchedules() { + PresenceHarness rt{Tp(0)}; + ServerId const a{1}; + ServerId const b{2}; + rt.AddServer(a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100), + 99, 0); + rt.AddServer(b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), Dur(200), + 95, 1); + rt.SetFixedDelay(a, Dur(20)); + rt.SetFixedDelay(b, Dur(20)); + rt.AdvanceTo(Tp(40)); + TEST_ASSERT_EQUAL(1000, ToMs(rt.machine(a).confirmed_interval())); + TEST_ASSERT_EQUAL(3000, ToMs(rt.machine(b).confirmed_interval())); + rt.SetConnectivity(a, false); + rt.AdvanceTo(rt.machine(a).confirmed_window_close() + Dur(1)); + TEST_ASSERT_FALSE(rt.policy().IsServerLocallyOnline(a, rt.now())); + TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(b, rt.now())); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); } struct RuntimeReport { int confirmed_cycles{}; Duration duration{}; - TimePoint fault_time{}; - TimePoint confirmed_window_close{}; - TimePoint detected_offline_time{}; - TimePoint recovered_time{}; - bool early_offline{false}; }; RuntimeReport g_stat_report{}; -RuntimeReport g_fault_report{}; void test_StatisticalRuntimePollingIsLocallyOnline() { - LocalPresenceRuntime rt{Tp(0)}; + PresenceHarness rt{Tp(0)}; ServerId const sid{1}; auto const interval = Dur(1000); auto const window = Dur(1000); auto const rtt = Dur(100); rt.AddServer(sid, RxTimingConf::Every(interval).WithWindow(window), rtt, 99); - rt.server(sid).pong_delay = Dur(20); + rt.SetDelayFn(sid, [&rt, sid](PingAttemptKind kind, int) { + auto const prefix1 = rt.counters(sid).prefix1; + if (kind == PingAttemptKind::kPrefix1) { + if ((prefix1 % 19) == 0) { + return Dur(100000); + } + if ((prefix1 % 11) == 0) { + return Dur(150); + } + if ((prefix1 % 17) == 0) { + return Dur(250); + } + } + if ((kind == PingAttemptKind::kPrefix2) && ((prefix1 % 19) == 0)) { + return Dur(100000); + } + return Dur(20); + }); rt.AdvanceTo(Tp(20)); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); auto const measure_start = rt.now(); - auto const min_duration = Dur(300000); - constexpr int kMinCycles = 300; - while (true) { rt.AdvancePolling(Dur(10), Dur(10), true); auto const elapsed_ms = std::chrono::duration_cast(rt.now() - measure_start).count(); - if (elapsed_ms >= 300000 && rt.counters().confirmed_pongs >= kMinCycles) { + if (elapsed_ms >= 300000) { break; } TEST_ASSERT_TRUE(elapsed_ms < 400000); } - g_stat_report.confirmed_cycles = rt.counters().confirmed_pongs; + g_stat_report.confirmed_cycles = rt.counters(sid).confirmed_pongs; g_stat_report.duration = std::chrono::duration_cast(rt.now() - measure_start); + auto const cycles = rt.counters(sid).confirmed_pongs; std::printf( "STATISTICAL runtime\n" " duration_ms=%lld confirmed_pongs=%d status_polls=%d online_samples=%d\n" - " false_offline_samples=%d false_offline_transitions=%d " - "max_false_offline_ms=%lld\n" - " prefix1=%d prefix2=%d post_prefix_retry=%d timeouts=%d recoveries=%d\n" + " false_offline_samples=%d false_offline_transitions=%d\n" + " prefix1=%d prefix2=%d post_prefix_retry=%d late_pongs=%d " + "timeouts=%d recoveries=%d restreams=%d\n" " rtt_percentile=99 selected_rtt_ms=%lld guard_ms=30\n", - static_cast(ToMs(g_stat_report.duration)), - rt.counters().confirmed_pongs, rt.poll_stats().status_poll_count, - rt.poll_stats().online_samples, rt.poll_stats().false_offline_samples, - rt.poll_stats().false_offline_transitions, - static_cast(ToMs(rt.poll_stats().max_false_offline_duration)), - rt.counters().prefix1, rt.counters().prefix2, rt.counters().retry, - rt.counters().timeouts, rt.counters().recoveries_to_online, - static_cast(ToMs(rtt))); + static_cast(ToMs(g_stat_report.duration)), cycles, + rt.poll_stats().status_poll_count, rt.poll_stats().online_samples, + rt.poll_stats().false_offline_samples, + rt.poll_stats().false_offline_transitions, rt.counters(sid).prefix1, + rt.counters(sid).prefix2, rt.counters(sid).retry, + rt.counters(sid).late_pongs, rt.counters(sid).scheduler_timeouts, + rt.counters(sid).recoveries_to_online, rt.counters(sid).restreams, + static_cast(ToMs(rt.SelectedRtt(sid)))); TEST_ASSERT_EQUAL(0, rt.poll_stats().false_offline_samples); TEST_ASSERT_EQUAL(0, rt.poll_stats().false_offline_transitions); - TEST_ASSERT_TRUE(rt.counters().confirmed_pongs >= kMinCycles); - TEST_ASSERT_TRUE(rt.counters().prefix2 == 0); + TEST_ASSERT_TRUE(cycles >= 280); + TEST_ASSERT_TRUE(cycles <= 330); + TEST_ASSERT_TRUE(rt.counters(sid).prefix2 > 0); + TEST_ASSERT_TRUE(rt.counters(sid).retry > 0); + TEST_ASSERT_EQUAL(0, rt.counters(sid).restreams); } void test_FaultOfflineNotBeforeWindowCloseThenRecovery() { - LocalPresenceRuntime rt{Tp(0)}; + PresenceHarness rt{Tp(0)}; ServerId const sid{1}; - rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), Dur(100)); - rt.server(sid).pong_delay = Dur(20); + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); rt.AdvanceTo(Tp(20)); rt.AdvancePolling(Dur(2000), Dur(10), true); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); - auto* st = rt.policy().FindServerPresence(sid); - auto const close = st->confirmed_window_close_local; - g_fault_report.fault_time = rt.now(); - g_fault_report.confirmed_window_close = close; - rt.server(sid).connectivity_ok = false; + auto const close = rt.machine(sid).confirmed_window_close(); + rt.SetConnectivity(sid, false); auto detected = TimePoint{}; while (rt.now() < close + Dur(2000)) { rt.AdvancePolling(Dur(10), Dur(10), false); if (!rt.IsLocallyOnline()) { detected = rt.now(); - g_fault_report.early_offline = !(detected > close); break; } } - g_fault_report.detected_offline_time = detected; - TEST_ASSERT_FALSE(g_fault_report.early_offline); TEST_ASSERT_TRUE(detected > close); - TEST_ASSERT_TRUE(ToMs(detected) >= ToMs(close)); - - rt.server(sid).connectivity_ok = true; + rt.SetConnectivity(sid, true); auto const recover_from = rt.now(); while (rt.now() < recover_from + Dur(2000)) { rt.AdvancePolling(Dur(10), Dur(10), false); if (rt.IsLocallyOnline()) { - g_fault_report.recovered_time = rt.now(); break; } } TEST_ASSERT_TRUE(rt.IsLocallyOnline()); - auto const recovery_ms = ToMs(g_fault_report.recovered_time) - ToMs(recover_from); - std::printf( - "FAULT runtime\n" - " fault_ms=%lld window_close_ms=%lld detected_offline_ms=%lld\n" - " early_offline=%s recovery_latency_ms=%lld\n", - static_cast(ToMs(g_fault_report.fault_time)), - static_cast(ToMs(close)), - static_cast(ToMs(detected)), - g_fault_report.early_offline ? "YES" : "NO", - static_cast(recovery_ms)); -} - -void test_MultiServerRuntimeIndependence() { - LocalPresenceRuntime rt{Tp(0)}; - ServerId const a{1}; - ServerId const b{2}; - rt.AddServer(a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100), 99); - rt.AddServer(b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), Dur(200), 95); - rt.AdvanceTo(Tp(40)); - auto* sa = rt.policy().FindServerPresence(a); - auto* sb = rt.policy().FindServerPresence(b); - TEST_ASSERT_EQUAL(1000, ToMs(sa->confirmed_interval)); - TEST_ASSERT_EQUAL(3000, ToMs(sb->confirmed_interval)); - TEST_ASSERT_EQUAL(300, ToMs(sa->confirmed_rx_window)); - TEST_ASSERT_EQUAL(700, ToMs(sb->confirmed_rx_window)); - auto const p1a = ComputePrefix1Time(sa->confirmed_window_open_local, Dur(100)); - auto const p1b = ComputePrefix1Time(sb->confirmed_window_open_local, Dur(200)); - TEST_ASSERT_TRUE(ToMs(p1a) != ToMs(p1b)); - rt.server(a).connectivity_ok = false; - rt.AdvanceTo(sa->confirmed_window_close_local + Dur(1)); - TEST_ASSERT_FALSE(rt.policy().IsServerLocallyOnline(a, rt.now())); - TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(b, rt.now())); - TEST_ASSERT_TRUE(rt.IsLocallyOnline()); } } // namespace ae::test_local_presence @@ -869,32 +861,29 @@ int main() { UNITY_BEGIN(); RUN_TEST(ae::test_local_presence::test_PrefixFormula); RUN_TEST(ae::test_local_presence::test_ConfirmOnlyAfterPong); + RUN_TEST(ae::test_local_presence::test_SelectedRttProjectionIgnoresMeasuredPong); RUN_TEST(ae::test_local_presence::test_PerServerIndependence); RUN_TEST(ae::test_local_presence::test_OfflineOnlyAfterWindowClose); RUN_TEST(ae::test_local_presence::test_RuntimeIntervalChangeKeepsOldConfirmed); - RUN_TEST(ae::test_local_presence::test_RecoveryAfterOffline); RUN_TEST(ae::test_local_presence::test_RuntimePercentile); RUN_TEST(ae::test_local_presence::test_ReliabilityP95VsP99PrefixTimes); RUN_TEST(ae::test_local_presence::test_AggregateIgnoresDeselected); RUN_TEST(ae::test_local_presence::test_OneWayProjection); RUN_TEST(ae::test_local_presence::test_MakeConfirmedScheduleDeterministic); - RUN_TEST(ae::test_local_presence::test_PlanPrefix1FailSchedulesPrefix2); - RUN_TEST(ae::test_local_presence::test_PlanPrefix2FailRetriesWhileOnline); - RUN_TEST(ae::test_local_presence::test_PlanAfterCloseIsRecoveryOffline); - RUN_TEST(ae::test_local_presence::test_StaleAttemptRejected); - RUN_TEST(ae::test_local_presence::test_PlanSuccessSchedulesPrefix1); + RUN_TEST(ae::test_local_presence::test_ConfigScopeOverrideAndPriority); RUN_TEST(ae::test_local_presence::test_SendWithoutPongDoesNotConfirm); - RUN_TEST(ae::test_local_presence::test_Prefix1SuccessCancelsPrefix2); - RUN_TEST(ae::test_local_presence::test_Prefix1FailSendsPrefix2OnTarget); - RUN_TEST(ae::test_local_presence::test_Prefix2SuccessNoOffline); - RUN_TEST(ae::test_local_presence::test_PostPrefixRetriesStayOnline); - RUN_TEST(ae::test_local_presence::test_RealOfflineAfterWindowClose); - RUN_TEST(ae::test_local_presence::test_RecoveryPongRestoresOnline); - RUN_TEST(ae::test_local_presence::test_RxWindowCloseDoesNotCloseTransport); - RUN_TEST(ae::test_local_presence::test_QuarantineIndependentAndReleaseRestartsPing); + RUN_TEST(ae::test_local_presence::test_LongSleepSuspendBetweenPongAndPrefix1); + RUN_TEST(ae::test_local_presence::test_CurrentVsNextWindowBlocker); + RUN_TEST(ae::test_local_presence::test_Prefix1LatePongAfterPrefix2); + RUN_TEST(ae::test_local_presence::test_RttTailEntersStatistics); + RUN_TEST(ae::test_local_presence::test_PxxMissDoesNotRestream); + RUN_TEST(ae::test_local_presence::test_EnsureLinkedErrorReleasesBlocker); + RUN_TEST(ae::test_local_presence::test_QuarantineKeepsConfirmedUntilClose); + RUN_TEST(ae::test_local_presence::test_HardRemovalDropsAggregateImmediately); RUN_TEST(ae::test_local_presence::test_RuntimeConfigChangeKeepsOldUntilPong); + RUN_TEST(ae::test_local_presence::test_Prefix1SuccessNoPrefix2); + RUN_TEST(ae::test_local_presence::test_MultiServerIndependentSchedules); RUN_TEST(ae::test_local_presence::test_StatisticalRuntimePollingIsLocallyOnline); RUN_TEST(ae::test_local_presence::test_FaultOfflineNotBeforeWindowCloseThenRecovery); - RUN_TEST(ae::test_local_presence::test_MultiServerRuntimeIndependence); return UNITY_END(); } From 5ea9920dfe85b378cf39f6488e9eaebe42355fb7 Mon Sep 17 00:00:00 2001 From: aethernet-io Date: Wed, 2 Sep 2026 18:40:15 -0700 Subject: [PATCH 04/11] Add client-side offline_detection_timeout and Remote Presence AND aggregation. Local ONLINE uses expected_open + offline_detection_timeout (not rx_window). QueryPeerPresence aggregates get_client_timing over peer/observer usable servers with Offline-any / Online-all / Unknown otherwise. Co-authored-by: Cursor --- aether/CMakeLists.txt | 3 +- aether/ae_actions/query_peer_presence.cpp | 371 ++++++++++++++++++ aether/ae_actions/query_peer_presence.h | 106 +++++ aether/client.cpp | 12 + aether/client.h | 4 + aether/client_connectivity_policy.cpp | 34 +- aether/client_connectivity_policy.h | 13 +- aether/cloud_connections/cloud_request.cpp | 75 +++- aether/cloud_connections/cloud_request.h | 40 ++ .../local_presence_machine.cpp | 31 +- .../local_presence_machine.h | 6 + .../local_presence_schedule.h | 22 ++ .../cloud_connections/ping_cloud_servers.cpp | 9 + aether/config.h | 7 + aether/remote_presence.h | 208 ++++++++++ aether/work_cloud_api/client_timing.h | 37 ++ .../work_server_api/authorized_api.cpp | 1 + .../work_server_api/authorized_api.h | 4 + tests/test-local-presence/firewall_live.cpp | 5 +- tests/test-local-presence/main.cpp | 214 +++++++++- 20 files changed, 1171 insertions(+), 31 deletions(-) create mode 100644 aether/ae_actions/query_peer_presence.cpp create mode 100644 aether/ae_actions/query_peer_presence.h create mode 100644 aether/remote_presence.h create mode 100644 aether/work_cloud_api/client_timing.h diff --git a/aether/CMakeLists.txt b/aether/CMakeLists.txt index 52c9885a..de300632 100644 --- a/aether/CMakeLists.txt +++ b/aether/CMakeLists.txt @@ -71,7 +71,8 @@ list(APPEND aether_srcs "ae_actions/ping.cpp" "ae_actions/check_access_for_send_message.cpp" "ae_actions/telemetry.cpp" - "ae_actions/select_client.cpp") + "ae_actions/select_client.cpp" + "ae_actions/query_peer_presence.cpp") list(APPEND aether_srcs "registration/api/client_reg_api_safe.cpp" diff --git a/aether/ae_actions/query_peer_presence.cpp b/aether/ae_actions/query_peer_presence.cpp new file mode 100644 index 00000000..37a59308 --- /dev/null +++ b/aether/ae_actions/query_peer_presence.cpp @@ -0,0 +1,371 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "aether/ae_actions/query_peer_presence.h" + +#include "aether/client.h" +#include "aether/cloud_connections/cloud_server_connection.h" +#include "aether/config.h" +#include "aether/connection_manager/client_cloud_manager.h" +#include "aether/server.h" +#include "aether/tele.h" +#include "aether/work_cloud_api/work_server_api/authorized_api.h" + +namespace ae { + +QueryPeerPresence::QueryPeerPresence(AeContext const& ae_context, + Client& client, Uid peer_uid) + : ae_context_{ae_context}, client_{&client}, peer_uid_{peer_uid} { + // Prefer peer Personal Cloud (cached or GetCloud). Fall back to the + // observer's linked cloud only when peer cloud is unavailable — each + // get_client_timing(uid) answer is still authoritative for that server. + auto cached = client_->cloud_manager()->GetCachedCloud(peer_uid_); + if (cached && cached.is_valid() && !cached->servers().empty()) { + dest_cloud_ = std::make_unique( + ae_context_, cached.Load(), + client_->server_connection_manager().GetServerConnectionFactory(), + AE_CLOUD_MAX_SERVER_CONNECTIONS); + work_cloud_ = dest_cloud_.get(); + StartQuery(); + return; + } + + auto& get_cloud = client_->cloud_manager()->GetCloud(peer_uid_); + get_cloud_sub_ = get_cloud.result_event().Subscribe( + [this](Result result) { OnCloud(std::move(result)); }); +} + +QueryPeerPresence::~QueryPeerPresence() { + finished_ = true; + timing_subs_.clear(); +} + +QueryPeerPresence::ResultEvent::Subscriber +QueryPeerPresence::result_event() noexcept { + return EventSubscriber{result_event_}; +} + +Duration QueryPeerPresence::OfflineTimeout() const noexcept { + auto policy = client_->connectivity_policy(); + if (!policy) { + return DefaultOfflineDetectionTimeout(); + } + return policy.Load()->offline_detection_timeout(); +} + +void QueryPeerPresence::OnCloud(Result result) { + if (finished_) { + return; + } + if (!result) { + // Peer cloud unavailable — fall back to observer cloud if it has usable + // servers. get_client_timing remains per-server authoritative. + auto& own = client_->cloud_connection(); + bool any_usable = false; + for (auto* sc : own.selected_servers()) { + if (sc != nullptr && sc->server() && !sc->quarantine()) { + any_usable = true; + break; + } + } + if (!any_usable) { + Complete(PeerPresence{PeerPresenceState::kUnknown}); + return; + } + work_cloud_ = &own; + StartQuery(); + return; + } + auto cloud = std::move(result).value(); + dest_cloud_ = std::make_unique( + ae_context_, cloud.Load(), + client_->server_connection_manager().GetServerConnectionFactory(), + AE_CLOUD_MAX_SERVER_CONNECTIONS); + work_cloud_ = dest_cloud_.get(); + StartQuery(); +} + +void QueryPeerPresence::RefreshUsableSet() { + samples_.clear(); + if (work_cloud_ == nullptr) { + return; + } + for (auto* sc : work_cloud_->selected_servers()) { + if (sc == nullptr || !sc->server()) { + continue; + } + RemoteServerPresenceSample sample{}; + sample.server_id = sc->server_id(); + if (sc->quarantine()) { + sample.status = RemoteServerPresence::kExcluded; + } else { + sample.status = RemoteServerPresence::kUnknown; + } + samples_.push_back(sample); + } +} + +void QueryPeerPresence::StartQuery() { + if (finished_ || work_cloud_ == nullptr) { + return; + } + RefreshUsableSet(); + std::size_t usable = 0; + for (auto const& sample : samples_) { + if (sample.status != RemoteServerPresence::kExcluded) { + ++usable; + } + } + if (usable == 0) { + Complete(PeerPresence{PeerPresenceState::kUnknown}); + return; + } + + quarantine_sub_ = + work_cloud_->server_quarantined_event().Subscribe( + [this](CloudServerConnection* sc) { + if (finished_ || sc == nullptr) { + return; + } + MarkExcluded(sc->server_id()); + MaybeComplete(); + }); + quarantine_release_sub_ = + work_cloud_->server_quarantine_release_event().Subscribe( + [this](CloudServerConnection* sc) { + if (finished_ || sc == nullptr) { + return; + } + // Recovered server re-enters usable set only after a new response. + RemoteServerPresenceSample sample{}; + sample.server_id = sc->server_id(); + sample.status = RemoteServerPresence::kUnknown; + bool found = false; + for (auto& existing : samples_) { + if (existing.server_id == sample.server_id) { + existing = sample; + found = true; + break; + } + } + if (!found) { + samples_.push_back(sample); + } + MaybeComplete(); + }); + + cloud_request_.emplace( + ae_context_, + ApiRequestHandler{[this](ApiContext& auth_api, + CloudServerConnection* sc, + CloudRequest* request) { + if (finished_ || sc == nullptr || !sc->server() || sc->quarantine()) { + return; + } + auto const server_id = sc->server_id(); + for (auto const& sample : samples_) { + if (sample.server_id == server_id && + (sample.status == RemoteServerPresence::kOnline || + sample.status == RemoteServerPresence::kOffline || + sample.status == RemoteServerPresence::kExcluded)) { + return; + } + } + auto& meta = attempts_[server_id]; + ++meta.generation; + meta.send_time = Now(); + auto const generation = meta.generation; + timing_subs_[server_id] = + auth_api->get_client_timing(peer_uid_).Subscribe( + [this, sc, generation](auto const& res) { + OnServerTiming(sc, generation, res); + }); + static_cast(request); + }}, + *work_cloud_, RequestPolicy::All{}, + /*max_retries=*/kRemotePresenceQueryRetryCount + 1); + + exhausted_sub_ = cloud_request_->attempt_exhausted_event().Subscribe( + [this](CloudServerConnection* sc) { + if (finished_ || sc == nullptr) { + return; + } + MarkUnknown(sc->server_id()); + MaybeComplete(); + }); + + cloud_request_sub_ = cloud_request_->result_event().Subscribe([this](bool ok) { + if (finished_) { + return; + } + if (ok) { + return; + } + for (auto& sample : samples_) { + if (sample.status == RemoteServerPresence::kUnknown && + !sample.has_timing) { + // leave as Unknown contribution + } + } + MaybeComplete(); + if (!finished_) { + // All servers exhausted without Offline/Online completion. + Complete(AggregateRemotePresence(samples_)); + } + }); +} + +void QueryPeerPresence::OnServerTiming( + CloudServerConnection* sc, std::uint64_t generation, + Result const& res) { + if (finished_ || sc == nullptr) { + return; + } + auto const server_id = sc->server_id(); + auto meta_it = attempts_.find(server_id); + if (meta_it == attempts_.end() || meta_it->second.generation != generation) { + return; + } + if (!res) { + if (cloud_request_.has_value()) { + auto const exhausted = cloud_request_->FailAttempt(sc); + if (exhausted) { + MarkUnknown(server_id); + MaybeComplete(); + } + } + return; + } + + auto const recv = Now(); + TimePoint expected{}; + TimePoint deadline{}; + auto const status = ClassifyRemoteServerPresence( + recv, meta_it->second.send_time, recv, res.value(), OfflineTimeout(), + &expected, &deadline); + + for (auto& sample : samples_) { + if (sample.server_id != server_id) { + continue; + } + sample.status = status; + sample.expected_open = expected; + sample.offline_deadline = deadline; + sample.next_ping_delta_ms = res.value().next_ping_delta_ms; + sample.has_timing = true; + break; + } + + AE_TELED_DEBUG( + "REMOTE_PRESENCE server {} next_delta {} status {}", server_id, + res.value().next_ping_delta_ms, static_cast(status)); + + if (cloud_request_.has_value()) { + cloud_request_->SucceedAttempt(sc); + } + MaybeComplete(); +} + +void QueryPeerPresence::MarkUnknown(ServerId server_id) { + for (auto& sample : samples_) { + if (sample.server_id == server_id && + sample.status != RemoteServerPresence::kExcluded && + sample.status != RemoteServerPresence::kOnline && + sample.status != RemoteServerPresence::kOffline) { + sample.status = RemoteServerPresence::kUnknown; + return; + } + } +} + +void QueryPeerPresence::MarkExcluded(ServerId server_id) { + for (auto& sample : samples_) { + if (sample.server_id == server_id) { + sample.status = RemoteServerPresence::kExcluded; + return; + } + } +} + +void QueryPeerPresence::MaybeComplete() { + if (finished_) { + return; + } + if (RemotePresenceCanEarlyCompleteOffline(samples_)) { + Complete(PeerPresence{PeerPresenceState::kOffline}); + return; + } + if (RemotePresenceReadyForOnline(samples_)) { + Complete(PeerPresence{PeerPresenceState::kOnline}); + return; + } + // All remaining usable samples known (Online/Unknown) and no Offline — + // wait until every usable server has a terminal observation. + bool any_pending = false; + std::size_t usable = 0; + for (auto const& sample : samples_) { + if (sample.status == RemoteServerPresence::kExcluded) { + continue; + } + ++usable; + if (sample.status == RemoteServerPresence::kUnknown && !sample.has_timing) { + // Still waiting unless retries exhausted left it Unknown without timing. + auto it = attempts_.find(sample.server_id); + if (it == attempts_.end()) { + any_pending = true; + } + } + } + if (usable == 0) { + Complete(PeerPresence{PeerPresenceState::kUnknown}); + return; + } + static_cast(any_pending); +} + +void QueryPeerPresence::Complete(PeerPresence const& presence) { + if (finished_) { + return; + } + finished_ = true; + timing_subs_.clear(); + exhausted_sub_.Reset(); + quarantine_sub_.Reset(); + quarantine_release_sub_.Reset(); + if (cloud_request_.has_value()) { + cloud_request_->Succeeded(); + } + result_event_.Emit(Ok{presence}); + Finish(); +} + +void QueryPeerPresence::Fail(int code) { + if (finished_) { + return; + } + finished_ = true; + timing_subs_.clear(); + exhausted_sub_.Reset(); + quarantine_sub_.Reset(); + quarantine_release_sub_.Reset(); + if (cloud_request_.has_value()) { + cloud_request_->Failed(); + } + result_event_.Emit(Error{code}); + Finish(); +} + +} // namespace ae diff --git a/aether/ae_actions/query_peer_presence.h b/aether/ae_actions/query_peer_presence.h new file mode 100644 index 00000000..ab7e1201 --- /dev/null +++ b/aether/ae_actions/query_peer_presence.h @@ -0,0 +1,106 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_AE_ACTIONS_QUERY_PEER_PRESENCE_H_ +#define AETHER_AE_ACTIONS_QUERY_PEER_PRESENCE_H_ + +#include +#include +#include +#include +#include + +#include "aether-miscpp/types/result.h" +#include "aether/ae_context.h" +#include "aether/actions/action.h" +#include "aether/cloud.h" +#include "aether/cloud_connections/cloud_request.h" +#include "aether/events/event_subscription.h" +#include "aether/events/events.h" +#include "aether/remote_presence.h" +#include "aether/types/server_id.h" +#include "aether/types/uid.h" + +namespace ae { + +class Client; + +enum class QueryPeerPresenceError : int { + kGetCloudFailed = 1, + kNoWorkServerAvailable = 2, + kGetClientTimingFailed = 3, +}; + +// Asynchronous Remote Presence over authoritative usable servers. +// Aggregation: Offline = any Offline; Online = all usable Online; +// Unknown = zero usable or incomplete Online set without Offline. +class QueryPeerPresence final : public Action { + public: + using ResultEvent = Event)>; + + QueryPeerPresence(AeContext const& ae_context, Client& client, Uid peer_uid); + ~QueryPeerPresence() override; + + AE_CLASS_NO_COPY_MOVE(QueryPeerPresence) + + ResultEvent::Subscriber result_event() noexcept; + Uid peer_uid() const noexcept { return peer_uid_; } + std::vector const& samples() const noexcept { + return samples_; + } + + private: + struct AttemptMeta { + TimePoint send_time{}; + std::uint64_t generation{0}; + std::size_t retries_used{0}; + }; + + void OnCloud(Result result); + void StartQuery(); + void RefreshUsableSet(); + void OnServerTiming(CloudServerConnection* sc, std::uint64_t generation, + Result const& res); + void MarkUnknown(ServerId server_id); + void MarkExcluded(ServerId server_id); + void MaybeComplete(); + void Complete(PeerPresence const& presence); + void Fail(int code); + Duration OfflineTimeout() const noexcept; + + AeContext ae_context_; + Client* client_{nullptr}; + Uid peer_uid_{}; + ResultEvent result_event_; + + Subscription get_cloud_sub_; + Subscription cloud_request_sub_; + Subscription exhausted_sub_; + Subscription quarantine_sub_; + Subscription quarantine_release_sub_; + + std::unique_ptr dest_cloud_; + CloudServerConnections* work_cloud_{nullptr}; + std::optional cloud_request_; + std::map timing_subs_; + std::map attempts_; + std::vector samples_; + bool finished_{false}; +}; + +} // namespace ae + +#endif // AETHER_AE_ACTIONS_QUERY_PEER_PRESENCE_H_ diff --git a/aether/client.cpp b/aether/client.cpp index a4a05469..719c58f5 100644 --- a/aether/client.cpp +++ b/aether/client.cpp @@ -18,6 +18,7 @@ #include +#include "aether/ae_actions/query_peer_presence.h" #include "aether/ae_actions/telemetry.h" #include "aether/aether.h" @@ -99,6 +100,17 @@ bool Client::IsLocallyOnline() const { return connectivity_policy_.Load()->IsLocallyOnline(); } +QueryPeerPresence& Client::QueryPeerPresence(Uid peer_uid) { + if (query_peer_presence_ && !query_peer_presence_->is_finished() && + query_peer_presence_->peer_uid() == peer_uid) { + return *query_peer_presence_; + } + query_peer_presence_ = + std::make_unique<::ae::QueryPeerPresence>(AeContext{*aether_}, *this, + peer_uid); + return *query_peer_presence_; +} + P2pMessageStreamManager& Client::message_stream_manager() { if (!message_stream_manager_) { message_stream_manager_ = std::make_unique( diff --git a/aether/client.h b/aether/client.h index a82c298c..f4c76c94 100644 --- a/aether/client.h +++ b/aether/client.h @@ -39,6 +39,7 @@ namespace ae { class Aether; class Telemetry; +class QueryPeerPresence; class Client : public Obj { AE_OBJECT(Client, Obj, 0) @@ -65,6 +66,8 @@ class Client : public Obj { ClientConnectivityPolicy::ptr const& connectivity_policy(); // Read-only aggregate Local ONLINE (no side effects). bool IsLocallyOnline() const; + // Asynchronous Remote Presence query (ONLINE / OFFLINE / UNKNOWN). + ::ae::QueryPeerPresence& QueryPeerPresence(Uid peer_uid); P2pMessageStreamManager& message_stream_manager(); void SetConfig(std::string client_id, Uid parent_uid, Uid uid, @@ -93,6 +96,7 @@ class Client : public Obj { std::unique_ptr server_connection_manager_; std::unique_ptr cloud_connection_; std::unique_ptr message_stream_manager_; + std::unique_ptr<::ae::QueryPeerPresence> query_peer_presence_; #if AE_ENABLE_PING std::unique_ptr ping_cloud_servers_; diff --git a/aether/client_connectivity_policy.cpp b/aether/client_connectivity_policy.cpp index 1814a308..0d3fe286 100644 --- a/aether/client_connectivity_policy.cpp +++ b/aether/client_connectivity_policy.cpp @@ -207,6 +207,20 @@ void ClientConnectivityPolicy::ConfirmServerPong(ServerId server_id, Duration rx_window, Duration selected_rtt) { auto& state = EnsureServerPresence(server_id); + // interval == 0 clears the future Presence promise after the server + // accepted the reset Ping. rx_window is unrelated to Presence. + if (interval <= Duration{}) { + state.has_confirmed_schedule = false; + state.confirmed_interval = {}; + state.confirmed_rx_window = rx_window; + state.confirmed_ping_send_time = send_time; + state.confirmed_pong_receive_time = pong_time; + state.confirmed_window_open_local = {}; + state.confirmed_window_close_local = {}; + state.config_change_pending = (state.desired.interval != interval) || + (state.desired.rx_window != rx_window); + return; + } auto const schedule = MakeConfirmedSchedule(send_time, pong_time, interval, rx_window, selected_rtt); state.has_confirmed_schedule = true; @@ -224,6 +238,14 @@ void ClientConnectivityPolicy::ClearServerPresence(ServerId server_id) { server_presence_.erase(server_id); } +void ClientConnectivityPolicy::SetOfflineDetectionTimeout( + Duration timeout) noexcept { + if (timeout <= Duration{}) { + timeout = std::chrono::milliseconds{AE_OFFLINE_DETECTION_TIMEOUT_MS}; + } + offline_detection_timeout_ = timeout; +} + bool ClientConnectivityPolicy::IsLocallyOnline() const noexcept { return IsLocallyOnline(Now()); } @@ -234,8 +256,10 @@ bool ClientConnectivityPolicy::IsLocallyOnline(TimePoint now) const noexcept { if (!state.selected_for_aggregate) { continue; } - if (IsConfirmedWindowOnline(state.has_confirmed_schedule, now, - state.confirmed_window_close_local)) { + if (IsLocalPresenceOnline(state.has_confirmed_schedule, + state.confirmed_interval, + state.confirmed_window_open_local, now, + offline_detection_timeout_)) { return true; } } @@ -248,8 +272,10 @@ bool ClientConnectivityPolicy::IsServerLocallyOnline( if (state == nullptr) { return false; } - return IsConfirmedWindowOnline(state->has_confirmed_schedule, now, - state->confirmed_window_close_local); + return IsLocalPresenceOnline(state->has_confirmed_schedule, + state->confirmed_interval, + state->confirmed_window_open_local, now, + offline_detection_timeout_); } void ClientConnectivityPolicy::ResetRuntimeState() { diff --git a/aether/client_connectivity_policy.h b/aether/client_connectivity_policy.h index 5c6f3522..b82827f4 100644 --- a/aether/client_connectivity_policy.h +++ b/aether/client_connectivity_policy.h @@ -184,8 +184,15 @@ class ClientConnectivityPolicy : public Obj { void ClearServerPresence(ServerId server_id); - // Read-only. No side effects. Aggregate: ONLINE iff any selected server - // still in Personal Cloud has a confirmed schedule that has not expired. + // Application-level Local/Remote Presence classification timeout. + // Not part of Ping / rx_window. Applies immediately (no new Ping required). + void SetOfflineDetectionTimeout(Duration timeout) noexcept; + Duration offline_detection_timeout() const noexcept { + return offline_detection_timeout_; + } + + // Read-only. No side effects. Aggregate OR: ONLINE iff any selected server + // has confirmed interval>0 and now <= expected_open + offline_detection_timeout. bool IsLocallyOnline() const noexcept; bool IsLocallyOnline(TimePoint now) const noexcept; bool IsServerLocallyOnline(ServerId server_id, TimePoint now) const noexcept; @@ -205,6 +212,8 @@ class ClientConnectivityPolicy : public Obj { bool can_suspend_{true}; std::uint8_t suspend_block_count_{}; + Duration offline_detection_timeout_{std::chrono::milliseconds{ + AE_OFFLINE_DETECTION_TIMEOUT_MS}}; Event suspend_allowed_event_; Event server_rx_timing_changed_event_; diff --git a/aether/cloud_connections/cloud_request.cpp b/aether/cloud_connections/cloud_request.cpp index 0a972cc7..bb0518b8 100644 --- a/aether/cloud_connections/cloud_request.cpp +++ b/aether/cloud_connections/cloud_request.cpp @@ -71,10 +71,57 @@ void CloudRequest::Failed() { result_event_.Emit(false); } + +void CloudRequest::SucceedAttempt(CloudServerConnection* sc) { + auto it = server_requests_.find(sc); + if (it == server_requests_.end()) { + return; + } + auto& sr = it->second; + if (sr.succeeded) { + return; + } + sr.state_subs.Reset(); + sr.timeout_sub.Reset(); + sr.succeeded = true; + EnqueueMakeRequest(); +} + +bool CloudRequest::FailAttempt(CloudServerConnection* sc) { + auto it = server_requests_.find(sc); + if (it == server_requests_.end()) { + return false; + } + auto& sr = it->second; + if (sr.succeeded) { + return false; + } + sr.state_subs.Reset(); + sr.timeout_sub.Reset(); + sr.retry_count++; + if (sr.retry_count >= max_retries_) { + AE_TELED_WARNING("Server {} retry budget exhausted on attempt failure", + sc->server_id()); + sr.exhausted = true; + EmitAttemptExhausted(sc); + } + EnqueueMakeRequest(); + return sr.exhausted; +} + CloudRequest::ResultEvent::Subscriber CloudRequest::result_event() { return EventSubscriber{result_event_}; } +CloudRequest::AttemptExhaustedEvent::Subscriber +CloudRequest::attempt_exhausted_event() { + return EventSubscriber{attempt_exhausted_event_}; +} + +void CloudRequest::EmitAttemptExhausted(CloudServerConnection* sc) { + attempt_exhausted_event_.Emit(sc); +} + void CloudRequest::PrefillServerRequests() { for (auto* sc : cloud_scs_->servers()) { server_requests_.emplace(sc, ServerRequest{}); @@ -92,7 +139,7 @@ void CloudRequest::MakeRequest() { sr = &new_it->second; } else { sr = &it->second; - if (sr->exhausted) { + if (sr->exhausted || sr->succeeded) { return; } } @@ -100,15 +147,17 @@ void CloudRequest::MakeRequest() { }, policy_); - // Check if all server requests are exhausted - bool all_exhausted = !server_requests_.empty(); + bool any_open = false; + bool any_succeeded = false; for (auto const& [sc, sr] : server_requests_) { - if (!sr.exhausted) { - all_exhausted = false; - break; + if (sr.succeeded) { + any_succeeded = true; + } else if (!sr.exhausted) { + any_open = true; } } - if (all_exhausted) { + if (!server_requests_.empty() && + CloudRequestShouldFailAll(any_open, any_succeeded)) { AE_TELED_ERROR("All server requests exhausted, failing"); Failed(); } @@ -181,9 +230,13 @@ void CloudRequest::OnChannelChanged(CloudServerConnection* sc) { return; } auto& sr = it->second; + if (sr.succeeded) { + return; + } if (sr.retry_count >= max_retries_) { AE_TELED_WARNING("Server {} retry budget exhausted", sc->server_id()); sr.exhausted = true; + EmitAttemptExhausted(sc); EnqueueMakeRequest(); return; } @@ -197,11 +250,15 @@ void CloudRequest::OnWriteFailed(CloudServerConnection* sc) { return; } auto& sr = it->second; + if (sr.succeeded) { + return; + } sr.retry_count++; if (sr.retry_count >= max_retries_) { AE_TELED_WARNING("Server {} retry budget exhausted on write failure", sc->server_id()); sr.exhausted = true; + EmitAttemptExhausted(sc); } EnqueueMakeRequest(); } @@ -212,10 +269,14 @@ void CloudRequest::OnServerRequestTimeout(CloudServerConnection* sc) { return; } auto& sr = it->second; + if (sr.succeeded) { + return; + } if (sr.retry_count >= max_retries_) { AE_TELED_WARNING("Server {} retry budget exhausted on timeout", sc->server_id()); sr.exhausted = true; + EmitAttemptExhausted(sc); EnqueueMakeRequest(); return; } diff --git a/aether/cloud_connections/cloud_request.h b/aether/cloud_connections/cloud_request.h index ceb6d6f5..78a15b52 100644 --- a/aether/cloud_connections/cloud_request.h +++ b/aether/cloud_connections/cloud_request.h @@ -35,12 +35,42 @@ namespace ae { * response. On success, listener must call CloudRequest::Succeeded(). On * failure, listener must call CloudRequest::Failed(). */ +// Testable per-server completion flags used by CloudRequest. +struct CloudRequestAttemptState { + bool exhausted{false}; + bool succeeded{false}; + std::size_t retry_count{0}; + + bool ShouldSkipMake() const noexcept { return exhausted || succeeded; } + + void MarkSucceeded() { succeeded = true; } + + // Returns true when this failure exhausted the retry budget. + bool MarkFailed(std::size_t max_retries) { + if (succeeded) { + return false; + } + ++retry_count; + if (retry_count >= max_retries) { + exhausted = true; + return true; + } + return false; + } +}; + +inline bool CloudRequestShouldFailAll(bool any_open, + bool any_succeeded) noexcept { + return !any_open && !any_succeeded; +} + class CloudRequest final : public Action { struct ServerRequest { MultiSubscription state_subs; TaskSubscription timeout_sub; std::size_t retry_count{0}; bool exhausted{false}; + bool succeeded{false}; }; public: @@ -49,6 +79,7 @@ class CloudRequest final : public Action { std::chrono::milliseconds{AE_CLOUD_REQUEST_TIMEOUT_MS}; using ResultEvent = Event; + using AttemptExhaustedEvent = Event; CloudRequest(AeContext const& ae_context, ApiCallWithListener&& api_call, CloudServerConnections& cloud_server_connections, @@ -66,8 +97,15 @@ class CloudRequest final : public Action { void Succeeded(); void Failed(); + // Per-server success: stop this server's timeout/channel/write + // subscriptions, skip further retries, and do not finish CloudRequest. + void SucceedAttempt(CloudServerConnection* sc); + // Listener-side attempt failure: retry/exhaust this server without + // ending the whole CloudRequest. Returns true when the server is exhausted. + bool FailAttempt(CloudServerConnection* sc); ResultEvent::Subscriber result_event(); + AttemptExhaustedEvent::Subscriber attempt_exhausted_event(); private: void MakeRequest(); @@ -81,6 +119,7 @@ class CloudRequest final : public Action { void RemoveRequest(CloudServerConnection* server_connection); void EnqueueMakeRequest(); + void EmitAttemptExhausted(CloudServerConnection* sc); void Finish(); @@ -95,6 +134,7 @@ class CloudRequest final : public Action { Subscription swa_sub_; Subscription server_changed_sub_; ResultEvent result_event_; + AttemptExhaustedEvent attempt_exhausted_event_; std::map server_requests_; }; diff --git a/aether/cloud_connections/local_presence_machine.cpp b/aether/cloud_connections/local_presence_machine.cpp index 8ddee26f..0af176da 100644 --- a/aether/cloud_connections/local_presence_machine.cpp +++ b/aether/cloud_connections/local_presence_machine.cpp @@ -69,6 +69,13 @@ void LocalPresenceMachine::SetDesired(TimePoint now, RxTimingConf conf, } } +void LocalPresenceMachine::SetOfflineDetectionTimeout(Duration timeout) noexcept { + if (timeout <= Duration{}) { + timeout = std::chrono::milliseconds{AE_OFFLINE_DETECTION_TIMEOUT_MS}; + } + offline_detection_timeout_ = timeout; +} + void LocalPresenceMachine::ArmInitial(TimePoint now) { if (removed_ || quarantined_) { return; @@ -210,6 +217,27 @@ LocalPresenceMachine::PongOutcome LocalPresenceMachine::OnPong( } auto const was_online = IsOnline(pong_time); + if (sent_desired_interval <= Duration{}) { + has_confirmed_ = false; + confirmed_open_ = {}; + confirmed_close_ = {}; + confirmed_interval_ = {}; + confirmed_window_ = sent_window; + config_pending_ = desired_.interval > Duration{}; + cycle_confirmed_ = true; + active_cycle_id_ = cycle_id; + last_following_target_ = following_open_target; + ++counters_.confirmed_pongs; + out.disposition = PongDisposition::kConfirmedSchedule; + current_window_blocker_held_ = false; + if (desired_.interval > Duration{}) { + ArmSend(PingAttemptKind::kInitial, pong_time); + } else { + send_armed_ = false; + } + return out; + } + has_confirmed_ = true; confirmed_open_ = out.schedule.window_open_local; confirmed_close_ = out.schedule.window_close_local; @@ -291,7 +319,8 @@ bool LocalPresenceMachine::IsOnline(TimePoint now) const noexcept { if (removed_) { return false; } - return IsConfirmedWindowOnline(has_confirmed_, now, confirmed_close_); + return IsLocalPresenceOnline(has_confirmed_, confirmed_interval_, + confirmed_open_, now, offline_detection_timeout_); } LocalPresenceMachine::Attempt* LocalPresenceMachine::FindAttempt( diff --git a/aether/cloud_connections/local_presence_machine.h b/aether/cloud_connections/local_presence_machine.h index c48d5fd4..5b454ae3 100644 --- a/aether/cloud_connections/local_presence_machine.h +++ b/aether/cloud_connections/local_presence_machine.h @@ -104,8 +104,12 @@ class LocalPresenceMachine { LocalPresenceMachine(); void SetDesired(TimePoint now, RxTimingConf conf, std::uint8_t percentile); + void SetOfflineDetectionTimeout(Duration timeout) noexcept; RxTimingConf const& desired() const noexcept { return desired_; } std::uint8_t percentile() const noexcept { return percentile_; } + Duration offline_detection_timeout() const noexcept { + return offline_detection_timeout_; + } void RestoreConfirmed(TimePoint open, TimePoint close, Duration interval, Duration window, TimePoint now, @@ -182,6 +186,8 @@ class LocalPresenceMachine { RxTimingConf desired_{RxTimingConf::Every( std::chrono::milliseconds{AE_PING_INTERVAL_MS})}; std::uint8_t percentile_{kDefaultRttReliabilityPercentile}; + Duration offline_detection_timeout_{std::chrono::milliseconds{ + AE_OFFLINE_DETECTION_TIMEOUT_MS}}; bool has_confirmed_{false}; TimePoint confirmed_open_{}; diff --git a/aether/cloud_connections/local_presence_schedule.h b/aether/cloud_connections/local_presence_schedule.h index cb58d9f7..ff35ef96 100644 --- a/aether/cloud_connections/local_presence_schedule.h +++ b/aether/cloud_connections/local_presence_schedule.h @@ -130,6 +130,28 @@ inline TimePoint ComputePrefix2Time( return window_open - rtt / 2 - guard; } +// Local Presence ONLINE when a confirmed future opening exists and now is +// still within expected_open + offline_detection_timeout. +// rx_window / confirmed_window_close are NOT used for Presence. +inline bool IsLocalPresenceOnline(bool has_confirmed, Duration confirmed_interval, + TimePoint expected_open, TimePoint now, + Duration offline_detection_timeout) noexcept { + if (!has_confirmed) { + return false; + } + if (confirmed_interval <= Duration{}) { + return false; + } + return now <= (expected_open + offline_detection_timeout); +} + +inline TimePoint LocalOfflineDeadline( + TimePoint expected_open, Duration offline_detection_timeout) noexcept { + return expected_open + offline_detection_timeout; +} + +// Deprecated name kept for transitional call sites that still pass close. +// Prefer IsLocalPresenceOnline. inline bool IsConfirmedWindowOnline(bool has_confirmed, TimePoint now, TimePoint window_close) noexcept { if (!has_confirmed) { diff --git a/aether/cloud_connections/ping_cloud_servers.cpp b/aether/cloud_connections/ping_cloud_servers.cpp index 3bb37354..884e8e31 100644 --- a/aether/cloud_connections/ping_cloud_servers.cpp +++ b/aether/cloud_connections/ping_cloud_servers.cpp @@ -49,6 +49,7 @@ PingCloudServers::ServerPing::ServerPing(AeContext const& ae_context, policy_->SetServerSelectedForAggregate(server_id_, true); machine_.SetDesired(Now(), presence.desired, presence.rtt_reliability_percentile); + machine_.SetOfflineDetectionTimeout(policy_->offline_detection_timeout()); if (presence.has_confirmed_schedule) { machine_.RestoreConfirmed( presence.confirmed_window_open_local, @@ -98,6 +99,7 @@ void PingCloudServers::ServerPing::NotifyConfigChanged() { } machine_.SetDesired(Now(), presence->desired, presence->rtt_reliability_percentile); + machine_.SetOfflineDetectionTimeout(policy_->offline_detection_timeout()); Pump(); } @@ -387,6 +389,13 @@ void PingCloudServers::ServerPing::ApplyConfirmed( LocalPresenceMachine::PongDisposition::kConfirmedSchedule) { return; } + if (!machine_.has_confirmed_schedule()) { + policy_->ConfirmServerPong(server_id_, outcome.schedule.ping_send_time, + outcome.schedule.pong_receive_time, Duration{}, + outcome.schedule.rx_window, + outcome.schedule.selected_rtt); + return; + } policy_->ConfirmServerPong( server_id_, outcome.schedule.ping_send_time, outcome.schedule.pong_receive_time, outcome.schedule.interval, diff --git a/aether/config.h b/aether/config.h index f9ef0d3c..30e1cf1f 100644 --- a/aether/config.h +++ b/aether/config.h @@ -295,6 +295,13 @@ # define AE_PING_INTERVAL_MS AE_DEFAULT_RESPONSE_TIMEOUT_MS + 1000 #endif +// Initial default for Local/Remote Presence offline classification timeout. +// Runtime value lives on ClientConnectivityPolicy and may change without +// a new Ping. +#ifndef AE_OFFLINE_DETECTION_TIMEOUT_MS +# define AE_OFFLINE_DETECTION_TIMEOUT_MS 1000 +#endif + // window size for safe stream response time statistics #ifndef AE_STATISTICS_SAFE_STREAM_WINDOW_SIZE # define AE_STATISTICS_SAFE_STREAM_WINDOW_SIZE 100 diff --git a/aether/remote_presence.h b/aether/remote_presence.h new file mode 100644 index 00000000..53fbbf4f --- /dev/null +++ b/aether/remote_presence.h @@ -0,0 +1,208 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_REMOTE_PRESENCE_H_ +#define AETHER_REMOTE_PRESENCE_H_ + +#include +#include +#include +#include +#include + +#include "aether/clock.h" +#include "aether/config.h" +#include "aether/types/server_id.h" +#include "aether/work_cloud_api/client_timing.h" + +namespace ae { + +enum class PeerPresenceState : std::uint8_t { + kOnline = 0, + kOffline, + kUnknown, +}; + +struct PeerPresence { + PeerPresenceState state{PeerPresenceState::kUnknown}; +}; + +// Per authoritative usable server contribution for Remote AND aggregation. +enum class RemoteServerPresence : std::uint8_t { + kOnline = 0, + kOffline, + // Query pending / failed after retries while server remains usable. + kUnknown, + // Quarantined / unselected / removed — excluded from aggregation. + kExcluded, +}; + +struct RemoteServerPresenceSample { + ServerId server_id{}; + RemoteServerPresence status{RemoteServerPresence::kUnknown}; + TimePoint expected_open{}; + TimePoint offline_deadline{}; + std::int64_t next_ping_delta_ms{}; + bool has_timing{false}; +}; + +inline constexpr std::size_t kRemotePresenceQueryRetryCount{1}; + +inline TimePoint TimePointOffsetByMs(TimePoint anchor, + std::int64_t delta_ms) noexcept { + if (delta_ms == 0) { + return anchor; + } + using ClockDuration = typename TimePoint::duration; + using Rep = typename ClockDuration::rep; + auto const max_safe_ms = + std::chrono::duration_cast( + ClockDuration{std::numeric_limits::max() / 4}) + .count(); + if (max_safe_ms > 0) { + if (delta_ms > max_safe_ms) { + return TimePoint::max(); + } + if (delta_ms < -max_safe_ms) { + return TimePoint::min(); + } + } + auto const offset = std::chrono::duration_cast( + std::chrono::milliseconds{delta_ms}); + auto const base = anchor.time_since_epoch().count(); + auto const add = offset.count(); + if (add > 0) { + if (base > std::numeric_limits::max() - add) { + return TimePoint::max(); + } + } else if (add < 0) { + if (base < std::numeric_limits::min() - add) { + return TimePoint::min(); + } + } + return TimePoint{ClockDuration{static_cast(base + add)}}; +} + +// midpoint = query_send + (response_receive - query_send) / 2 +inline TimePoint QueryMidpoint(TimePoint query_send, + TimePoint response_receive) noexcept { + if (response_receive <= query_send) { + return query_send; + } + return query_send + (response_receive - query_send) / 2; +} + +inline TimePoint ProjectRemoteExpectedOpen( + TimePoint query_send, TimePoint response_receive, + std::int64_t next_ping_delta_ms) noexcept { + return TimePointOffsetByMs(QueryMidpoint(query_send, response_receive), + next_ping_delta_ms); +} + +// Classify one authoritative server response. next_ping_delta == 0 means no +// future promise => Offline immediately (do not wait offline_detection_timeout). +inline RemoteServerPresence ClassifyRemoteServerPresence( + TimePoint now, TimePoint query_send, TimePoint response_receive, + ClientTiming const& timing, Duration offline_detection_timeout, + TimePoint* expected_open_out = nullptr, + TimePoint* offline_deadline_out = nullptr) noexcept { + if (timing.next_ping_delta_ms == 0) { + if (expected_open_out != nullptr) { + *expected_open_out = {}; + } + if (offline_deadline_out != nullptr) { + *offline_deadline_out = {}; + } + return RemoteServerPresence::kOffline; + } + auto const expected = ProjectRemoteExpectedOpen( + query_send, response_receive, timing.next_ping_delta_ms); + auto const deadline = expected + offline_detection_timeout; + if (expected_open_out != nullptr) { + *expected_open_out = expected; + } + if (offline_deadline_out != nullptr) { + *offline_deadline_out = deadline; + } + if (now <= deadline) { + return RemoteServerPresence::kOnline; + } + return RemoteServerPresence::kOffline; +} + +// Remote aggregation AND over usable authoritative servers. +// Offline: any usable Offline. +// Online: usable_count > 0 and every usable sample Online. +// Unknown: usable_count == 0, or no Offline but not every usable Online. +inline PeerPresence AggregateRemotePresence( + std::vector const& samples) noexcept { + PeerPresence out{}; + std::size_t usable = 0; + std::size_t online = 0; + bool any_offline = false; + bool any_unknown = false; + for (auto const& sample : samples) { + if (sample.status == RemoteServerPresence::kExcluded) { + continue; + } + ++usable; + if (sample.status == RemoteServerPresence::kOffline) { + any_offline = true; + } else if (sample.status == RemoteServerPresence::kOnline) { + ++online; + } else { + any_unknown = true; + } + } + if (usable == 0) { + out.state = PeerPresenceState::kUnknown; + return out; + } + if (any_offline) { + out.state = PeerPresenceState::kOffline; + return out; + } + if (online == usable && !any_unknown) { + out.state = PeerPresenceState::kOnline; + return out; + } + out.state = PeerPresenceState::kUnknown; + return out; +} + +inline bool RemotePresenceCanEarlyCompleteOffline( + std::vector const& samples) noexcept { + for (auto const& sample : samples) { + if (sample.status == RemoteServerPresence::kOffline) { + return true; + } + } + return false; +} + +inline bool RemotePresenceReadyForOnline( + std::vector const& samples) noexcept { + auto const aggregated = AggregateRemotePresence(samples); + return aggregated.state == PeerPresenceState::kOnline; +} + +inline Duration DefaultOfflineDetectionTimeout() noexcept { + return std::chrono::milliseconds{AE_OFFLINE_DETECTION_TIMEOUT_MS}; +} + +} // namespace ae + +#endif // AETHER_REMOTE_PRESENCE_H_ diff --git a/aether/work_cloud_api/client_timing.h b/aether/work_cloud_api/client_timing.h new file mode 100644 index 00000000..46300145 --- /dev/null +++ b/aether/work_cloud_api/client_timing.h @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_WORK_CLOUD_API_CLIENT_TIMING_H_ +#define AETHER_WORK_CLOUD_API_CLIENT_TIMING_H_ + +#include + +#include "aether-miscpp/reflect/reflect.h" + +namespace ae { + +// Wire DTO for AuthorizedApi.get_client_timing. Field order matches ADSL: +// nextPingDeltaMs then lastConnectDeltaMs. Do not change wire layout here. +struct ClientTiming { + AE_REFLECT_MEMBERS(next_ping_delta_ms, last_connect_delta_ms) + + std::int64_t next_ping_delta_ms{}; + std::int64_t last_connect_delta_ms{}; +}; + +} // namespace ae + +#endif // AETHER_WORK_CLOUD_API_CLIENT_TIMING_H_ diff --git a/aether/work_cloud_api/work_server_api/authorized_api.cpp b/aether/work_cloud_api/work_server_api/authorized_api.cpp index 22542c94..f267b426 100644 --- a/aether/work_cloud_api/work_server_api/authorized_api.cpp +++ b/aether/work_cloud_api/work_server_api/authorized_api.cpp @@ -26,5 +26,6 @@ AuthorizedApi::AuthorizedApi(ProtocolContext& protocol_context) resolver_servers{protocol_context}, resolver_clouds{protocol_context}, send_telemetry{protocol_context}, + get_client_timing{protocol_context}, report_applied_config{protocol_context} {} } // namespace ae diff --git a/aether/work_cloud_api/work_server_api/authorized_api.h b/aether/work_cloud_api/work_server_api/authorized_api.h index d72c4d66..daba96f9 100644 --- a/aether/work_cloud_api/work_server_api/authorized_api.h +++ b/aether/work_cloud_api/work_server_api/authorized_api.h @@ -26,6 +26,7 @@ #include "aether/work_cloud_api/ae_message.h" #include "aether/work_cloud_api/telemetric.h" #include "aether/work_cloud_api/cloud_configs.h" +#include "aether/work_cloud_api/client_timing.h" namespace ae { @@ -44,6 +45,9 @@ class AuthorizedApi : public ApiClass { Method<18, void(Telemetric telemetric)> send_telemetry; + // Existing server Method 35. Client binding only — wire DTO unchanged. + Method<35, ApiPromise(Uid uid)> get_client_timing; + Method<38, void(std::vector configs)> report_applied_config; }; } // namespace ae diff --git a/tests/test-local-presence/firewall_live.cpp b/tests/test-local-presence/firewall_live.cpp index 4d17066f..f786d40f 100644 --- a/tests/test-local-presence/firewall_live.cpp +++ b/tests/test-local-presence/firewall_live.cpp @@ -28,6 +28,7 @@ #include "aether/client.h" #include "aether/client_connectivity_policy.h" #include "aether/cloud_connections/cloud_server_connection.h" +#include "aether/cloud_connections/local_presence_schedule.h" #include "aether/global_ids.h" #include "aether/types/uid.h" @@ -173,7 +174,8 @@ TimePoint ServerConfirmedClose(ClientConnectivityPolicy& policy, ServerId id) { if (state == nullptr || !state->has_confirmed_schedule) { return TimePoint::max(); } - return state->confirmed_window_close_local; + return LocalOfflineDeadline(state->confirmed_window_open_local, + policy.offline_detection_timeout()); } #endif @@ -181,6 +183,7 @@ void ApplyOneSecondTimings(Client& client) { auto policy = client.connectivity_policy(); TEST_ASSERT_TRUE(static_cast(policy)); policy->ResetRxTimings(); + policy->SetOfflineDetectionTimeout(1s); policy->ConfigureRxTimings(RequestPolicy::All{}) .ForAllPriorities(RxTimingConf::Every(kInterval).WithWindow(kWindow)); for (auto* server : client.cloud_connection().selected_servers()) { diff --git a/tests/test-local-presence/main.cpp b/tests/test-local-presence/main.cpp index e0e0b0e3..c3f67dde 100644 --- a/tests/test-local-presence/main.cpp +++ b/tests/test-local-presence/main.cpp @@ -27,7 +27,9 @@ #include "aether/client_connectivity_policy.h" #include "aether/cloud_connections/local_presence_machine.h" #include "aether/cloud_connections/local_presence_schedule.h" +#include "aether/remote_presence.h" #include "aether/types/statistic_counter.h" +#include "aether/work_cloud_api/client_timing.h" namespace ae::test_local_presence { @@ -76,9 +78,13 @@ void test_ConfirmOnlyAfterPong() { TEST_ASSERT_TRUE(state->has_confirmed_schedule); TEST_ASSERT_EQUAL(2050, ToMs(state->confirmed_window_open_local)); TEST_ASSERT_EQUAL(2350, ToMs(state->confirmed_window_close_local)); + auto const deadline = + LocalOfflineDeadline(state->confirmed_window_open_local, + policy.offline_detection_timeout()); + TEST_ASSERT_EQUAL(3050, ToMs(deadline)); TEST_ASSERT_TRUE(policy.IsServerLocallyOnline(sid, Tp(2050))); - TEST_ASSERT_TRUE(policy.IsServerLocallyOnline(sid, Tp(2350))); - TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(sid, Tp(2351))); + TEST_ASSERT_TRUE(policy.IsServerLocallyOnline(sid, deadline)); + TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(sid, deadline + Dur(1))); } void test_SelectedRttProjectionIgnoresMeasuredPong() { @@ -108,12 +114,18 @@ void test_PerServerIndependence() { TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(50))); } -void test_OfflineOnlyAfterWindowClose() { +void test_OfflineOnlyAfterOfflineDetectionTimeout() { ClientConnectivityPolicy policy; ServerId const sid{3}; - policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(200), Dur(40)); - TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(1220))); - TEST_ASSERT_FALSE(policy.IsLocallyOnline(Tp(1221))); + policy.SetOfflineDetectionTimeout(Dur(1000)); + policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(10000), Dur(40)); + // open = 0 + 20 + 1000 = 1020; deadline = 2020 even with rx_window=10s. + TEST_ASSERT_EQUAL(1020, ToMs(policy.FindServerPresence(sid) + ->confirmed_window_open_local)); + TEST_ASSERT_EQUAL( + 11020, ToMs(policy.FindServerPresence(sid)->confirmed_window_close_local)); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(Tp(2020))); + TEST_ASSERT_FALSE(policy.IsLocallyOnline(Tp(2021))); } void test_RuntimeIntervalChangeKeepsOldConfirmed() { @@ -271,6 +283,7 @@ class PresenceHarness { s.stats.Add(seed_rtt); } s.machine.SetDesired(now_, conf, percentile); + s.machine.SetOfflineDetectionTimeout(policy_.offline_detection_timeout()); s.machine.ArmInitial(now_); servers_.emplace(id, std::move(s)); } @@ -668,13 +681,15 @@ void test_QuarantineKeepsConfirmedUntilClose() { Dur(100)); rt.SetFixedDelay(sid, Dur(20)); rt.AdvanceTo(Tp(20)); - auto const close = rt.machine(sid).confirmed_window_close(); - TEST_ASSERT_TRUE(ToMs(close) >= 2000); + auto const open = rt.machine(sid).confirmed_window_open(); + auto const deadline = + LocalOfflineDeadline(open, rt.policy().offline_detection_timeout()); + TEST_ASSERT_TRUE(ToMs(deadline) >= 2000); rt.machine(sid).OnQuarantine(Tp(1000)); TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(sid, Tp(1500))); TEST_ASSERT_TRUE(rt.policy().IsLocallyOnline(Tp(1500))); - TEST_ASSERT_TRUE(rt.policy().IsLocallyOnline(close)); - TEST_ASSERT_FALSE(rt.policy().IsLocallyOnline(close + Dur(1))); + TEST_ASSERT_TRUE(rt.policy().IsLocallyOnline(deadline)); + TEST_ASSERT_FALSE(rt.policy().IsLocallyOnline(deadline + Dur(1))); } void test_HardRemovalDropsAggregateImmediately() { @@ -738,7 +753,10 @@ void test_MultiServerIndependentSchedules() { TEST_ASSERT_EQUAL(1000, ToMs(rt.machine(a).confirmed_interval())); TEST_ASSERT_EQUAL(3000, ToMs(rt.machine(b).confirmed_interval())); rt.SetConnectivity(a, false); - rt.AdvanceTo(rt.machine(a).confirmed_window_close() + Dur(1)); + auto const offline_deadline = LocalOfflineDeadline( + rt.machine(a).confirmed_window_open(), + rt.policy().offline_detection_timeout()); + rt.AdvanceTo(offline_deadline + Dur(1)); TEST_ASSERT_FALSE(rt.policy().IsServerLocallyOnline(a, rt.now())); TEST_ASSERT_TRUE(rt.policy().IsServerLocallyOnline(b, rt.now())); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); @@ -780,15 +798,25 @@ void test_StatisticalRuntimePollingIsLocallyOnline() { rt.AdvanceTo(Tp(20)); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); auto const measure_start = rt.now(); + bool window_bumped = false; while (true) { rt.AdvancePolling(Dur(10), Dur(10), true); auto const elapsed_ms = std::chrono::duration_cast(rt.now() - measure_start).count(); + if (!window_bumped && elapsed_ms >= 60000) { + // Presence must ignore rx_window change without new confirming Pong. + rt.policy().ConfigureServerRxTiming( + sid, RxTimingConf::Every(interval).WithWindow(Dur(10000))); + rt.machine(sid).SetDesired( + rt.now(), RxTimingConf::Every(interval).WithWindow(Dur(10000)), 99); + window_bumped = true; + } if (elapsed_ms >= 300000) { break; } TEST_ASSERT_TRUE(elapsed_ms < 400000); } + TEST_ASSERT_TRUE(window_bumped); g_stat_report.confirmed_cycles = rt.counters(sid).confirmed_pongs; g_stat_report.duration = @@ -829,18 +857,20 @@ void test_FaultOfflineNotBeforeWindowCloseThenRecovery() { rt.AdvanceTo(Tp(20)); rt.AdvancePolling(Dur(2000), Dur(10), true); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); - auto const close = rt.machine(sid).confirmed_window_close(); + auto const deadline = LocalOfflineDeadline( + rt.machine(sid).confirmed_window_open(), + rt.policy().offline_detection_timeout()); rt.SetConnectivity(sid, false); auto detected = TimePoint{}; - while (rt.now() < close + Dur(2000)) { + while (rt.now() < deadline + Dur(2000)) { rt.AdvancePolling(Dur(10), Dur(10), false); if (!rt.IsLocallyOnline()) { detected = rt.now(); break; } } - TEST_ASSERT_TRUE(detected > close); + TEST_ASSERT_TRUE(detected > deadline); rt.SetConnectivity(sid, true); auto const recover_from = rt.now(); while (rt.now() < recover_from + Dur(2000)) { @@ -852,6 +882,154 @@ void test_FaultOfflineNotBeforeWindowCloseThenRecovery() { TEST_ASSERT_TRUE(rt.IsLocallyOnline()); } +void test_RxWindowDoesNotAffectLocalPresenceDeadline() { + ClientConnectivityPolicy policy; + ServerId const sid{21}; + policy.SetOfflineDetectionTimeout(Dur(1000)); + policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(100), Dur(40)); + auto const open = policy.FindServerPresence(sid)->confirmed_window_open_local; + auto const deadline_before = + LocalOfflineDeadline(open, policy.offline_detection_timeout()); + policy.ConfigureServerRxTiming( + sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(10000))); + // Confirmed open unchanged; Presence deadline unchanged by rx_window. + TEST_ASSERT_EQUAL(ToMs(open), ToMs(policy.FindServerPresence(sid) + ->confirmed_window_open_local)); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(deadline_before)); + TEST_ASSERT_FALSE(policy.IsLocallyOnline(deadline_before + Dur(1))); +} + +void test_OfflineDetectionTimeoutRuntimeChangeAppliesImmediately() { + ClientConnectivityPolicy policy; + ServerId const sid{22}; + policy.SetOfflineDetectionTimeout(Dur(1000)); + policy.ConfirmServerPong(sid, Tp(0), Tp(40), Dur(1000), Dur(1000), Dur(40)); + auto const open = policy.FindServerPresence(sid)->confirmed_window_open_local; + TEST_ASSERT_FALSE(policy.IsLocallyOnline(open + Dur(1001))); + policy.SetOfflineDetectionTimeout(Dur(2000)); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(open + Dur(1001))); + TEST_ASSERT_TRUE(policy.IsLocallyOnline(open + Dur(2000))); + TEST_ASSERT_FALSE(policy.IsLocallyOnline(open + Dur(2001))); +} + +void test_RetriesContinueAfterLocalOffline() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + rt.SetConnectivity(sid, false); + auto const deadline = LocalOfflineDeadline( + rt.machine(sid).confirmed_window_open(), + rt.policy().offline_detection_timeout()); + rt.AdvanceTo(deadline + Dur(1)); + TEST_ASSERT_FALSE(rt.IsLocallyOnline()); + auto const retries_before = rt.counters(sid).retry + rt.counters(sid).prefix2 + + rt.counters(sid).recovery; + rt.AdvanceTo(deadline + Dur(500)); + auto const retries_after = rt.counters(sid).retry + rt.counters(sid).prefix2 + + rt.counters(sid).recovery; + TEST_ASSERT_TRUE(retries_after > retries_before); + rt.SetConnectivity(sid, true); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(rt.now() + Dur(300)); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_IntervalZeroWithoutPongKeepsConfirmed() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + rt.AddServer(sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), + Dur(100)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + TEST_ASSERT_TRUE(rt.machine(sid).has_confirmed_schedule()); + rt.machine(sid).SetDesired(rt.now(), + RxTimingConf::Every(Dur(0)).WithWindow(Dur(1000)), + 99); + TEST_ASSERT_TRUE(rt.machine(sid).has_confirmed_schedule()); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + +void test_IntervalZeroWithPongClearsFuturePresence() { + LocalPresenceMachine machine; + machine.SetDesired(Tp(0), RxTimingConf::Every(Dur(0)).WithWindow(Dur(1000)), + 99); + machine.ArmInitial(Tp(0)); + auto tick = machine.TickNow(Tp(0), Dur(100)); + TEST_ASSERT_TRUE(tick.want_send); + machine.OnSendStarting(); + machine.OnAttemptSent(tick.send, Tp(0)); + auto outcome = machine.OnPong(tick.send.attempt_id, tick.send.cycle_id, Tp(0), + Tp(20), Dur(0), Dur(0), Dur(1000), + tick.send.following_open_target, Dur(100)); + TEST_ASSERT_EQUAL( + static_cast(LocalPresenceMachine::PongDisposition::kConfirmedSchedule), + static_cast(outcome.disposition)); + TEST_ASSERT_FALSE(machine.has_confirmed_schedule()); + TEST_ASSERT_FALSE(machine.IsOnline(Tp(20))); +} + +void test_RemoteTimingProjectionAndAggregation() { + ClientTiming timing{}; + timing.next_ping_delta_ms = 500; + timing.last_connect_delta_ms = 0; + TimePoint expected{}; + TimePoint deadline{}; + auto status = ClassifyRemoteServerPresence( + Tp(2550), Tp(1000), Tp(1100), timing, Dur(1000), &expected, &deadline); + TEST_ASSERT_EQUAL(1550, ToMs(expected)); + TEST_ASSERT_EQUAL(2550, ToMs(deadline)); + TEST_ASSERT_EQUAL(static_cast(RemoteServerPresence::kOnline), + static_cast(status)); + status = ClassifyRemoteServerPresence(Tp(2551), Tp(1000), Tp(1100), timing, + Dur(1000), &expected, &deadline); + TEST_ASSERT_EQUAL(static_cast(RemoteServerPresence::kOffline), + static_cast(status)); + + timing.next_ping_delta_ms = -200; + status = ClassifyRemoteServerPresence(Tp(1850), Tp(1000), Tp(1100), timing, + Dur(1000), &expected, &deadline); + TEST_ASSERT_EQUAL(850, ToMs(expected)); + TEST_ASSERT_EQUAL(1850, ToMs(deadline)); + TEST_ASSERT_EQUAL(static_cast(RemoteServerPresence::kOnline), + static_cast(status)); + status = ClassifyRemoteServerPresence(Tp(1851), Tp(1000), Tp(1100), timing, + Dur(1000), &expected, &deadline); + TEST_ASSERT_EQUAL(static_cast(RemoteServerPresence::kOffline), + static_cast(status)); + + timing.next_ping_delta_ms = 0; + status = ClassifyRemoteServerPresence(Tp(0), Tp(1000), Tp(1100), timing, + Dur(1000)); + TEST_ASSERT_EQUAL(static_cast(RemoteServerPresence::kOffline), + static_cast(status)); + + std::vector samples(3); + samples[0] = {ServerId{1}, RemoteServerPresence::kOnline, {}, {}, 0, true}; + samples[1] = {ServerId{2}, RemoteServerPresence::kOnline, {}, {}, 0, true}; + samples[2] = {ServerId{3}, RemoteServerPresence::kOnline, {}, {}, 0, true}; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kOnline), + static_cast(AggregateRemotePresence(samples).state)); + samples[1].status = RemoteServerPresence::kOffline; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kOffline), + static_cast(AggregateRemotePresence(samples).state)); + samples[1].status = RemoteServerPresence::kUnknown; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kUnknown), + static_cast(AggregateRemotePresence(samples).state)); + samples[1].status = RemoteServerPresence::kExcluded; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kOnline), + static_cast(AggregateRemotePresence(samples).state)); + samples[0].status = RemoteServerPresence::kExcluded; + samples[1].status = RemoteServerPresence::kExcluded; + samples[2].status = RemoteServerPresence::kExcluded; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kUnknown), + static_cast(AggregateRemotePresence(samples).state)); + TEST_ASSERT_TRUE(RemotePresenceCanEarlyCompleteOffline( + {{ServerId{1}, RemoteServerPresence::kOffline, {}, {}, 0, true}})); +} + } // namespace ae::test_local_presence void setUp() {} @@ -863,7 +1041,7 @@ int main() { RUN_TEST(ae::test_local_presence::test_ConfirmOnlyAfterPong); RUN_TEST(ae::test_local_presence::test_SelectedRttProjectionIgnoresMeasuredPong); RUN_TEST(ae::test_local_presence::test_PerServerIndependence); - RUN_TEST(ae::test_local_presence::test_OfflineOnlyAfterWindowClose); + RUN_TEST(ae::test_local_presence::test_OfflineOnlyAfterOfflineDetectionTimeout); RUN_TEST(ae::test_local_presence::test_RuntimeIntervalChangeKeepsOldConfirmed); RUN_TEST(ae::test_local_presence::test_RuntimePercentile); RUN_TEST(ae::test_local_presence::test_ReliabilityP95VsP99PrefixTimes); @@ -883,6 +1061,12 @@ int main() { RUN_TEST(ae::test_local_presence::test_RuntimeConfigChangeKeepsOldUntilPong); RUN_TEST(ae::test_local_presence::test_Prefix1SuccessNoPrefix2); RUN_TEST(ae::test_local_presence::test_MultiServerIndependentSchedules); + RUN_TEST(ae::test_local_presence::test_RxWindowDoesNotAffectLocalPresenceDeadline); + RUN_TEST(ae::test_local_presence::test_OfflineDetectionTimeoutRuntimeChangeAppliesImmediately); + RUN_TEST(ae::test_local_presence::test_RetriesContinueAfterLocalOffline); + RUN_TEST(ae::test_local_presence::test_IntervalZeroWithoutPongKeepsConfirmed); + RUN_TEST(ae::test_local_presence::test_IntervalZeroWithPongClearsFuturePresence); + RUN_TEST(ae::test_local_presence::test_RemoteTimingProjectionAndAggregation); RUN_TEST(ae::test_local_presence::test_StatisticalRuntimePollingIsLocallyOnline); RUN_TEST(ae::test_local_presence::test_FaultOfflineNotBeforeWindowCloseThenRecovery); return UNITY_END(); From efa916e9f04528d7d2b1649b62210766812700c2 Mon Sep 17 00:00:00 2001 From: aethernet-io Date: Wed, 2 Sep 2026 19:14:47 -0700 Subject: [PATCH 05/11] Remove own-cloud fallback from Remote Presence and align authoritative set. QueryPeerPresence now returns UNKNOWN when the peer Personal Cloud is unavailable, aggregates only over selected_servers() (same Local Presence contract), and re-queries recovered servers fresh. Adds unit coverage and a live A/B harness. Co-authored-by: Cursor --- CMakeLists.txt | 1 + aether/ae_actions/query_peer_presence.cpp | 227 +++--- aether/ae_actions/query_peer_presence.h | 24 +- aether/remote_presence.h | 26 + examples/remote_presence_live/CMakeLists.txt | 31 + examples/remote_presence_live/main.cpp | 19 + .../remote_presence_live.cpp | 655 ++++++++++++++++++ tests/test-local-presence/main.cpp | 106 +++ 8 files changed, 1005 insertions(+), 84 deletions(-) create mode 100644 examples/remote_presence_live/CMakeLists.txt create mode 100644 examples/remote_presence_live/main.cpp create mode 100644 examples/remote_presence_live/remote_presence_live.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d2a08193..5acebc39 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -384,6 +384,7 @@ if(AE_BUILD_EXAMPLES) add_subdirectory(examples/common) add_subdirectory(examples/cloud) add_subdirectory(examples/a_b_message_exchange) + add_subdirectory(examples/remote_presence_live) add_subdirectory(examples/message_server) add_subdirectory(examples/capi/oddity) add_subdirectory(examples/benches/send_message_delays) diff --git a/aether/ae_actions/query_peer_presence.cpp b/aether/ae_actions/query_peer_presence.cpp index 37a59308..f5c20e4b 100644 --- a/aether/ae_actions/query_peer_presence.cpp +++ b/aether/ae_actions/query_peer_presence.cpp @@ -16,6 +16,10 @@ #include "aether/ae_actions/query_peer_presence.h" +#include +#include + +#include "aether/api_protocol/sub_api.h" #include "aether/client.h" #include "aether/cloud_connections/cloud_server_connection.h" #include "aether/config.h" @@ -29,16 +33,11 @@ namespace ae { QueryPeerPresence::QueryPeerPresence(AeContext const& ae_context, Client& client, Uid peer_uid) : ae_context_{ae_context}, client_{&client}, peer_uid_{peer_uid} { - // Prefer peer Personal Cloud (cached or GetCloud). Fall back to the - // observer's linked cloud only when peer cloud is unavailable — each - // get_client_timing(uid) answer is still authoritative for that server. + static_cast(AllowObserverCloudFallbackForPeerPresence()); + auto cached = client_->cloud_manager()->GetCachedCloud(peer_uid_); if (cached && cached.is_valid() && !cached->servers().empty()) { - dest_cloud_ = std::make_unique( - ae_context_, cached.Load(), - client_->server_connection_manager().GetServerConnectionFactory(), - AE_CLOUD_MAX_SERVER_CONNECTIONS); - work_cloud_ = dest_cloud_.get(); + BindPeerCloud(cached); StartQuery(); return; } @@ -66,47 +65,58 @@ Duration QueryPeerPresence::OfflineTimeout() const noexcept { return policy.Load()->offline_detection_timeout(); } -void QueryPeerPresence::OnCloud(Result result) { - if (finished_) { - return; +void QueryPeerPresence::BindPeerCloud(Cloud::ptr cloud) { + used_observer_cloud_ = false; + + // Full peer Personal Cloud (priority order). Authoritative Presence set is + // later taken from selected_servers() — same contract as Local Presence. + peer_cloud_server_ids_.clear(); + std::vector> ordered; + ordered.reserve(cloud->servers().size()); + for (auto const& [id, entry] : cloud->servers()) { + ordered.emplace_back(entry.priority, id); } - if (!result) { - // Peer cloud unavailable — fall back to observer cloud if it has usable - // servers. get_client_timing remains per-server authoritative. - auto& own = client_->cloud_connection(); - bool any_usable = false; - for (auto* sc : own.selected_servers()) { - if (sc != nullptr && sc->server() && !sc->quarantine()) { - any_usable = true; - break; - } - } - if (!any_usable) { - Complete(PeerPresence{PeerPresenceState::kUnknown}); - return; - } - work_cloud_ = &own; - StartQuery(); - return; + std::sort(ordered.begin(), ordered.end()); + for (auto const& item : ordered) { + peer_cloud_server_ids_.push_back(item.second); } - auto cloud = std::move(result).value(); + + // Same connection budget as Local Presence / peer PingCloudServers. dest_cloud_ = std::make_unique( ae_context_, cloud.Load(), client_->server_connection_manager().GetServerConnectionFactory(), AE_CLOUD_MAX_SERVER_CONNECTIONS); work_cloud_ = dest_cloud_.get(); +} + +void QueryPeerPresence::OnCloud(Result result) { + if (finished_) { + return; + } + if (!result) { + // Peer Personal Cloud unavailable — never fall back to observer cloud. + used_observer_cloud_ = false; + Complete(PeerPresence{PeerPresenceState::kUnknown}); + return; + } + BindPeerCloud(std::move(result).value()); StartQuery(); } void QueryPeerPresence::RefreshUsableSet() { samples_.clear(); + authoritative_server_ids_.clear(); if (work_cloud_ == nullptr) { return; } + + // Local Presence contract = selected_servers() under RequestPolicy::All + // (bounded by AE_CLOUD_MAX_SERVER_CONNECTIONS). Remote AND uses the same set. for (auto* sc : work_cloud_->selected_servers()) { if (sc == nullptr || !sc->server()) { continue; } + authoritative_server_ids_.push_back(sc->server_id()); RemoteServerPresenceSample sample{}; sample.server_id = sc->server_id(); if (sc->quarantine()) { @@ -116,6 +126,12 @@ void QueryPeerPresence::RefreshUsableSet() { } samples_.push_back(sample); } + + AE_TELED_DEBUG( + "REMOTE_PRESENCE peer_cloud_count={} authoritative_count={} " + "selected_count={} max_connections={}", + peer_cloud_server_ids_.size(), authoritative_server_ids_.size(), + work_cloud_->selected_servers().size(), work_cloud_->max_connections()); } void QueryPeerPresence::StartQuery() { @@ -134,37 +150,21 @@ void QueryPeerPresence::StartQuery() { return; } - quarantine_sub_ = - work_cloud_->server_quarantined_event().Subscribe( - [this](CloudServerConnection* sc) { - if (finished_ || sc == nullptr) { - return; - } - MarkExcluded(sc->server_id()); - MaybeComplete(); - }); + quarantine_sub_ = work_cloud_->server_quarantined_event().Subscribe( + [this](CloudServerConnection* sc) { + if (finished_ || sc == nullptr) { + return; + } + MarkExcluded(sc->server_id()); + MaybeComplete(); + }); quarantine_release_sub_ = work_cloud_->server_quarantine_release_event().Subscribe( [this](CloudServerConnection* sc) { if (finished_ || sc == nullptr) { return; } - // Recovered server re-enters usable set only after a new response. - RemoteServerPresenceSample sample{}; - sample.server_id = sc->server_id(); - sample.status = RemoteServerPresence::kUnknown; - bool found = false; - for (auto& existing : samples_) { - if (existing.server_id == sample.server_id) { - existing = sample; - found = true; - break; - } - } - if (!found) { - samples_.push_back(sample); - } - MaybeComplete(); + OnServerRecovered(sc); }); cloud_request_.emplace( @@ -172,6 +172,7 @@ void QueryPeerPresence::StartQuery() { ApiRequestHandler{[this](ApiContext& auth_api, CloudServerConnection* sc, CloudRequest* request) { + static_cast(request); if (finished_ || sc == nullptr || !sc->server() || sc->quarantine()) { return; } @@ -184,6 +185,10 @@ void QueryPeerPresence::StartQuery() { return; } } + if (std::find(queried_server_ids_.begin(), queried_server_ids_.end(), + server_id) == queried_server_ids_.end()) { + queried_server_ids_.push_back(server_id); + } auto& meta = attempts_[server_id]; ++meta.generation; meta.send_time = Now(); @@ -193,7 +198,6 @@ void QueryPeerPresence::StartQuery() { [this, sc, generation](auto const& res) { OnServerTiming(sc, generation, res); }); - static_cast(request); }}, *work_cloud_, RequestPolicy::All{}, /*max_retries=*/kRemotePresenceQueryRetryCount + 1); @@ -214,20 +218,73 @@ void QueryPeerPresence::StartQuery() { if (ok) { return; } - for (auto& sample : samples_) { - if (sample.status == RemoteServerPresence::kUnknown && - !sample.has_timing) { - // leave as Unknown contribution - } - } MaybeComplete(); - if (!finished_) { - // All servers exhausted without Offline/Online completion. + if (!finished_ && AllUsableTerminal()) { Complete(AggregateRemotePresence(samples_)); } }); } +void QueryPeerPresence::RequestTiming(CloudServerConnection* sc) { + if (finished_ || sc == nullptr || !sc->server() || sc->quarantine()) { + return; + } + auto* conn = sc->client_connection(); + if (conn == nullptr) { + return; + } + auto const server_id = sc->server_id(); + for (auto const& sample : samples_) { + if (sample.server_id == server_id && + (sample.status == RemoteServerPresence::kOnline || + sample.status == RemoteServerPresence::kOffline || + sample.status == RemoteServerPresence::kExcluded)) { + return; + } + } + + if (std::find(queried_server_ids_.begin(), queried_server_ids_.end(), + server_id) == queried_server_ids_.end()) { + queried_server_ids_.push_back(server_id); + } + + auto& meta = attempts_[server_id]; + ++meta.generation; + meta.send_time = Now(); + auto const generation = meta.generation; + + // AuthorizedApiCall requires an active ApiContext path via CloudRequest's + // handler; for recovered servers we re-enter through a one-server request. + conn->AuthorizedApiCall(SubApi{[&, sc, generation]( + ApiContext& auth_api) { + timing_subs_[server_id] = auth_api->get_client_timing(peer_uid_).Subscribe( + [this, sc, generation](auto const& res) { + OnServerTiming(sc, generation, res); + }); + }}); +} + +void QueryPeerPresence::OnServerRecovered(CloudServerConnection* sc) { + RemoteServerPresenceSample sample{}; + sample.server_id = sc->server_id(); + sample.status = RemoteServerPresence::kUnknown; + bool found = false; + for (auto& existing : samples_) { + if (existing.server_id == sample.server_id) { + existing = sample; + found = true; + break; + } + } + if (!found) { + samples_.push_back(sample); + authoritative_server_ids_.push_back(sample.server_id); + } + // Fresh timing required — do not keep a stale ONLINE. + RequestTiming(sc); + MaybeComplete(); +} + void QueryPeerPresence::OnServerTiming( CloudServerConnection* sc, std::uint64_t generation, Result const& res) { @@ -246,6 +303,9 @@ void QueryPeerPresence::OnServerTiming( MarkUnknown(server_id); MaybeComplete(); } + } else { + MarkUnknown(server_id); + MaybeComplete(); } return; } @@ -270,8 +330,9 @@ void QueryPeerPresence::OnServerTiming( } AE_TELED_DEBUG( - "REMOTE_PRESENCE server {} next_delta {} status {}", server_id, - res.value().next_ping_delta_ms, static_cast(status)); + "REMOTE_PRESENCE server {} next_delta {} status {} expected {} deadline {}", + server_id, res.value().next_ping_delta_ms, static_cast(status), + expected, deadline); if (cloud_request_.has_value()) { cloud_request_->SucceedAttempt(sc); @@ -286,6 +347,8 @@ void QueryPeerPresence::MarkUnknown(ServerId server_id) { sample.status != RemoteServerPresence::kOnline && sample.status != RemoteServerPresence::kOffline) { sample.status = RemoteServerPresence::kUnknown; + // Terminal unknown after retries — treat as observed for completion. + sample.has_timing = true; return; } } @@ -300,6 +363,19 @@ void QueryPeerPresence::MarkExcluded(ServerId server_id) { } } +bool QueryPeerPresence::AllUsableTerminal() const noexcept { + for (auto const& sample : samples_) { + if (sample.status == RemoteServerPresence::kExcluded) { + continue; + } + if (!sample.has_timing && + sample.status == RemoteServerPresence::kUnknown) { + return false; + } + } + return true; +} + void QueryPeerPresence::MaybeComplete() { if (finished_) { return; @@ -312,28 +388,19 @@ void QueryPeerPresence::MaybeComplete() { Complete(PeerPresence{PeerPresenceState::kOnline}); return; } - // All remaining usable samples known (Online/Unknown) and no Offline — - // wait until every usable server has a terminal observation. - bool any_pending = false; std::size_t usable = 0; for (auto const& sample : samples_) { - if (sample.status == RemoteServerPresence::kExcluded) { - continue; - } - ++usable; - if (sample.status == RemoteServerPresence::kUnknown && !sample.has_timing) { - // Still waiting unless retries exhausted left it Unknown without timing. - auto it = attempts_.find(sample.server_id); - if (it == attempts_.end()) { - any_pending = true; - } + if (sample.status != RemoteServerPresence::kExcluded) { + ++usable; } } if (usable == 0) { Complete(PeerPresence{PeerPresenceState::kUnknown}); return; } - static_cast(any_pending); + if (AllUsableTerminal()) { + Complete(AggregateRemotePresence(samples_)); + } } void QueryPeerPresence::Complete(PeerPresence const& presence) { diff --git a/aether/ae_actions/query_peer_presence.h b/aether/ae_actions/query_peer_presence.h index ab7e1201..e4a508be 100644 --- a/aether/ae_actions/query_peer_presence.h +++ b/aether/ae_actions/query_peer_presence.h @@ -44,9 +44,8 @@ enum class QueryPeerPresenceError : int { kGetClientTimingFailed = 3, }; -// Asynchronous Remote Presence over authoritative usable servers. -// Aggregation: Offline = any Offline; Online = all usable Online; -// Unknown = zero usable or incomplete Online set without Offline. +// Asynchronous Remote Presence over peer Personal Cloud authoritative servers. +// Never falls back to the observer/requester own cloud. class QueryPeerPresence final : public Action { public: using ResultEvent = Event)>; @@ -61,25 +60,38 @@ class QueryPeerPresence final : public Action { std::vector const& samples() const noexcept { return samples_; } + std::vector const& peer_cloud_server_ids() const noexcept { + return peer_cloud_server_ids_; + } + std::vector const& authoritative_server_ids() const noexcept { + return authoritative_server_ids_; + } + std::vector const& queried_server_ids() const noexcept { + return queried_server_ids_; + } + bool used_observer_cloud() const noexcept { return used_observer_cloud_; } private: struct AttemptMeta { TimePoint send_time{}; std::uint64_t generation{0}; - std::size_t retries_used{0}; }; void OnCloud(Result result); + void BindPeerCloud(Cloud::ptr cloud); void StartQuery(); void RefreshUsableSet(); + void RequestTiming(CloudServerConnection* sc); void OnServerTiming(CloudServerConnection* sc, std::uint64_t generation, Result const& res); void MarkUnknown(ServerId server_id); void MarkExcluded(ServerId server_id); + void OnServerRecovered(CloudServerConnection* sc); void MaybeComplete(); void Complete(PeerPresence const& presence); void Fail(int code); Duration OfflineTimeout() const noexcept; + bool AllUsableTerminal() const noexcept; AeContext ae_context_; Client* client_{nullptr}; @@ -98,6 +110,10 @@ class QueryPeerPresence final : public Action { std::map timing_subs_; std::map attempts_; std::vector samples_; + std::vector peer_cloud_server_ids_; + std::vector authoritative_server_ids_; + std::vector queried_server_ids_; + bool used_observer_cloud_{false}; bool finished_{false}; }; diff --git a/aether/remote_presence.h b/aether/remote_presence.h index 53fbbf4f..cd037fa6 100644 --- a/aether/remote_presence.h +++ b/aether/remote_presence.h @@ -17,6 +17,7 @@ #ifndef AETHER_REMOTE_PRESENCE_H_ #define AETHER_REMOTE_PRESENCE_H_ +#include #include #include #include @@ -203,6 +204,31 @@ inline Duration DefaultOfflineDetectionTimeout() noexcept { return std::chrono::milliseconds{AE_OFFLINE_DETECTION_TIMEOUT_MS}; } +// Local Presence (PingCloudServers + RequestPolicy::All) only maintains +// schedules on selected_servers(), which is bounded by +// AE_CLOUD_MAX_SERVER_CONNECTIONS. Remote Presence must AND over that same +// contract set — not an observer's own cloud, and not a silent subset smaller +// than the peer Presence obligation when peer cloud size <= max_connections. +inline std::vector AuthoritativePresenceServerIds( + std::vector const& peer_cloud_ids_priority_order, + std::size_t max_connections) noexcept { + std::vector out; + if (max_connections == 0) { + return out; + } + auto const n = + std::min(peer_cloud_ids_priority_order.size(), max_connections); + out.assign(peer_cloud_ids_priority_order.begin(), + peer_cloud_ids_priority_order.begin() + + static_cast(n)); + return out; +} + +// Never substitute the observer/requester cloud for the peer Personal Cloud. +inline bool AllowObserverCloudFallbackForPeerPresence() noexcept { + return false; +} + } // namespace ae #endif // AETHER_REMOTE_PRESENCE_H_ diff --git a/examples/remote_presence_live/CMakeLists.txt b/examples/remote_presence_live/CMakeLists.txt new file mode 100644 index 00000000..ceeace24 --- /dev/null +++ b/examples/remote_presence_live/CMakeLists.txt @@ -0,0 +1,31 @@ +# Copyright 2026 Aethernet Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.16.0) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +if(NOT CM_PLATFORM) + project("remote-presence-live" VERSION "1.0.0" LANGUAGES C CXX) + set(TARGET_NAME ${PROJECT_NAME}) + add_executable(${TARGET_NAME} main.cpp remote_presence_live.cpp) + target_include_directories(${TARGET_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(${TARGET_NAME} PRIVATE aether_examples_common) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(${TARGET_NAME} PRIVATE /Zc:preprocessor) + endif() +else() + message(WARNING "remote_presence_live is desktop-only") +endif() diff --git a/examples/remote_presence_live/main.cpp b/examples/remote_presence_live/main.cpp new file mode 100644 index 00000000..3ee7485c --- /dev/null +++ b/examples/remote_presence_live/main.cpp @@ -0,0 +1,19 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +extern int RemotePresenceLiveMain(int argc, char** argv); + +int main(int argc, char** argv) { return RemotePresenceLiveMain(argc, argv); } diff --git a/examples/remote_presence_live/remote_presence_live.cpp b/examples/remote_presence_live/remote_presence_live.cpp new file mode 100644 index 00000000..2ac98969 --- /dev/null +++ b/examples/remote_presence_live/remote_presence_live.cpp @@ -0,0 +1,655 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Live Remote Presence harness: client A (peer) + client B (observer). + * + * Env / args: + * AE_REMOTE_PRESENCE_HEALTHY_SEC healthy window seconds (default 300) + * --healthy-sec N + * --skip-fault skip firewall fault/recovery phases + * --fault-only skip healthy statistical window + */ + +#define AE_EXAMPLE_LORA_MODULE 0 +#define AE_EXAMPLE_MODEM 0 +#ifdef ESP_PLATFORM +# define AE_EXAMPLE_ESP_WIFI 1 +#else +# define AE_EXAMPLE_ETHERNET 1 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "aether-miscpp/format/format.h" +#include "aether/ae_actions/query_peer_presence.h" +#include "aether/all.h" +#include "aether/client_connectivity_policy.h" +#include "aether/cloud_connections/local_presence_schedule.h" +#include "aether/config.h" +#include "aether/remote_presence.h" + +// IWYU pragma: begin_keeps +#include "../common/aether_construct_esp_wifi.h" +#include "../common/aether_construct_ethernet.h" +#include "../common/aether_construct_lora_module.h" +#include "../common/aether_construct_modem.h" +// IWYU pragma: end_keeps + +#if defined(_WIN32) +# include +#endif + +namespace ae::examples { +namespace { + +using namespace std::chrono_literals; + +static constexpr auto kParentUid = + Uid::FromString("3ac93165-3d37-4970-87a6-fa4ee27744e4"); +static constexpr auto kInterval = 1s; +static constexpr auto kWindow = 1s; +static constexpr auto kOfflineTimeout = 1s; +static constexpr auto kQueryPeriod = 250ms; +static constexpr auto kPoll = 10ms; + +template +void Log(FormatScheme const& format, Args&&... args) { + Format(std::cout, ">>> [{:time}] ", Now()); + Format(std::cout, format, std::forward(args)...); + std::cout << '\n'; +} + +std::int64_t EpochMs(TimePoint tp) { + return std::chrono::duration_cast( + tp.time_since_epoch()) + .count(); +} + +char const* StateName(PeerPresenceState s) { + switch (s) { + case PeerPresenceState::kOnline: + return "ONLINE"; + case PeerPresenceState::kOffline: + return "OFFLINE"; + case PeerPresenceState::kUnknown: + return "UNKNOWN"; + } + return "?"; +} + +char const* SampleName(RemoteServerPresence s) { + switch (s) { + case RemoteServerPresence::kOnline: + return "ONLINE"; + case RemoteServerPresence::kOffline: + return "OFFLINE"; + case RemoteServerPresence::kUnknown: + return "UNKNOWN"; + case RemoteServerPresence::kExcluded: + return "EXCLUDED"; + } + return "?"; +} + +void Pump(AetherApp& app, TimePoint until) { + while (!app.IsExited() && Now() < until) { + auto const next = app.Update(Now()); + auto const poll_at = Now() + kPoll; + app.WaitUntil(next < poll_at ? next : poll_at); + } +} + +void ApplyTimings(Client& client) { + auto policy = client.connectivity_policy(); + if (!policy) { + return; + } + policy->ResetRxTimings(); + policy->SetOfflineDetectionTimeout(kOfflineTimeout); + policy->ConfigureRxTimings(RequestPolicy::All{}) + .ForAllPriorities(RxTimingConf::Every(kInterval).WithWindow(kWindow)); + for (auto* server : client.cloud_connection().selected_servers()) { + if (server == nullptr) { + continue; + } + policy->ConfigureServerRxTiming( + server->server_id(), + RxTimingConf::Every(kInterval).WithWindow(kWindow), 99); + } +} + +bool WaitLocalOnline(AetherApp& app, Client& client, Duration budget) { + auto const deadline = Now() + budget; + while (Now() < deadline && !app.IsExited()) { + Pump(app, Now() + kPoll); + if (client.IsLocallyOnline()) { + return true; + } + } + return client.IsLocallyOnline(); +} + +struct QueryStats { + std::uint64_t query_count{0}; + std::uint64_t online_count{0}; + std::uint64_t offline_count{0}; + std::uint64_t unknown_count{0}; + std::uint64_t false_offline_samples{0}; + std::uint64_t false_offline_transitions{0}; + std::uint64_t unknown_max_duration_ms{0}; + PeerPresenceState last{PeerPresenceState::kUnknown}; + TimePoint unknown_started{}; + bool in_unknown{false}; +}; + +struct QueryResult { + PeerPresence presence{}; + std::vector samples; + std::vector peer_cloud_ids; + std::vector authoritative_ids; + std::vector queried_ids; + bool used_observer_cloud{false}; + TimePoint start{}; + TimePoint complete{}; + std::uint64_t query_id{0}; +}; + +void LogIds(char const* label, std::vector const& ids) { + std::cout << " " << label << "=["; + for (std::size_t i = 0; i < ids.size(); ++i) { + if (i != 0) { + std::cout << ','; + } + std::cout << ids[i]; + } + std::cout << "]\n"; +} + +void LogQuery(QueryResult const& q) { + Log("query_id={} start_ms={} complete_ms={} aggregate={} " + "used_observer_cloud={}", + q.query_id, EpochMs(q.start), EpochMs(q.complete), + StateName(q.presence.state), q.used_observer_cloud ? 1 : 0); + LogIds("peer_cloud_server_ids", q.peer_cloud_ids); + LogIds("authoritative_server_ids", q.authoritative_ids); + LogIds("queried_server_ids", q.queried_ids); + for (auto const& s : q.samples) { + Log(" server={} status={} next_ping_delta_ms={} expected_open_ms={} " + "offline_deadline_ms={} has_timing={}", + s.server_id, SampleName(s.status), s.next_ping_delta_ms, + EpochMs(s.expected_open), EpochMs(s.offline_deadline), + s.has_timing ? 1 : 0); + } +} + +QueryResult RunOneQuery(AetherApp& app, Client& observer, Uid peer_uid, + std::uint64_t query_id) { + QueryResult out{}; + out.query_id = query_id; + out.start = Now(); + bool done = false; + auto& action = observer.QueryPeerPresence(peer_uid); + auto sub = action.result_event().Subscribe([&](auto const& res) { + out.complete = Now(); + if (res) { + out.presence = res.value(); + } else { + out.presence.state = PeerPresenceState::kUnknown; + } + out.samples = action.samples(); + out.peer_cloud_ids = action.peer_cloud_server_ids(); + out.authoritative_ids = action.authoritative_server_ids(); + out.queried_ids = action.queried_server_ids(); + out.used_observer_cloud = action.used_observer_cloud(); + done = true; + }); + auto const deadline = Now() + 30s; + while (!done && Now() < deadline && !app.IsExited()) { + Pump(app, Now() + kPoll); + } + if (!done) { + out.complete = Now(); + out.presence.state = PeerPresenceState::kUnknown; + } + LogQuery(out); + return out; +} + +void UpdateStats(QueryStats& stats, QueryResult const& q, bool peer_alive) { + ++stats.query_count; + switch (q.presence.state) { + case PeerPresenceState::kOnline: + ++stats.online_count; + break; + case PeerPresenceState::kOffline: + ++stats.offline_count; + if (peer_alive) { + ++stats.false_offline_samples; + if (stats.last != PeerPresenceState::kOffline) { + ++stats.false_offline_transitions; + } + } + break; + case PeerPresenceState::kUnknown: + ++stats.unknown_count; + break; + } + if (q.presence.state == PeerPresenceState::kUnknown) { + if (!stats.in_unknown) { + stats.in_unknown = true; + stats.unknown_started = q.complete; + } + } else if (stats.in_unknown) { + auto const dur = std::chrono::duration_cast( + q.complete - stats.unknown_started) + .count(); + if (dur > 0 && + static_cast(dur) > stats.unknown_max_duration_ms) { + stats.unknown_max_duration_ms = static_cast(dur); + } + stats.in_unknown = false; + } + stats.last = q.presence.state; +} + +void PrintStats(char const* title, QueryStats const& s) { + Log("STATS {} query_count={} ONLINE={} OFFLINE={} UNKNOWN={} " + "false_OFFLINE_samples={} false_OFFLINE_transitions={} " + "unknown_max_duration_ms={}", + title, s.query_count, s.online_count, s.offline_count, s.unknown_count, + s.false_offline_samples, s.false_offline_transitions, + s.unknown_max_duration_ms); +} + +#if defined(_WIN32) +std::wstring ThisExePath() { + wchar_t path[MAX_PATH]{}; + auto const n = GetModuleFileNameW(nullptr, path, MAX_PATH); + if (n == 0 || n >= MAX_PATH) { + return {}; + } + return std::wstring{path, static_cast(n)}; +} + +int RunHidden(std::wstring cmd) { + STARTUPINFOW si{}; + si.cb = sizeof(si); + si.dwFlags = STARTF_USESHOWWINDOW; + si.wShowWindow = SW_HIDE; + PROCESS_INFORMATION pi{}; + if (!CreateProcessW(nullptr, cmd.data(), nullptr, nullptr, FALSE, + CREATE_NO_WINDOW, nullptr, nullptr, &si, &pi)) { + return -1; + } + WaitForSingleObject(pi.hProcess, 20000); + DWORD code = 1; + GetExitCodeProcess(pi.hProcess, &code); + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + return static_cast(code); +} + +bool IsElevated() { + HANDLE token = nullptr; + if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &token)) { + return false; + } + TOKEN_ELEVATION elevation{}; + DWORD size = 0; + auto const ok = GetTokenInformation(token, TokenElevation, &elevation, + sizeof(elevation), &size); + CloseHandle(token); + return ok && (elevation.TokenIsElevated != 0); +} + +class WindowsExeFirewall { + public: + explicit WindowsExeFirewall(std::wstring exe_path) + : exe_path_{std::move(exe_path)}, + tag_{std::to_wstring(GetCurrentProcessId())} {} + ~WindowsExeFirewall() { Unblock(); } + WindowsExeFirewall(WindowsExeFirewall const&) = delete; + WindowsExeFirewall& operator=(WindowsExeFirewall const&) = delete; + + bool Block() { + Unblock(); + auto const quoted = L"\"" + exe_path_ + L"\""; + out_name_ = L"ae-rp-fw-out-" + tag_; + in_name_ = L"ae-rp-fw-in-" + tag_; + auto const out_cmd = + L"netsh advfirewall firewall add rule name=\"" + out_name_ + + L"\" dir=out action=block enable=yes profile=any program=" + quoted; + auto const in_cmd = + L"netsh advfirewall firewall add rule name=\"" + in_name_ + + L"\" dir=in action=block enable=yes profile=any program=" + quoted; + if (RunHidden(out_cmd) != 0 || RunHidden(in_cmd) != 0) { + Unblock(); + return false; + } + active_ = true; + return true; + } + + void Unblock() { + if (!out_name_.empty()) { + RunHidden(L"netsh advfirewall firewall delete rule name=\"" + out_name_ + + L"\""); + } + if (!in_name_.empty()) { + RunHidden(L"netsh advfirewall firewall delete rule name=\"" + in_name_ + + L"\""); + } + active_ = false; + } + + bool active() const { return active_; } + + private: + std::wstring exe_path_; + std::wstring tag_; + std::wstring out_name_; + std::wstring in_name_; + bool active_{false}; +}; +#endif + +struct Options { + int healthy_sec{300}; + bool skip_fault{false}; + bool fault_only{false}; +}; + +Options ParseOptions(int argc, char** argv) { + Options opt{}; + if (char const* env = std::getenv("AE_REMOTE_PRESENCE_HEALTHY_SEC")) { + opt.healthy_sec = std::atoi(env); + } + for (int i = 1; i < argc; ++i) { + std::string_view a{argv[i]}; + if (a == "--skip-fault") { + opt.skip_fault = true; + } else if (a == "--fault-only") { + opt.fault_only = true; + } else if (a == "--healthy-sec" && i + 1 < argc) { + opt.healthy_sec = std::atoi(argv[++i]); + } + } + if (opt.healthy_sec < 0) { + opt.healthy_sec = 0; + } + return opt; +} + +} // namespace + +int RemotePresenceLiveMain(int argc, char** argv) { + auto const opt = ParseOptions(argc, argv); + Log("remote_presence_live.start healthy_sec={} skip_fault={} fault_only={}", + opt.healthy_sec, opt.skip_fault ? 1 : 0, opt.fault_only ? 1 : 0); + + auto app = construct_aether_app(); + Client::ptr client_a; + Client::ptr client_b; + + { + auto& sa = app->aether()->SelectClient(kParentUid, "presence-A"); + sa.result_event().Subscribe([&](auto const& res) { + if (res) { + client_a = res.value(); + } + }); + auto& sb = app->aether()->SelectClient(kParentUid, "presence-B"); + sb.result_event().Subscribe([&](auto const& res) { + if (res) { + client_b = res.value(); + } + }); + Pump(*app, Now() + 60s); + } + + if (!client_a || !client_b) { + Log("FAIL SelectClient A/B (no cloud / network) — SKIP live validation"); + return 2; + } + + Log("clients ready A={} B={}", client_a->uid(), client_b->uid()); + ApplyTimings(*client_a.Load()); + ApplyTimings(*client_b.Load()); + (void)client_a->cloud_connection(); + (void)client_b->cloud_connection(); + + if (!WaitLocalOnline(*app, *client_a.Load(), 45s) || + !WaitLocalOnline(*app, *client_b.Load(), 45s)) { + Log("FAIL clients did not become locally ONLINE — SKIP"); + return 2; + } + + // Authoritative set diagnostics from a single probe query. + { + auto probe = RunOneQuery(*app, *client_b.Load(), client_a->uid(), 0); + Log("AUTHORITATIVE_SET peer_cloud_count={} authoritative_count={} " + "queried_count={} selected_observer_count={} " + "AE_CLOUD_MAX_SERVER_CONNECTIONS={}", + probe.peer_cloud_ids.size(), probe.authoritative_ids.size(), + probe.queried_ids.size(), + client_b->cloud_connection().selected_servers().size(), + AE_CLOUD_MAX_SERVER_CONNECTIONS); + if (probe.used_observer_cloud) { + Log("FAIL used_observer_cloud=true (own-cloud fallback must be removed)"); + return 1; + } + for (auto const qid : probe.queried_ids) { + auto const in_peer = + std::find(probe.peer_cloud_ids.begin(), probe.peer_cloud_ids.end(), + qid) != probe.peer_cloud_ids.end(); + if (!probe.peer_cloud_ids.empty() && !in_peer) { + Log("FAIL queried server {} not in peer cloud", qid); + return 1; + } + } + } + + std::uint64_t query_id = 1; + QueryStats healthy{}; + + if (!opt.fault_only && opt.healthy_sec > 0) { + Log("HEALTHY_REMOTE start duration_sec={}", opt.healthy_sec); + auto const end = Now() + std::chrono::seconds{opt.healthy_sec}; + while (Now() < end && !app->IsExited()) { + auto q = + RunOneQuery(*app, *client_b.Load(), client_a->uid(), query_id++); + UpdateStats(healthy, q, /*peer_alive=*/true); + if (q.used_observer_cloud) { + Log("FAIL own-cloud fallback during healthy"); + return 1; + } + Pump(*app, Now() + kQueryPeriod); + } + PrintStats("healthy_remote", healthy); + if (healthy.false_offline_samples != 0 || + healthy.false_offline_transitions != 0) { + Log("FAIL healthy remote false OFFLINE"); + return 1; + } + Log("HEALTHY_REMOTE PASS"); + } + + if (opt.skip_fault) { + Log("skip fault/recovery phases"); + return 0; + } + +#if defined(_WIN32) + if (!IsElevated()) { + Log("SKIP fault/recovery: Administrator required for firewall block"); + return 0; + } + + WindowsExeFirewall fw{ThisExePath()}; + // Fault: block this process network — both A and B share the process, so + // true "A-only" isolation is not possible in-process. Measure Remote + // OFFLINE under full process block as a transport-dominated bound, and + // report Local A OFFLINE latency from the same fault. + Log("FAULT start (process firewall block — A and B share process)"); + auto const fault_time = Now(); + if (!fw.Block()) { + Log("SKIP fault: netsh advfirewall failed"); + return 0; + } + + TimePoint local_offline_time{}; + TimePoint remote_offline_time{}; + bool saw_local_offline = false; + bool saw_remote_offline = false; + auto const fault_deadline = fault_time + 30s; + while (Now() < fault_deadline && !app->IsExited()) { + Pump(*app, Now() + kPoll); + if (!saw_local_offline && !client_a->IsLocallyOnline()) { + local_offline_time = Now(); + saw_local_offline = true; + Log("Local A OFFLINE at_ms={} fault->OFFLINE_ms={}", + EpochMs(local_offline_time), + EpochMs(local_offline_time) - EpochMs(fault_time)); + } + auto q = RunOneQuery(*app, *client_b.Load(), client_a->uid(), query_id++); + if (!saw_remote_offline && + q.presence.state == PeerPresenceState::kOffline) { + remote_offline_time = q.complete; + saw_remote_offline = true; + Log("Remote A OFFLINE at_ms={} fault->OFFLINE_ms={}", + EpochMs(remote_offline_time), + EpochMs(remote_offline_time) - EpochMs(fault_time)); + break; + } + Pump(*app, Now() + kQueryPeriod); + } + + if (!saw_remote_offline) { + Log("NOTE: Remote OFFLINE not observed under process-wide block " + "(observer B also lost cloud — expected UNKNOWN, not OFFLINE)"); + } + + Log("RECOVERY unblock"); + auto const unblock_time = Now(); + fw.Unblock(); + + TimePoint local_online_time{}; + TimePoint remote_online_time{}; + bool saw_local_online = false; + bool saw_remote_online = false; + auto const recover_deadline = unblock_time + 60s; + while (Now() < recover_deadline && !app->IsExited()) { + Pump(*app, Now() + kPoll); + if (!saw_local_online && client_a->IsLocallyOnline()) { + local_online_time = Now(); + saw_local_online = true; + Log("Local A ONLINE at_ms={} unblock->ONLINE_ms={}", + EpochMs(local_online_time), + EpochMs(local_online_time) - EpochMs(unblock_time)); + } + auto q = RunOneQuery(*app, *client_b.Load(), client_a->uid(), query_id++); + if (!saw_remote_online && + q.presence.state == PeerPresenceState::kOnline) { + remote_online_time = q.complete; + saw_remote_online = true; + Log("Remote A ONLINE at_ms={} unblock->ONLINE_ms={}", + EpochMs(remote_online_time), + EpochMs(remote_online_time) - EpochMs(unblock_time)); + break; + } + Pump(*app, Now() + kQueryPeriod); + } + + Log("FAULT_SUMMARY interval=1s offline_detection_timeout=1s " + "fault_ms={} remote_offline_ms={} fault_to_remote_offline_ms={} " + "local_offline_ms={} fault_to_local_offline_ms={} " + "unblock_ms={} local_online_ms={} remote_online_ms={} " + "unblock_to_local_ms={} unblock_to_remote_ms={}", + EpochMs(fault_time), + saw_remote_offline ? EpochMs(remote_offline_time) : -1, + saw_remote_offline ? (EpochMs(remote_offline_time) - EpochMs(fault_time)) + : -1, + saw_local_offline ? EpochMs(local_offline_time) : -1, + saw_local_offline ? (EpochMs(local_offline_time) - EpochMs(fault_time)) + : -1, + EpochMs(unblock_time), + saw_local_online ? EpochMs(local_online_time) : -1, + saw_remote_online ? EpochMs(remote_online_time) : -1, + saw_local_online ? (EpochMs(local_online_time) - EpochMs(unblock_time)) + : -1, + saw_remote_online ? (EpochMs(remote_online_time) - EpochMs(unblock_time)) + : -1); + + // All-servers-unavailable under process block should be UNKNOWN, never + // Offline solely because the observer lost cloud. Re-check with a short + // block while capturing aggregate. + { + Log("ALL_SERVERS_UNAVAILABLE probe"); + if (!fw.Block()) { + Log("SKIP all-servers probe"); + } else { + auto q = RunOneQuery(*app, *client_b.Load(), client_a->uid(), query_id++); + Log("all_servers_unavailable aggregate={} (expect UNKNOWN, never " + "OFFLINE-from-own-loss alone)", + StateName(q.presence.state)); + bool pass = q.presence.state != PeerPresenceState::kOffline || + !q.authoritative_ids.empty(); + // If every usable authoritative server is unreachable, status must be + // UNKNOWN (usable_count==0 or unresolved), not a fabricated Offline. + if (q.presence.state == PeerPresenceState::kOffline) { + bool any_offline_sample = false; + for (auto const& s : q.samples) { + if (s.status == RemoteServerPresence::kOffline) { + any_offline_sample = true; + } + } + pass = any_offline_sample; + } else { + pass = q.presence.state == PeerPresenceState::kUnknown; + } + Log("ALL_SERVERS_UNAVAILABLE {}", pass ? "PASS" : "FAIL"); + fw.Unblock(); + if (!pass) { + return 1; + } + WaitLocalOnline(*app, *client_a.Load(), 45s); + WaitLocalOnline(*app, *client_b.Load(), 45s); + } + } + + Log("ONE_SERVER_UNAVAILABLE SKIP (requires multi-server isolation of one " + "peer server from B only — not available in shared-process harness)"); + Log("FIREWALL phases completed (process-wide block)"); +#else + Log("SKIP fault/recovery/firewall: Win32-only in this harness"); +#endif + + Log("remote_presence_live.done"); + return 0; +} + +} // namespace ae::examples + +int RemotePresenceLiveMain(int argc, char** argv) { + return ae::examples::RemotePresenceLiveMain(argc, argv); +} diff --git a/tests/test-local-presence/main.cpp b/tests/test-local-presence/main.cpp index c3f67dde..98ce6b3e 100644 --- a/tests/test-local-presence/main.cpp +++ b/tests/test-local-presence/main.cpp @@ -1030,6 +1030,108 @@ void test_RemoteTimingProjectionAndAggregation() { {{ServerId{1}, RemoteServerPresence::kOffline, {}, {}, 0, true}})); } +void test_NoObserverCloudFallbackAndAuthoritativeSet() { + TEST_ASSERT_FALSE(AllowObserverCloudFallbackForPeerPresence()); + + // Peer Personal Cloud with 4 servers; Local Presence contract is bounded by + // AE_CLOUD_MAX_SERVER_CONNECTIONS (default 3) via selected_servers(). + std::vector peer_cloud{10, 11, 12, 13}; + auto const contract = AuthoritativePresenceServerIds( + peer_cloud, AE_CLOUD_MAX_SERVER_CONNECTIONS); + TEST_ASSERT_EQUAL(4, peer_cloud.size()); + TEST_ASSERT_EQUAL(AE_CLOUD_MAX_SERVER_CONNECTIONS, contract.size()); + TEST_ASSERT_EQUAL(10, contract[0]); + TEST_ASSERT_EQUAL(11, contract[1]); + TEST_ASSERT_EQUAL(12, contract[2]); + // Server 13 is in peer cloud but outside the Local Presence contract when + // N > AE_CLOUD_MAX_SERVER_CONNECTIONS — Remote AND must not require it. + TEST_ASSERT_TRUE(std::find(contract.begin(), contract.end(), + static_cast(13)) == contract.end()); + + // When N <= max, authoritative set equals the full peer cloud. + std::vector peer_small{21, 22}; + auto const full = AuthoritativePresenceServerIds( + peer_small, AE_CLOUD_MAX_SERVER_CONNECTIONS); + TEST_ASSERT_EQUAL(2, full.size()); + TEST_ASSERT_EQUAL(21, full[0]); + TEST_ASSERT_EQUAL(22, full[1]); + + // Observer cloud IDs must never be treated as peer Presence substitutes. + std::vector observer{90, 91}; + auto const observer_contract = + AuthoritativePresenceServerIds(observer, AE_CLOUD_MAX_SERVER_CONNECTIONS); + for (auto const id : observer_contract) { + TEST_ASSERT_TRUE(std::find(contract.begin(), contract.end(), id) == + contract.end()); + TEST_ASSERT_TRUE(std::find(full.begin(), full.end(), id) == full.end()); + } + + // Peer cloud unavailable => aggregate UNKNOWN with zero usable samples. + // QueryPeerPresence::OnCloud(Error) completes with this result and never + // binds the observer cloud (used_observer_cloud remains false; queried + // server list stays empty — no observer servers contacted). + std::vector empty; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kUnknown), + static_cast(AggregateRemotePresence(empty).state)); + TEST_ASSERT_EQUAL(0, empty.size()); +} + +void test_PeerCloudNotObserverCloudAuthoritativeIds() { + // Deterministic peer vs observer cloud sets (integration contract). + std::vector peer{101, 102}; + std::vector observer{201, 202}; + auto const auth = + AuthoritativePresenceServerIds(peer, AE_CLOUD_MAX_SERVER_CONNECTIONS); + TEST_ASSERT_EQUAL(2, auth.size()); + TEST_ASSERT_EQUAL(101, auth[0]); + TEST_ASSERT_EQUAL(102, auth[1]); + for (auto const oid : observer) { + TEST_ASSERT_TRUE(std::find(auth.begin(), auth.end(), oid) == auth.end()); + } + // Unknown peer cloud: no authoritative servers => UNKNOWN, no fallback set. + auto const none = + AuthoritativePresenceServerIds({}, AE_CLOUD_MAX_SERVER_CONNECTIONS); + TEST_ASSERT_EQUAL(0, none.size()); + TEST_ASSERT_EQUAL( + static_cast(PeerPresenceState::kUnknown), + static_cast( + AggregateRemotePresence(std::vector{}) + .state)); +} + +void test_RecoveredServerRequiresFreshOnline() { + std::vector samples{ + {1, RemoteServerPresence::kOnline, {}, {}, 1, true}, + {2, RemoteServerPresence::kExcluded, {}, {}, 0, false}, + }; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kOnline), + static_cast(AggregateRemotePresence(samples).state)); + // Recovered server re-enters as UNKNOWN — cannot keep stale ONLINE. + samples[1].status = RemoteServerPresence::kUnknown; + samples[1].has_timing = false; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kUnknown), + static_cast(AggregateRemotePresence(samples).state)); + samples[1].status = RemoteServerPresence::kOnline; + samples[1].has_timing = true; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kOnline), + static_cast(AggregateRemotePresence(samples).state)); +} + +void test_QueryFailureIsUnknownNotOffline() { + // Usable server with failed timing (UNKNOWN after retries) must not force + // peer Offline; zero Offline contributions + incomplete ONLINE => UNKNOWN. + std::vector samples{ + {1, RemoteServerPresence::kOnline, {}, {}, 1, true}, + {2, RemoteServerPresence::kUnknown, {}, {}, 0, true}, + }; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kUnknown), + static_cast(AggregateRemotePresence(samples).state)); + // After quarantine/unselect, remaining ONLINE servers may aggregate ONLINE. + samples[1].status = RemoteServerPresence::kExcluded; + TEST_ASSERT_EQUAL(static_cast(PeerPresenceState::kOnline), + static_cast(AggregateRemotePresence(samples).state)); +} + } // namespace ae::test_local_presence void setUp() {} @@ -1067,6 +1169,10 @@ int main() { RUN_TEST(ae::test_local_presence::test_IntervalZeroWithoutPongKeepsConfirmed); RUN_TEST(ae::test_local_presence::test_IntervalZeroWithPongClearsFuturePresence); RUN_TEST(ae::test_local_presence::test_RemoteTimingProjectionAndAggregation); + RUN_TEST(ae::test_local_presence::test_NoObserverCloudFallbackAndAuthoritativeSet); + RUN_TEST(ae::test_local_presence::test_PeerCloudNotObserverCloudAuthoritativeIds); + RUN_TEST(ae::test_local_presence::test_RecoveredServerRequiresFreshOnline); + RUN_TEST(ae::test_local_presence::test_QueryFailureIsUnknownNotOffline); RUN_TEST(ae::test_local_presence::test_StatisticalRuntimePollingIsLocallyOnline); RUN_TEST(ae::test_local_presence::test_FaultOfflineNotBeforeWindowCloseThenRecovery); return UNITY_END(); From 142bc58e72d27ea9a5bde4be5449b8b8801b0c31 Mon Sep 17 00:00:00 2001 From: aethernet-io Date: Wed, 2 Sep 2026 19:45:10 -0700 Subject: [PATCH 06/11] Add CloudRequestExecutionPolicy with soft timeout, retry, and hedge. Replace fixed CloudRequest timeouts and Restream-on-timeout with a runtime snapshot policy (pXX RTT x factor, retry_count, hedge_next_servers), quarantine only after attempt exhaustion, and accept late responses. QueryPeerPresence uses the client policy; Local Presence is unchanged. Co-authored-by: Cursor --- aether/ae_actions/query_peer_presence.cpp | 64 ++- aether/ae_actions/query_peer_presence.h | 13 +- aether/client_connectivity_policy.cpp | 11 + aether/client_connectivity_policy.h | 11 + aether/cloud_connections/cloud_request.cpp | 430 +++++++++++++----- aether/cloud_connections/cloud_request.h | 121 +++-- .../cloud_request_execution_policy.h | 184 ++++++++ .../cloud_server_connections.cpp | 5 + .../cloud_server_connections.h | 6 + aether/remote_presence.h | 2 - .../remote_presence_live.cpp | 4 + tests/CMakeLists.txt | 1 + tests/test-cloud-request/CMakeLists.txt | 29 ++ tests/test-cloud-request/main.cpp | 257 +++++++++++ 14 files changed, 927 insertions(+), 211 deletions(-) create mode 100644 aether/cloud_connections/cloud_request_execution_policy.h create mode 100644 tests/test-cloud-request/CMakeLists.txt create mode 100644 tests/test-cloud-request/main.cpp diff --git a/aether/ae_actions/query_peer_presence.cpp b/aether/ae_actions/query_peer_presence.cpp index f5c20e4b..c9b6c1eb 100644 --- a/aether/ae_actions/query_peer_presence.cpp +++ b/aether/ae_actions/query_peer_presence.cpp @@ -65,6 +65,14 @@ Duration QueryPeerPresence::OfflineTimeout() const noexcept { return policy.Load()->offline_detection_timeout(); } +CloudRequestExecutionPolicy QueryPeerPresence::ExecutionPolicy() const noexcept { + auto policy = client_->connectivity_policy(); + if (!policy) { + return CloudRequestExecutionPolicy::Default(); + } + return policy.Load()->cloud_request_execution_policy(); +} + void QueryPeerPresence::BindPeerCloud(Cloud::ptr cloud) { used_observer_cloud_ = false; @@ -190,17 +198,17 @@ void QueryPeerPresence::StartQuery() { queried_server_ids_.push_back(server_id); } auto& meta = attempts_[server_id]; - ++meta.generation; - meta.send_time = Now(); - auto const generation = meta.generation; - timing_subs_[server_id] = + auto const generation = ++meta.next_generation; + meta.send_times[generation] = Now(); + // Accumulate subscribers — do not replace, so late responses from + // earlier soft-timeout attempts remain deliverable. + timing_subs_[server_id] += auth_api->get_client_timing(peer_uid_).Subscribe( [this, sc, generation](auto const& res) { OnServerTiming(sc, generation, res); }); }}, - *work_cloud_, RequestPolicy::All{}, - /*max_retries=*/kRemotePresenceQueryRetryCount + 1); + *work_cloud_, RequestPolicy::All{}, ExecutionPolicy()); exhausted_sub_ = cloud_request_->attempt_exhausted_event().Subscribe( [this](CloudServerConnection* sc) { @@ -249,18 +257,18 @@ void QueryPeerPresence::RequestTiming(CloudServerConnection* sc) { } auto& meta = attempts_[server_id]; - ++meta.generation; - meta.send_time = Now(); - auto const generation = meta.generation; + auto const generation = ++meta.next_generation; + meta.send_times[generation] = Now(); // AuthorizedApiCall requires an active ApiContext path via CloudRequest's // handler; for recovered servers we re-enter through a one-server request. conn->AuthorizedApiCall(SubApi{[&, sc, generation]( ApiContext& auth_api) { - timing_subs_[server_id] = auth_api->get_client_timing(peer_uid_).Subscribe( - [this, sc, generation](auto const& res) { - OnServerTiming(sc, generation, res); - }); + timing_subs_[server_id] += + auth_api->get_client_timing(peer_uid_).Subscribe( + [this, sc, generation](auto const& res) { + OnServerTiming(sc, generation, res); + }); }}); } @@ -292,10 +300,27 @@ void QueryPeerPresence::OnServerTiming( return; } auto const server_id = sc->server_id(); + for (auto const& sample : samples_) { + if (sample.server_id == server_id && + (sample.status == RemoteServerPresence::kOnline || + sample.status == RemoteServerPresence::kOffline || + sample.status == RemoteServerPresence::kExcluded)) { + // Already terminal for this server — ignore duplicate/late extras. + return; + } + } + auto meta_it = attempts_.find(server_id); - if (meta_it == attempts_.end() || meta_it->second.generation != generation) { + if (meta_it == attempts_.end()) { + return; + } + auto send_it = meta_it->second.send_times.find(generation); + if (send_it == meta_it->second.send_times.end()) { return; } + auto const send_time = send_it->second; + meta_it->second.send_times.erase(send_it); + if (!res) { if (cloud_request_.has_value()) { auto const exhausted = cloud_request_->FailAttempt(sc); @@ -314,8 +339,8 @@ void QueryPeerPresence::OnServerTiming( TimePoint expected{}; TimePoint deadline{}; auto const status = ClassifyRemoteServerPresence( - recv, meta_it->second.send_time, recv, res.value(), OfflineTimeout(), - &expected, &deadline); + recv, send_time, recv, res.value(), OfflineTimeout(), &expected, + &deadline); for (auto& sample : samples_) { if (sample.server_id != server_id) { @@ -330,13 +355,16 @@ void QueryPeerPresence::OnServerTiming( } AE_TELED_DEBUG( - "REMOTE_PRESENCE server {} next_delta {} status {} expected {} deadline {}", + "REMOTE_PRESENCE server {} next_delta {} status {} expected {} deadline " + "{} (generation {})", server_id, res.value().next_ping_delta_ms, static_cast(status), - expected, deadline); + expected, deadline, generation); if (cloud_request_.has_value()) { cloud_request_->SucceedAttempt(sc); } + // Drop remaining attempt send times — server is done. + meta_it->second.send_times.clear(); MaybeComplete(); } diff --git a/aether/ae_actions/query_peer_presence.h b/aether/ae_actions/query_peer_presence.h index e4a508be..2140ca0a 100644 --- a/aether/ae_actions/query_peer_presence.h +++ b/aether/ae_actions/query_peer_presence.h @@ -30,9 +30,11 @@ #include "aether/cloud_connections/cloud_request.h" #include "aether/events/event_subscription.h" #include "aether/events/events.h" +#include "aether/events/multi_subscription.h" #include "aether/remote_presence.h" #include "aether/types/server_id.h" #include "aether/types/uid.h" +#include "aether/cloud_connections/cloud_request_execution_policy.h" namespace ae { @@ -73,8 +75,10 @@ class QueryPeerPresence final : public Action { private: struct AttemptMeta { - TimePoint send_time{}; - std::uint64_t generation{0}; + std::uint64_t next_generation{0}; + // Per-attempt send times so late responses from earlier attempts remain + // classifiable after a soft-timeout retry is launched. + std::map send_times; }; void OnCloud(Result result); @@ -91,6 +95,7 @@ class QueryPeerPresence final : public Action { void Complete(PeerPresence const& presence); void Fail(int code); Duration OfflineTimeout() const noexcept; + CloudRequestExecutionPolicy ExecutionPolicy() const noexcept; bool AllUsableTerminal() const noexcept; AeContext ae_context_; @@ -107,7 +112,9 @@ class QueryPeerPresence final : public Action { std::unique_ptr dest_cloud_; CloudServerConnections* work_cloud_{nullptr}; std::optional cloud_request_; - std::map timing_subs_; + // MultiSubscription so soft-timeout retries do not destroy earlier + // get_client_timing response subscribers (late responses must be accepted). + std::map timing_subs_; std::map attempts_; std::vector samples_; std::vector peer_cloud_server_ids_; diff --git a/aether/client_connectivity_policy.cpp b/aether/client_connectivity_policy.cpp index 0d3fe286..c5396027 100644 --- a/aether/client_connectivity_policy.cpp +++ b/aether/client_connectivity_policy.cpp @@ -246,6 +246,17 @@ void ClientConnectivityPolicy::SetOfflineDetectionTimeout( offline_detection_timeout_ = timeout; } +void ClientConnectivityPolicy::SetCloudRequestExecutionPolicy( + CloudRequestExecutionPolicy policy) noexcept { + if (policy.response_percentile > 100) { + policy.response_percentile = 100; + } + if (policy.timeout_factor_permille == 0) { + policy.timeout_factor_permille = 1000; + } + cloud_request_execution_policy_ = policy; +} + bool ClientConnectivityPolicy::IsLocallyOnline() const noexcept { return IsLocallyOnline(Now()); } diff --git a/aether/client_connectivity_policy.h b/aether/client_connectivity_policy.h index b82827f4..5d9b289b 100644 --- a/aether/client_connectivity_policy.h +++ b/aether/client_connectivity_policy.h @@ -24,6 +24,7 @@ #include #include +#include "aether/cloud_connections/cloud_request_execution_policy.h" #include "aether/cloud_connections/local_presence_schedule.h" #include "aether/cloud_connections/request_policy.h" #include "aether/config.h" @@ -191,6 +192,15 @@ class ClientConnectivityPolicy : public Obj { return offline_detection_timeout_; } + // Runtime CloudRequest soft-timeout / retry / hedge policy (not wire). + // Applies to NEW CloudRequest operations only (snapshot at construction). + void SetCloudRequestExecutionPolicy( + CloudRequestExecutionPolicy policy) noexcept; + CloudRequestExecutionPolicy const& cloud_request_execution_policy() + const noexcept { + return cloud_request_execution_policy_; + } + // Read-only. No side effects. Aggregate OR: ONLINE iff any selected server // has confirmed interval>0 and now <= expected_open + offline_detection_timeout. bool IsLocallyOnline() const noexcept; @@ -214,6 +224,7 @@ class ClientConnectivityPolicy : public Obj { std::uint8_t suspend_block_count_{}; Duration offline_detection_timeout_{std::chrono::milliseconds{ AE_OFFLINE_DETECTION_TIMEOUT_MS}}; + CloudRequestExecutionPolicy cloud_request_execution_policy_{}; Event suspend_allowed_event_; Event server_rx_timing_changed_event_; diff --git a/aether/cloud_connections/cloud_request.cpp b/aether/cloud_connections/cloud_request.cpp index bb0518b8..5eca7da6 100644 --- a/aether/cloud_connections/cloud_request.cpp +++ b/aether/cloud_connections/cloud_request.cpp @@ -16,49 +16,75 @@ #include "aether/cloud_connections/cloud_request.h" +#include +#include #include #include "aether-miscpp/misc/override.h" #include "aether/aether.h" +#include "aether/channels/channel.h" #include "aether/server.h" #include "aether/write_action/write_action.h" #include "aether/cloud_connections/cloud_connections_tele.h" namespace ae { +namespace { + +#if defined(AE_TELE_ENABLED) && AE_TELE_ENABLED +# define AE_CLOUD_REQ_DEBUG(...) AE_TELED_DEBUG(__VA_ARGS__) +# define AE_CLOUD_REQ_WARNING(...) AE_TELED_WARNING(__VA_ARGS__) +# define AE_CLOUD_REQ_ERROR(...) AE_TELED_ERROR(__VA_ARGS__) +#else +# define AE_CLOUD_REQ_DEBUG(...) +# define AE_CLOUD_REQ_WARNING(...) +# define AE_CLOUD_REQ_ERROR(...) +#endif + +} // namespace CloudRequest::CloudRequest(AeContext const& ae_context, ApiCallWithListener&& api_call, CloudServerConnections& cloud_server_connections, RequestPolicy::Variant policy, - std::size_t max_retries, Duration request_timeout) + CloudRequestExecutionPolicy exec_policy) : ae_context_{ae_context}, request_{std::move(api_call)}, cloud_scs_{&cloud_server_connections}, policy_{policy}, - max_retries_{max_retries}, - request_timeout_{request_timeout}, + exec_policy_{exec_policy}, server_changed_sub_{cloud_scs_->servers_update_event().Subscribe( MethodPtr<&CloudRequest::ServersUpdated>{this})} { - PrefillServerRequests(); - EnqueueMakeRequest(); + AE_CLOUD_REQ_DEBUG( + "CLOUD_REQUEST_START percentile={} factor_permille={} retry_count={} " + "hedge_next_servers={}", + exec_policy_.response_percentile, exec_policy_.timeout_factor_permille, + exec_policy_.retry_count, exec_policy_.hedge_next_servers); + RebuildCandidates(); + ActivateInitial(); + EnqueuePump(); } CloudRequest::CloudRequest(AeContext const& ae_context, ApiRequestHandler&& api_request, CloudServerConnections& cloud_server_connections, RequestPolicy::Variant policy, - std::size_t max_retries, Duration request_timeout) + CloudRequestExecutionPolicy exec_policy) : ae_context_{ae_context}, request_{std::move(api_request)}, cloud_scs_{&cloud_server_connections}, policy_{policy}, - max_retries_{max_retries}, - request_timeout_{request_timeout}, + exec_policy_{exec_policy}, server_changed_sub_{cloud_scs_->servers_update_event().Subscribe( MethodPtr<&CloudRequest::ServersUpdated>{this})} { - PrefillServerRequests(); - EnqueueMakeRequest(); + AE_CLOUD_REQ_DEBUG( + "CLOUD_REQUEST_START percentile={} factor_permille={} retry_count={} " + "hedge_next_servers={}", + exec_policy_.response_percentile, exec_policy_.timeout_factor_permille, + exec_policy_.retry_count, exec_policy_.hedge_next_servers); + RebuildCandidates(); + ActivateInitial(); + EnqueuePump(); } void CloudRequest::Succeeded() { @@ -71,20 +97,26 @@ void CloudRequest::Failed() { result_event_.Emit(false); } - void CloudRequest::SucceedAttempt(CloudServerConnection* sc) { auto it = server_requests_.find(sc); if (it == server_requests_.end()) { return; } auto& sr = it->second; - if (sr.succeeded) { + if (sr.exec.succeeded || sr.exec.exhausted) { return; } - sr.state_subs.Reset(); - sr.timeout_sub.Reset(); - sr.succeeded = true; - EnqueueMakeRequest(); + AE_CLOUD_REQ_DEBUG("SERVER_ATTEMPT_SUCCESS server_id={}", sc->server_id()); + // Cancel future soft timers; keep response_subs so in-flight late responses + // can still be observed by the listener if needed, but mark success first. + for (auto& attempt : sr.attempts) { + attempt.timeout_sub.Reset(); + } + sr.exec.MarkSucceeded(); + // Sequential hedge=0 style: after success, activate the next candidate if + // RequestPolicy still requires more servers. + ActivateFollowing(1); + EnqueuePump(); } bool CloudRequest::FailAttempt(CloudServerConnection* sc) { @@ -93,20 +125,28 @@ bool CloudRequest::FailAttempt(CloudServerConnection* sc) { return false; } auto& sr = it->second; - if (sr.succeeded) { + if (sr.exec.succeeded || sr.exec.exhausted) { + return false; + } + // Treat application-reported failure like a soft miss for budget purposes: + // do not Restream; retry if budget remains; otherwise quarantine. + bool const first_soft_miss = !sr.exec.first_soft_miss_seen; + auto const action = sr.exec.OnSoftTimeout(exec_policy_); + if (first_soft_miss && exec_policy_.hedge_next_servers > 0) { + ActivateFollowing(exec_policy_.hedge_next_servers, /*as_hedge=*/true, sc); + } + if (action == CloudRequestServerExecState::SoftTimeoutAction::kRetry) { + LaunchAttempt(sc, sr); + EnqueuePump(); return false; } - sr.state_subs.Reset(); - sr.timeout_sub.Reset(); - sr.retry_count++; - if (sr.retry_count >= max_retries_) { - AE_TELED_WARNING("Server {} retry budget exhausted on attempt failure", - sc->server_id()); - sr.exhausted = true; - EmitAttemptExhausted(sc); - } - EnqueueMakeRequest(); - return sr.exhausted; + if (action == CloudRequestServerExecState::SoftTimeoutAction::kExhaust) { + ExhaustServerNoResponse(sc, sr); + EnqueuePump(); + return true; + } + EnqueuePump(); + return sr.exec.exhausted; } CloudRequest::ResultEvent::Subscriber CloudRequest::result_event() { @@ -122,69 +162,121 @@ void CloudRequest::EmitAttemptExhausted(CloudServerConnection* sc) { attempt_exhausted_event_.Emit(sc); } -void CloudRequest::PrefillServerRequests() { - for (auto* sc : cloud_scs_->servers()) { - server_requests_.emplace(sc, ServerRequest{}); +void CloudRequest::RebuildCandidates() { + std::vector next; + cloud_scs_->ForServers([&](CloudServerConnection* sc) { next.push_back(sc); }, + policy_); + // Preserve activation cursor semantics: append newly eligible servers after + // existing candidate order; keep already-known pointers stable. + for (auto* sc : next) { + auto const known = + std::find(candidates_.begin(), candidates_.end(), sc) != + candidates_.end(); + if (!known) { + candidates_.push_back(sc); + server_requests_.emplace(sc, ServerRequest{}); + } } } -void CloudRequest::MakeRequest() { - cloud_scs_->ForServers( - [&](auto& sc) { - auto it = server_requests_.find(sc); - ServerRequest* sr; - if (it == server_requests_.end()) { - // New server added to cloud after construction - auto [new_it, ok] = server_requests_.emplace(sc, ServerRequest{}); - sr = &new_it->second; - } else { - sr = &it->second; - if (sr->exhausted || sr->succeeded) { - return; - } - } - MakeServerRequest(sc, *sr); - }, - policy_); +void CloudRequest::ActivateInitial() { + if (candidates_.empty()) { + return; + } + ActivateFollowing(1); +} - bool any_open = false; - bool any_succeeded = false; - for (auto const& [sc, sr] : server_requests_) { - if (sr.succeeded) { - any_succeeded = true; - } else if (!sr.exhausted) { - any_open = true; +void CloudRequest::ActivateFollowing(std::uint8_t count, bool as_hedge, + CloudServerConnection* source) { + while (count > 0 && activate_cursor_ < candidates_.size()) { + auto* sc = candidates_[activate_cursor_++]; + auto& sr = server_requests_[sc]; + if (sr.exec.activated || sr.exec.succeeded || sr.exec.exhausted) { + continue; + } + if (as_hedge) { + AE_CLOUD_REQ_DEBUG( + "SERVER_HEDGE_ACTIVATED source_server={} new_server={}", + source != nullptr ? source->server_id() : ServerId{}, sc->server_id()); } + ActivateServer(sc); + --count; } - if (!server_requests_.empty() && - CloudRequestShouldFailAll(any_open, any_succeeded)) { - AE_TELED_ERROR("All server requests exhausted, failing"); - Failed(); +} + +void CloudRequest::ActivateServer(CloudServerConnection* sc) { + auto& sr = server_requests_[sc]; + if (sr.exec.activated) { + return; } + sr.exec.activated = true; + LaunchAttempt(sc, sr); } -void CloudRequest::MakeServerRequest(CloudServerConnection* sc, - ServerRequest& sr) { - AE_TELED_DEBUG("Make request to server {}", sc->server_id()); +Duration CloudRequest::SoftTimeoutFor(CloudServerConnection* sc) const { + auto* conn = sc->client_connection(); + if (conn == nullptr) { + return ComputeCloudRequestSoftTimeout(FallbackCloudRequestRtt(), + exec_policy_); + } + auto channel = conn->server_connection().current_channel(); + if (!channel) { + return ComputeCloudRequestSoftTimeout(FallbackCloudRequestRtt(), + exec_policy_); + } + auto const& stats = + channel->channel_statistics().response_time_statistics(); + if (stats.empty()) { + return ComputeCloudRequestSoftTimeout(FallbackCloudRequestRtt(), + exec_policy_); + } + auto const rtt = + stats.PercentileValue(exec_policy_.response_percentile); + return ComputeCloudRequestSoftTimeout(rtt, exec_policy_); +} - // Clear previous subscriptions and timeout - sr.state_subs.Reset(); - sr.timeout_sub.Reset(); +void CloudRequest::LaunchAttempt(CloudServerConnection* sc, + ServerRequest& sr) { + auto const attempt_index = sr.exec.StartAttempt(exec_policy_); + if (attempt_index == 0) { + return; + } auto* conn = sc->client_connection(); - assert((conn != nullptr) && "Client connection is null"); + if (conn == nullptr) { + AE_CLOUD_REQ_WARNING("SERVER_ATTEMPT skipped disconnected server {}", + sc->server_id()); + // Hard unusable: exhaust and quarantine via existing health path if link + // errors; for a missing connection treat as attempt failure. + auto const action = sr.exec.OnSoftTimeout(exec_policy_); + if (action == CloudRequestServerExecState::SoftTimeoutAction::kExhaust) { + ExhaustServerNoResponse(sc, sr); + } else if (action == + CloudRequestServerExecState::SoftTimeoutAction::kRetry) { + // Will retry on next pump when connection appears. + sr.exec.attempts_started = + static_cast(sr.exec.attempts_started - 1); + } + return; + } + + auto const timeout = SoftTimeoutFor(sc); + AE_CLOUD_REQ_DEBUG( + "SERVER_ATTEMPT server_id={} attempt_index={} timeout_ms={}", + sc->server_id(), attempt_index, + std::chrono::duration_cast(timeout).count()); + + AttemptState attempt{}; + attempt.attempt_index = attempt_index; - // make request depends on saved request kind auto& swa = std::visit(Override{ - // ApiCallWithListener [&](ApiCallWithListener& api_call) -> decltype(auto) { return conn->AuthorizedApiCall( SubApi{[&](ApiContext& api) { api_call.call(api, sc); }}); }, - // ApiRequestHandler [&](ApiRequestHandler& api_request) -> decltype(auto) { return conn->AuthorizedApiCall( SubApi{[&](ApiContext& api) { @@ -194,121 +286,209 @@ void CloudRequest::MakeServerRequest(CloudServerConnection* sc, }, request_); - // if request write failed - sr.state_subs += swa.status_event().Subscribe([this, sc](auto status) { + // Write failure is a hard-ish send problem — count toward budget without + // Restream from soft timeout path. + attempt.subs += swa.status_event().Subscribe([this, sc](auto status) { if (status == WriteAction::Status::kFail) { - AE_TELED_WARNING("Request write error"); + AE_CLOUD_REQ_WARNING("Request write error server {}", sc->server_id()); OnWriteFailed(sc); } }); - // if server stream changed its channel, retry on new channel - sr.state_subs += + attempt.subs += conn->server_connection().channel_changed_event().Subscribe([this, sc]() { - AE_TELED_WARNING("Request server channel changed"); + AE_CLOUD_REQ_WARNING("Request server channel changed {}", + sc->server_id()); OnChannelChanged(sc); }); - // Set per-server request timeout - sr.timeout_sub = ae_context_.scheduler().DelayedTask( - [this, sc]() { - AE_TELED_WARNING("Request timeout for server {}", sc->server_id()); - OnServerRequestTimeout(sc); - }, - request_timeout_); - if (std::holds_alternative(request_)) { auto& listener = std::get(request_).listener; if (listener) { - sr.state_subs += listener(conn->client_safe_api(), sc, this); + // Keep listener subscriptions durable so late responses from earlier + // attempts are not destroyed when a retry is launched. + sr.response_subs += listener(conn->client_safe_api(), sc, this); } } + + attempt.timeout_sub = ae_context_.scheduler().DelayedTask( + [this, sc, attempt_index]() { OnSoftTimeout(sc, attempt_index); }, + timeout); + + sr.attempts.push_back(std::move(attempt)); } -void CloudRequest::OnChannelChanged(CloudServerConnection* sc) { +void CloudRequest::OnSoftTimeout(CloudServerConnection* sc, + std::uint8_t attempt_index) { auto it = server_requests_.find(sc); if (it == server_requests_.end()) { return; } auto& sr = it->second; - if (sr.succeeded) { + if (sr.exec.succeeded || sr.exec.exhausted) { + return; + } + + for (auto& attempt : sr.attempts) { + if (attempt.attempt_index == attempt_index) { + attempt.timed_out = true; + attempt.timeout_sub.Reset(); + break; + } + } + + AE_CLOUD_REQ_DEBUG( + "SERVER_SOFT_TIMEOUT server_id={} attempt_index={} (no Restream)", + sc->server_id(), attempt_index); + + bool const first_soft_miss = !sr.exec.first_soft_miss_seen; + auto const action = sr.exec.OnSoftTimeout(exec_policy_); + if (first_soft_miss && exec_policy_.hedge_next_servers > 0) { + ActivateFollowing(exec_policy_.hedge_next_servers, /*as_hedge=*/true, sc); + } + + if (action == CloudRequestServerExecState::SoftTimeoutAction::kRetry) { + LaunchAttempt(sc, sr); + EnqueuePump(); return; } - if (sr.retry_count >= max_retries_) { - AE_TELED_WARNING("Server {} retry budget exhausted", sc->server_id()); - sr.exhausted = true; - EmitAttemptExhausted(sc); - EnqueueMakeRequest(); + if (action == CloudRequestServerExecState::SoftTimeoutAction::kExhaust) { + ExhaustServerNoResponse(sc, sr); + EnqueuePump(); return; } - // Channel already changed, just re-send. - EnqueueMakeRequest(); + EnqueuePump(); } -void CloudRequest::OnWriteFailed(CloudServerConnection* sc) { +void CloudRequest::ExhaustServerNoResponse(CloudServerConnection* sc, + ServerRequest& sr) { + if (sr.exec.succeeded) { + return; + } + sr.exec.MarkExhausted(); + for (auto& attempt : sr.attempts) { + attempt.timeout_sub.Reset(); + } + AE_CLOUD_REQ_WARNING( + "SERVER_RETRY_EXHAUSTED server_id={} attempts={} -> " + "SERVER_QUARANTINE_NO_RESPONSE", + sc->server_id(), sr.exec.attempts_started); + cloud_scs_->QuarantineForNoResponse(*sc); + EmitAttemptExhausted(sc); + // After quarantine/reconcile, ServersUpdated may add a replacement — also + // advance sequential activation for remaining required candidates. + ActivateFollowing(1); +} + +void CloudRequest::OnChannelChanged(CloudServerConnection* sc) { auto it = server_requests_.find(sc); if (it == server_requests_.end()) { return; } auto& sr = it->second; - if (sr.succeeded) { + if (sr.exec.succeeded || sr.exec.exhausted || !sr.exec.activated) { return; } - sr.retry_count++; - if (sr.retry_count >= max_retries_) { - AE_TELED_WARNING("Server {} retry budget exhausted on write failure", - sc->server_id()); - sr.exhausted = true; - EmitAttemptExhausted(sc); + // Channel change is infrastructure: re-send without Restream-from-soft-timeout, + // using remaining attempt budget if available. + if (!sr.exec.CanStartAttempt(exec_policy_)) { + ExhaustServerNoResponse(sc, sr); + EnqueuePump(); + return; } - EnqueueMakeRequest(); + LaunchAttempt(sc, sr); + EnqueuePump(); } -void CloudRequest::OnServerRequestTimeout(CloudServerConnection* sc) { +void CloudRequest::OnWriteFailed(CloudServerConnection* sc) { auto it = server_requests_.find(sc); if (it == server_requests_.end()) { return; } auto& sr = it->second; - if (sr.succeeded) { + if (sr.exec.succeeded || sr.exec.exhausted) { return; } - if (sr.retry_count >= max_retries_) { - AE_TELED_WARNING("Server {} retry budget exhausted on timeout", - sc->server_id()); - sr.exhausted = true; - EmitAttemptExhausted(sc); - EnqueueMakeRequest(); - return; + // Write failure: do not soft-Restream; apply retry budget. Hard LinkError + // quarantine remains on the connection health path. + bool const first_soft_miss = !sr.exec.first_soft_miss_seen; + auto const action = sr.exec.OnSoftTimeout(exec_policy_); + if (first_soft_miss && exec_policy_.hedge_next_servers > 0) { + ActivateFollowing(exec_policy_.hedge_next_servers, /*as_hedge=*/true, sc); } - sr.retry_count++; - // Timeout means something is wrong with the stream. - // Restream to switch channels; channel_changed_event will trigger re-send. - sc->Restream(); + if (action == CloudRequestServerExecState::SoftTimeoutAction::kRetry) { + LaunchAttempt(sc, sr); + } else if (action == + CloudRequestServerExecState::SoftTimeoutAction::kExhaust) { + ExhaustServerNoResponse(sc, sr); + } + EnqueuePump(); } -void CloudRequest::ServersUpdated() { EnqueueMakeRequest(); } - -void CloudRequest::RemoveRequest(CloudServerConnection* server_connection) { - server_requests_.erase(server_connection); +void CloudRequest::ServersUpdated() { + RebuildCandidates(); + // If nothing is active yet, start the first candidate. Otherwise activate + // newly appended replacements only when sequential policy needs them or + // hedge already opened the window — ActivateFollowing(1) after exhaustion + // handles the common replacement case. + bool any_activated = false; + for (auto const& [sc, sr] : server_requests_) { + static_cast(sc); + if (sr.exec.activated && !sr.exec.exhausted && !sr.exec.succeeded) { + any_activated = true; + break; + } + } + if (!any_activated) { + ActivateInitial(); + } else { + // Ensure replacements beyond the cursor can be pulled in for All. + ActivateFollowing(1); + } + EnqueuePump(); } -void CloudRequest::EnqueueMakeRequest() { - // enqueue only once at a time +void CloudRequest::EnqueuePump() { if (task_sub_) { return; } task_sub_ = ae_context_.scheduler().Task([this]() { task_sub_.Reset(); - MakeRequest(); + Pump(); }); } +void CloudRequest::Pump() { + bool any_open = false; + bool any_succeeded = false; + for (auto const& [sc, sr] : server_requests_) { + static_cast(sc); + if (sr.exec.succeeded) { + any_succeeded = true; + } else if (sr.exec.activated && !sr.exec.exhausted) { + any_open = true; + } else if (!sr.exec.activated && !sr.exec.exhausted) { + // Candidate not yet activated — still open for sequential/hedge. + any_open = true; + } + } + // Candidates reserved but not activated still count as open work. + if (activate_cursor_ < candidates_.size()) { + any_open = true; + } + + if (!server_requests_.empty() && + CloudRequestShouldFailAll(any_open, any_succeeded) && + activate_cursor_ >= candidates_.size()) { + AE_CLOUD_REQ_ERROR("All server requests exhausted, failing"); + Failed(); + } +} + void CloudRequest::Finish() { - swa_sub_.Reset(); server_changed_sub_.Reset(); task_sub_.Reset(); server_requests_.clear(); - + candidates_.clear(); Action::Finish(); } diff --git a/aether/cloud_connections/cloud_request.h b/aether/cloud_connections/cloud_request.h index 78a15b52..6b1b0506 100644 --- a/aether/cloud_connections/cloud_request.h +++ b/aether/cloud_connections/cloud_request.h @@ -16,126 +16,121 @@ #ifndef AETHER_CLOUD_CONNECTIONS_CLOUD_REQUEST_H_ #define AETHER_CLOUD_CONNECTIONS_CLOUD_REQUEST_H_ +#include +#include #include +#include #include "aether/common.h" #include "aether/ae_context.h" #include "aether/actions/action.h" +#include "aether/cloud_connections/cloud_request_execution_policy.h" #include "aether/cloud_connections/request_policy.h" #include "aether/cloud_connections/cloud_callbacks.h" #include "aether/cloud_connections/cloud_server_connections.h" +#include "aether/events/multi_subscription.h" namespace ae { /** - * \brief Makes request according to the request policy. - * If request fails or times out, it will restream the cloud connection and - * retry on different channels. When a server exhausts its retry budget, - * it moves on to the next server in the list. - * ResponseSubscriber must subscribe to client_api and handle the - * response. On success, listener must call CloudRequest::Succeeded(). On - * failure, listener must call CloudRequest::Failed(). + * \brief Makes request according to RequestPolicy (candidate set) and + * CloudRequestExecutionPolicy (soft timeout / retry / hedge / quarantine). + * + * Soft response timeout does NOT Restream or quarantine. Quarantine for + * no-response happens only after the per-server retry budget is exhausted. + * Late valid responses from earlier attempts are accepted. + * + * ResponseSubscriber / ApiRequestHandler must handle responses. On whole + * request success, call CloudRequest::Succeeded(); on whole failure, + * CloudRequest::Failed(). Per-server: SucceedAttempt / FailAttempt. */ -// Testable per-server completion flags used by CloudRequest. -struct CloudRequestAttemptState { - bool exhausted{false}; - bool succeeded{false}; - std::size_t retry_count{0}; - - bool ShouldSkipMake() const noexcept { return exhausted || succeeded; } - - void MarkSucceeded() { succeeded = true; } - - // Returns true when this failure exhausted the retry budget. - bool MarkFailed(std::size_t max_retries) { - if (succeeded) { - return false; - } - ++retry_count; - if (retry_count >= max_retries) { - exhausted = true; - return true; - } - return false; - } -}; - -inline bool CloudRequestShouldFailAll(bool any_open, - bool any_succeeded) noexcept { - return !any_open && !any_succeeded; -} +// Compatibility alias for older unit helpers. +using CloudRequestAttemptState = CloudRequestServerExecState; class CloudRequest final : public Action { - struct ServerRequest { - MultiSubscription state_subs; + struct AttemptState { + MultiSubscription subs; TaskSubscription timeout_sub; - std::size_t retry_count{0}; - bool exhausted{false}; - bool succeeded{false}; + std::uint8_t attempt_index{0}; + bool timed_out{false}; }; - public: - static constexpr std::size_t kDefaultMaxRetries = 5; - static constexpr Duration kDefaultRequestTimeout = - std::chrono::milliseconds{AE_CLOUD_REQUEST_TIMEOUT_MS}; + struct ServerRequest { + CloudRequestServerExecState exec{}; + // Durable across attempts so late responses remain deliverable. + MultiSubscription response_subs; + std::vector attempts; + }; + public: using ResultEvent = Event; using AttemptExhaustedEvent = Event; CloudRequest(AeContext const& ae_context, ApiCallWithListener&& api_call, CloudServerConnections& cloud_server_connections, RequestPolicy::Variant policy, - std::size_t max_retries = kDefaultMaxRetries, - Duration request_timeout = kDefaultRequestTimeout); + CloudRequestExecutionPolicy exec_policy = + CloudRequestExecutionPolicy::Default()); CloudRequest(AeContext const& ae_context, ApiRequestHandler&& api_request, CloudServerConnections& cloud_server_connections, RequestPolicy::Variant policy, - std::size_t max_retries = kDefaultMaxRetries, - Duration request_timeout = kDefaultRequestTimeout); + CloudRequestExecutionPolicy exec_policy = + CloudRequestExecutionPolicy::Default()); AE_CLASS_NO_COPY_MOVE(CloudRequest) void Succeeded(); void Failed(); - // Per-server success: stop this server's timeout/channel/write - // subscriptions, skip further retries, and do not finish CloudRequest. + // Per-server success: accept late responses, cancel future retries, no + // Restream / quarantine. void SucceedAttempt(CloudServerConnection* sc); - // Listener-side attempt failure: retry/exhaust this server without - // ending the whole CloudRequest. Returns true when the server is exhausted. + // Listener-side attempt failure (e.g. API error). Soft retry budget applies. + // Returns true when the server is exhausted (and quarantined for no usable + // response path). bool FailAttempt(CloudServerConnection* sc); + CloudRequestExecutionPolicy const& execution_policy() const noexcept { + return exec_policy_; + } + ResultEvent::Subscriber result_event(); AttemptExhaustedEvent::Subscriber attempt_exhausted_event(); private: - void MakeRequest(); - void MakeServerRequest(CloudServerConnection* sc, ServerRequest& sr); - void PrefillServerRequests(); + void RebuildCandidates(); + void ActivateInitial(); + void ActivateFollowing(std::uint8_t count, bool as_hedge = false, + CloudServerConnection* source = nullptr); + void ActivateServer(CloudServerConnection* sc); + void LaunchAttempt(CloudServerConnection* sc, ServerRequest& sr); + + Duration SoftTimeoutFor(CloudServerConnection* sc) const; + void OnSoftTimeout(CloudServerConnection* sc, std::uint8_t attempt_index); + void ExhaustServerNoResponse(CloudServerConnection* sc, ServerRequest& sr); void ServersUpdated(); void OnChannelChanged(CloudServerConnection* sc); - void OnServerRequestTimeout(CloudServerConnection* sc); void OnWriteFailed(CloudServerConnection* sc); - void RemoveRequest(CloudServerConnection* server_connection); - void EnqueueMakeRequest(); + void EnqueuePump(); + void Pump(); void EmitAttemptExhausted(CloudServerConnection* sc); - void Finish(); AeContext ae_context_; std::variant request_; CloudServerConnections* cloud_scs_; RequestPolicy::Variant policy_; - std::size_t max_retries_; - Duration request_timeout_; - TaskSubscription task_sub_; + // Snapshot at construction ? runtime policy changes do not affect this op. + CloudRequestExecutionPolicy exec_policy_; - Subscription swa_sub_; + TaskSubscription task_sub_; Subscription server_changed_sub_; ResultEvent result_event_; AttemptExhaustedEvent attempt_exhausted_event_; + std::vector candidates_; + std::size_t activate_cursor_{0}; std::map server_requests_; }; diff --git a/aether/cloud_connections/cloud_request_execution_policy.h b/aether/cloud_connections/cloud_request_execution_policy.h new file mode 100644 index 00000000..467c0bf7 --- /dev/null +++ b/aether/cloud_connections/cloud_request_execution_policy.h @@ -0,0 +1,184 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AETHER_CLOUD_CONNECTIONS_CLOUD_REQUEST_EXECUTION_POLICY_H_ +#define AETHER_CLOUD_CONNECTIONS_CLOUD_REQUEST_EXECUTION_POLICY_H_ + +#include +#include + +#include + +#include "aether/clock.h" +#include "aether/config.h" + +namespace ae { + +// Runtime (non-wire) CloudRequest latency / retry / hedge policy. +// Orthogonal to RequestPolicy (which servers are candidates). +struct CloudRequestExecutionPolicy { + // Soft response timeout uses channel response RTT percentile. + std::uint8_t response_percentile{99}; + // Public semantic 1.2x stored as fixed-point permille (1200 => 1.2). + std::uint16_t timeout_factor_permille{1200}; + // Retries after the initial attempt. retry_count=0 => 1 attempt total. + std::uint8_t retry_count{1}; + // How many not-yet-activated following candidates to start on first soft miss. + std::uint8_t hedge_next_servers{0}; + + static constexpr CloudRequestExecutionPolicy Default() noexcept { + return CloudRequestExecutionPolicy{}; + } + + [[nodiscard]] std::size_t TotalAttempts() const noexcept { + return static_cast(retry_count) + 1; + } + + [[nodiscard]] double TimeoutFactor() const noexcept { + return static_cast(timeout_factor_permille) / 1000.0; + } + + static constexpr CloudRequestExecutionPolicy FromFactor( + std::uint8_t percentile, double factor, std::uint8_t retries, + std::uint8_t hedge) noexcept { + CloudRequestExecutionPolicy p{}; + p.response_percentile = percentile; + if (factor <= 0.0) { + p.timeout_factor_permille = 1000; + } else { + auto const scaled = factor * 1000.0 + 0.5; + if (scaled >= 65535.0) { + p.timeout_factor_permille = 65535; + } else { + p.timeout_factor_permille = static_cast(scaled); + } + } + p.retry_count = retries; + p.hedge_next_servers = hedge; + return p; + } +}; + +// T = round_nearest(rtt_ms * timeout_factor_permille / 1000). +inline Duration ScaleDurationByPermille(Duration base, + std::uint16_t factor_permille) noexcept { + using Ms = std::chrono::milliseconds; + auto const base_ms = + std::chrono::duration_cast(base).count(); + if (base_ms <= 0) { + return std::chrono::duration_cast(Ms{1}); + } + auto const product = + static_cast(base_ms) * + static_cast(factor_permille); + auto const scaled = (product + 500) / 1000; + if (scaled <= 0) { + return std::chrono::duration_cast(Ms{1}); + } + return std::chrono::duration_cast(Ms{scaled}); +} + +inline Duration ComputeCloudRequestSoftTimeout( + Duration rtt_percentile, + CloudRequestExecutionPolicy const& policy) noexcept { + return ScaleDurationByPermille(rtt_percentile, policy.timeout_factor_permille); +} + +inline Duration FallbackCloudRequestRtt() noexcept { + return std::chrono::duration_cast( + std::chrono::milliseconds{AE_DEFAULT_RESPONSE_TIMEOUT_MS}); +} + +// Per-server execution state used by CloudRequest (unit-testable). +struct CloudRequestServerExecState { + bool activated{false}; + bool succeeded{false}; + bool exhausted{false}; + bool first_soft_miss_seen{false}; + std::uint8_t attempts_started{0}; + std::uint8_t soft_timeouts{0}; + + [[nodiscard]] bool ShouldSkip() const noexcept { + return succeeded || exhausted || !activated; + } + + [[nodiscard]] bool CanStartAttempt( + CloudRequestExecutionPolicy const& policy) const noexcept { + if (!activated || succeeded || exhausted) { + return false; + } + return attempts_started < policy.TotalAttempts(); + } + + // Start a new attempt. Returns 1-based attempt index, or 0 if not allowed. + std::uint8_t StartAttempt(CloudRequestExecutionPolicy const& policy) { + if (!CanStartAttempt(policy)) { + return 0; + } + ++attempts_started; + return attempts_started; + } + + enum class SoftTimeoutAction : std::uint8_t { + kIgnore = 0, + kRetry, + kExhaust, + }; + + // Soft timeout for the current in-flight attempt. Does not Restream. + SoftTimeoutAction OnSoftTimeout(CloudRequestExecutionPolicy const& policy) { + if (succeeded || exhausted || !activated) { + return SoftTimeoutAction::kIgnore; + } + ++soft_timeouts; + bool const first_miss = !first_soft_miss_seen; + first_soft_miss_seen = true; + static_cast(first_miss); + if (attempts_started < policy.TotalAttempts()) { + return SoftTimeoutAction::kRetry; + } + exhausted = true; + return SoftTimeoutAction::kExhaust; + } + + // How many following candidates to activate because of this soft miss. + [[nodiscard]] std::uint8_t HedgeCountOnThisMiss( + CloudRequestExecutionPolicy const& policy) const noexcept { + // Hedge only on the first soft miss of this server. + if (soft_timeouts != 1) { + return 0; + } + return policy.hedge_next_servers; + } + + void MarkSucceeded() { + if (exhausted) { + return; + } + succeeded = true; + } + + void MarkExhausted() { exhausted = true; } +}; + +inline bool CloudRequestShouldFailAll(bool any_open, + bool any_succeeded) noexcept { + return !any_open && !any_succeeded; +} + +} // namespace ae + +#endif // AETHER_CLOUD_CONNECTIONS_CLOUD_REQUEST_EXECUTION_POLICY_H_ diff --git a/aether/cloud_connections/cloud_server_connections.cpp b/aether/cloud_connections/cloud_server_connections.cpp index 993eb898..1a4a76c4 100644 --- a/aether/cloud_connections/cloud_server_connections.cpp +++ b/aether/cloud_connections/cloud_server_connections.cpp @@ -126,6 +126,11 @@ void CloudServerConnections::Restream() { } } +void CloudServerConnections::QuarantineForNoResponse( + CloudServerConnection& server_connection) { + QuarantineAndReconcile(server_connection); +} + void CloudServerConnections::InitServerConnections() { auto cloud = cloud_.Lock(); assert(cloud && "cloud must outlive its connections"); diff --git a/aether/cloud_connections/cloud_server_connections.h b/aether/cloud_connections/cloud_server_connections.h index 7b7dc56f..025b1669 100644 --- a/aether/cloud_connections/cloud_server_connections.h +++ b/aether/cloud_connections/cloud_server_connections.h @@ -93,6 +93,12 @@ class CloudServerConnections { */ void Restream(); + /** + * \brief Quarantine a server after CloudRequest response-retry exhaustion. + * Uses the existing quarantine / reconcile / replacement path. + */ + void QuarantineForNoResponse(CloudServerConnection& server_connection); + /** * \brief Iterate over servers according to the request policy. * Calls func with CloudServerConnection* for each server. diff --git a/aether/remote_presence.h b/aether/remote_presence.h index cd037fa6..1538da21 100644 --- a/aether/remote_presence.h +++ b/aether/remote_presence.h @@ -60,8 +60,6 @@ struct RemoteServerPresenceSample { bool has_timing{false}; }; -inline constexpr std::size_t kRemotePresenceQueryRetryCount{1}; - inline TimePoint TimePointOffsetByMs(TimePoint anchor, std::int64_t delta_ms) noexcept { if (delta_ms == 0) { diff --git a/examples/remote_presence_live/remote_presence_live.cpp b/examples/remote_presence_live/remote_presence_live.cpp index 2ac98969..3d76dc9e 100644 --- a/examples/remote_presence_live/remote_presence_live.cpp +++ b/examples/remote_presence_live/remote_presence_live.cpp @@ -44,6 +44,7 @@ #include "aether/ae_actions/query_peer_presence.h" #include "aether/all.h" #include "aether/client_connectivity_policy.h" +#include "aether/cloud_connections/cloud_request_execution_policy.h" #include "aether/cloud_connections/local_presence_schedule.h" #include "aether/config.h" #include "aether/remote_presence.h" @@ -126,6 +127,9 @@ void ApplyTimings(Client& client) { } policy->ResetRxTimings(); policy->SetOfflineDetectionTimeout(kOfflineTimeout); + policy->SetCloudRequestExecutionPolicy( + CloudRequestExecutionPolicy::FromFactor(99, 1.2, /*retries=*/2, + /*hedge=*/2)); policy->ConfigureRxTimings(RequestPolicy::All{}) .ForAllPriorities(RxTimingConf::Every(kInterval).WithWindow(kWindow)); for (auto* server : client.cloud_connection().selected_servers()) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index aedd2dc6..7c080a34 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -47,5 +47,6 @@ add_subdirectory(test-tasks) add_subdirectory(test-server-connection) add_subdirectory(test-local-presence) +add_subdirectory(test-cloud-request) add_subdirectory(third_party_tests) diff --git a/tests/test-cloud-request/CMakeLists.txt b/tests/test-cloud-request/CMakeLists.txt new file mode 100644 index 00000000..2138a121 --- /dev/null +++ b/tests/test-cloud-request/CMakeLists.txt @@ -0,0 +1,29 @@ +# Copyright 2026 Aethernet Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.16) + +if(NOT CM_PLATFORM) + project(test-cloud-request LANGUAGES CXX) + + add_executable(${PROJECT_NAME} main.cpp) + target_include_directories(${PROJECT_NAME} PRIVATE ${ROOT_DIR}) + target_link_libraries(${PROJECT_NAME} PRIVATE unity aether) + if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(${PROJECT_NAME} PRIVATE /Zc:preprocessor) + endif() + add_test(NAME ${PROJECT_NAME} COMMAND $) +else() + message(WARNING "Not implemented for ${CM_PLATFORM}") +endif() diff --git a/tests/test-cloud-request/main.cpp b/tests/test-cloud-request/main.cpp new file mode 100644 index 00000000..80da08d4 --- /dev/null +++ b/tests/test-cloud-request/main.cpp @@ -0,0 +1,257 @@ +/* + * Copyright 2026 Aethernet Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "aether/cloud_connections/cloud_request_execution_policy.h" +#include "aether/types/statistic_counter.h" + +namespace ae::test_cloud_request { + +using Ms = std::chrono::milliseconds; + +void test_TimeoutCalculation() { + // Deterministic RTT samples: 50,100,150,200,250,300,350,400,450,500 + StatisticsCounter stats; + for (int i = 1; i <= 10; ++i) { + stats.Add(Duration{Ms{50 * i}}); + } + auto const p95 = stats.PercentileValue(95); + auto const p99 = stats.PercentileValue(99); + // index = ceil((10-1)*pct/100): p95 -> ceil(8.55)=9 -> 500? wait + // sorted 50..500, index ceil(9*0.95)=ceil(8.55)=9 -> value_buffer[9]=500 + // p99: ceil(9*0.99)=ceil(8.91)=9 -> 500 + TEST_ASSERT_EQUAL(500, std::chrono::duration_cast(p95).count()); + TEST_ASSERT_EQUAL(500, std::chrono::duration_cast(p99).count()); + + auto const t95_10 = + ComputeCloudRequestSoftTimeout(p95, CloudRequestExecutionPolicy::FromFactor( + 95, 1.0, 1, 0)); + auto const t95_12 = + ComputeCloudRequestSoftTimeout(p95, CloudRequestExecutionPolicy::FromFactor( + 95, 1.2, 1, 0)); + auto const t99_10 = + ComputeCloudRequestSoftTimeout(p99, CloudRequestExecutionPolicy::FromFactor( + 99, 1.0, 1, 0)); + auto const t99_12 = + ComputeCloudRequestSoftTimeout(p99, CloudRequestExecutionPolicy::FromFactor( + 99, 1.2, 1, 0)); + TEST_ASSERT_EQUAL(500, std::chrono::duration_cast(t95_10).count()); + TEST_ASSERT_EQUAL(600, std::chrono::duration_cast(t95_12).count()); + TEST_ASSERT_EQUAL(500, std::chrono::duration_cast(t99_10).count()); + TEST_ASSERT_EQUAL(600, std::chrono::duration_cast(t99_12).count()); + + // Rounding: 100ms * 1.2 = 120 exactly; 101 * 1.2 = 121.2 -> 121 + TEST_ASSERT_EQUAL( + 120, std::chrono::duration_cast( + ScaleDurationByPermille(Duration{Ms{100}}, 1200)) + .count()); + TEST_ASSERT_EQUAL( + 121, std::chrono::duration_cast( + ScaleDurationByPermille(Duration{Ms{101}}, 1200)) + .count()); +} + +void test_RetryCountSemantics() { + CloudRequestExecutionPolicy p0 = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 0, 0); + TEST_ASSERT_EQUAL(1, p0.TotalAttempts()); + CloudRequestServerExecState s0; + s0.activated = true; + TEST_ASSERT_EQUAL(1, s0.StartAttempt(p0)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kExhaust), + static_cast(s0.OnSoftTimeout(p0))); + TEST_ASSERT_TRUE(s0.exhausted); + + CloudRequestExecutionPolicy p1 = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 1, 0); + TEST_ASSERT_EQUAL(2, p1.TotalAttempts()); + CloudRequestServerExecState s1; + s1.activated = true; + TEST_ASSERT_EQUAL(1, s1.StartAttempt(p1)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(s1.OnSoftTimeout(p1))); + TEST_ASSERT_FALSE(s1.exhausted); + TEST_ASSERT_EQUAL(2, s1.StartAttempt(p1)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kExhaust), + static_cast(s1.OnSoftTimeout(p1))); + TEST_ASSERT_TRUE(s1.exhausted); + + CloudRequestExecutionPolicy p2 = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 0); + TEST_ASSERT_EQUAL(3, p2.TotalAttempts()); + CloudRequestServerExecState s2; + s2.activated = true; + TEST_ASSERT_EQUAL(1, s2.StartAttempt(p2)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(s2.OnSoftTimeout(p2))); + TEST_ASSERT_EQUAL(2, s2.StartAttempt(p2)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(s2.OnSoftTimeout(p2))); + TEST_ASSERT_EQUAL(3, s2.StartAttempt(p2)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kExhaust), + static_cast(s2.OnSoftTimeout(p2))); + TEST_ASSERT_EQUAL(3, s2.attempts_started); + TEST_ASSERT_EQUAL(3, s2.soft_timeouts); +} + +void test_NoQuarantineBeforeExhaustionAndHedge() { + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 2); + CloudRequestServerExecState s; + s.activated = true; + TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); + auto const a1 = s.OnSoftTimeout(policy); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(a1)); + TEST_ASSERT_FALSE(s.exhausted); + TEST_ASSERT_EQUAL(2, s.HedgeCountOnThisMiss(policy)); + + TEST_ASSERT_EQUAL(2, s.StartAttempt(policy)); + auto const a2 = s.OnSoftTimeout(policy); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(a2)); + TEST_ASSERT_EQUAL(0, s.HedgeCountOnThisMiss(policy)); // only first miss + TEST_ASSERT_FALSE(s.exhausted); + + TEST_ASSERT_EQUAL(3, s.StartAttempt(policy)); + auto const a3 = s.OnSoftTimeout(policy); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kExhaust), + static_cast(a3)); + TEST_ASSERT_TRUE(s.exhausted); +} + +void test_HedgeZeroKeepsSequential() { + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 0); + CloudRequestServerExecState s1; + s1.activated = true; + s1.StartAttempt(policy); + s1.OnSoftTimeout(policy); + TEST_ASSERT_EQUAL(0, s1.HedgeCountOnThisMiss(policy)); +} + +void test_LateResponseAfterSoftTimeout() { + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 0); + CloudRequestServerExecState s; + s.activated = true; + TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(s.OnSoftTimeout(policy))); + TEST_ASSERT_EQUAL(2, s.StartAttempt(policy)); + // Late success for attempt #1 — mark succeeded, no further attempts / exhaust. + s.MarkSucceeded(); + TEST_ASSERT_TRUE(s.succeeded); + TEST_ASSERT_FALSE(s.exhausted); + TEST_ASSERT_FALSE(s.CanStartAttempt(policy)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kIgnore), + static_cast(s.OnSoftTimeout(policy))); +} + +void test_PerServerTimeoutIndependent() { + auto const t1 = ComputeCloudRequestSoftTimeout( + Duration{Ms{100}}, + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 1, 0)); + auto const t2 = ComputeCloudRequestSoftTimeout( + Duration{Ms{300}}, + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 1, 0)); + TEST_ASSERT_EQUAL(120, std::chrono::duration_cast(t1).count()); + TEST_ASSERT_EQUAL(360, std::chrono::duration_cast(t2).count()); +} + +void test_PolicySnapshotDefaults() { + auto const d = CloudRequestExecutionPolicy::Default(); + TEST_ASSERT_EQUAL(99, d.response_percentile); + TEST_ASSERT_EQUAL(1200, d.timeout_factor_permille); + TEST_ASSERT_EQUAL(1, d.retry_count); + TEST_ASSERT_EQUAL(0, d.hedge_next_servers); + TEST_ASSERT_EQUAL(2, d.TotalAttempts()); +} + +void test_DeterministicLatencyTimeline() { + // p99=100ms, factor=1.2 => T=120ms per attempt when RTT fixed. + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 1); + auto const T = + ComputeCloudRequestSoftTimeout(Duration{Ms{100}}, policy); + TEST_ASSERT_EQUAL(120, std::chrono::duration_cast(T).count()); + + // Case A: response at 110 < 120 => no soft miss conceptually (timer cancelled). + // Case B: soft miss at 120 => retry + hedge; late at 130 accepted. + CloudRequestServerExecState s; + s.activated = true; + s.StartAttempt(policy); + auto const miss = s.OnSoftTimeout(policy); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(miss)); + TEST_ASSERT_EQUAL(1, s.HedgeCountOnThisMiss(policy)); + s.StartAttempt(policy); + s.MarkSucceeded(); // late response from attempt #1 + TEST_ASSERT_TRUE(s.succeeded); + TEST_ASSERT_FALSE(s.exhausted); + + // Case C: no responses, retry_count=2 => three timeouts then exhaust. + CloudRequestServerExecState never; + never.activated = true; + std::int64_t t_ms = 0; + never.StartAttempt(policy); + t_ms += 120; + never.OnSoftTimeout(policy); + never.StartAttempt(policy); + t_ms += 120; + never.OnSoftTimeout(policy); + never.StartAttempt(policy); + t_ms += 120; + never.OnSoftTimeout(policy); + TEST_ASSERT_TRUE(never.exhausted); + TEST_ASSERT_EQUAL(360, t_ms); + TEST_ASSERT_EQUAL(3, never.attempts_started); +} + +} // namespace ae::test_cloud_request + +extern "C" void setUp(void) {} +extern "C" void tearDown(void) {} + +int main() { + UNITY_BEGIN(); + RUN_TEST(ae::test_cloud_request::test_TimeoutCalculation); + RUN_TEST(ae::test_cloud_request::test_RetryCountSemantics); + RUN_TEST(ae::test_cloud_request::test_NoQuarantineBeforeExhaustionAndHedge); + RUN_TEST(ae::test_cloud_request::test_HedgeZeroKeepsSequential); + RUN_TEST(ae::test_cloud_request::test_LateResponseAfterSoftTimeout); + RUN_TEST(ae::test_cloud_request::test_PerServerTimeoutIndependent); + RUN_TEST(ae::test_cloud_request::test_PolicySnapshotDefaults); + RUN_TEST(ae::test_cloud_request::test_DeterministicLatencyTimeline); + return UNITY_END(); +} From 60f7e42866f147c843342731d2453f631d067cb3 Mon Sep 17 00:00:00 2001 From: aethernet-io Date: Wed, 2 Sep 2026 20:45:49 -0700 Subject: [PATCH 07/11] Fix CloudRequestExecutionPolicy edge cases before merge. Cap retry_count to 0..31, subscribe once per server to channel_changed, and treat authenticated API errors as alive (no no-response quarantine). Co-authored-by: Cursor --- aether/ae_actions/query_peer_presence.cpp | 13 +- aether/client_connectivity_policy.cpp | 7 +- aether/cloud_connections/cloud_request.cpp | 140 +++++++-------- aether/cloud_connections/cloud_request.h | 23 ++- .../cloud_request_execution_policy.h | 69 ++++++- tests/test-cloud-request/main.cpp | 168 ++++++++++++++++++ 6 files changed, 320 insertions(+), 100 deletions(-) diff --git a/aether/ae_actions/query_peer_presence.cpp b/aether/ae_actions/query_peer_presence.cpp index c9b6c1eb..ac7d0afc 100644 --- a/aether/ae_actions/query_peer_presence.cpp +++ b/aether/ae_actions/query_peer_presence.cpp @@ -322,16 +322,13 @@ void QueryPeerPresence::OnServerTiming( meta_it->second.send_times.erase(send_it); if (!res) { + // Authenticated API error: server is alive — do not use no-response + // FailAttempt / quarantine path. if (cloud_request_.has_value()) { - auto const exhausted = cloud_request_->FailAttempt(sc); - if (exhausted) { - MarkUnknown(server_id); - MaybeComplete(); - } - } else { - MarkUnknown(server_id); - MaybeComplete(); + cloud_request_->CompleteAttemptWithRemoteError(sc); } + MarkUnknown(server_id); + MaybeComplete(); return; } diff --git a/aether/client_connectivity_policy.cpp b/aether/client_connectivity_policy.cpp index c5396027..b0a365a8 100644 --- a/aether/client_connectivity_policy.cpp +++ b/aether/client_connectivity_policy.cpp @@ -248,12 +248,7 @@ void ClientConnectivityPolicy::SetOfflineDetectionTimeout( void ClientConnectivityPolicy::SetCloudRequestExecutionPolicy( CloudRequestExecutionPolicy policy) noexcept { - if (policy.response_percentile > 100) { - policy.response_percentile = 100; - } - if (policy.timeout_factor_permille == 0) { - policy.timeout_factor_permille = 1000; - } + NormalizeCloudRequestExecutionPolicy(policy); cloud_request_execution_policy_ = policy; } diff --git a/aether/cloud_connections/cloud_request.cpp b/aether/cloud_connections/cloud_request.cpp index 5eca7da6..b90a7b37 100644 --- a/aether/cloud_connections/cloud_request.cpp +++ b/aether/cloud_connections/cloud_request.cpp @@ -55,6 +55,7 @@ CloudRequest::CloudRequest(AeContext const& ae_context, exec_policy_{exec_policy}, server_changed_sub_{cloud_scs_->servers_update_event().Subscribe( MethodPtr<&CloudRequest::ServersUpdated>{this})} { + NormalizeCloudRequestExecutionPolicy(exec_policy_); AE_CLOUD_REQ_DEBUG( "CLOUD_REQUEST_START percentile={} factor_permille={} retry_count={} " "hedge_next_servers={}", @@ -77,6 +78,7 @@ CloudRequest::CloudRequest(AeContext const& ae_context, exec_policy_{exec_policy}, server_changed_sub_{cloud_scs_->servers_update_event().Subscribe( MethodPtr<&CloudRequest::ServersUpdated>{this})} { + NormalizeCloudRequestExecutionPolicy(exec_policy_); AE_CLOUD_REQ_DEBUG( "CLOUD_REQUEST_START percentile={} factor_permille={} retry_count={} " "hedge_next_servers={}", @@ -97,56 +99,47 @@ void CloudRequest::Failed() { result_event_.Emit(false); } +void CloudRequest::StopServerTimers(ServerRequest& sr) { + for (auto& attempt : sr.attempts) { + attempt.timeout_sub.Reset(); + } + sr.channel_changed_sub.Reset(); +} + void CloudRequest::SucceedAttempt(CloudServerConnection* sc) { auto it = server_requests_.find(sc); if (it == server_requests_.end()) { return; } auto& sr = it->second; - if (sr.exec.succeeded || sr.exec.exhausted) { + if (sr.exec.IsTerminal()) { return; } AE_CLOUD_REQ_DEBUG("SERVER_ATTEMPT_SUCCESS server_id={}", sc->server_id()); - // Cancel future soft timers; keep response_subs so in-flight late responses - // can still be observed by the listener if needed, but mark success first. - for (auto& attempt : sr.attempts) { - attempt.timeout_sub.Reset(); - } + StopServerTimers(sr); sr.exec.MarkSucceeded(); - // Sequential hedge=0 style: after success, activate the next candidate if - // RequestPolicy still requires more servers. ActivateFollowing(1); EnqueuePump(); } -bool CloudRequest::FailAttempt(CloudServerConnection* sc) { +void CloudRequest::CompleteAttemptWithRemoteError(CloudServerConnection* sc) { auto it = server_requests_.find(sc); if (it == server_requests_.end()) { - return false; + return; } auto& sr = it->second; - if (sr.exec.succeeded || sr.exec.exhausted) { - return false; - } - // Treat application-reported failure like a soft miss for budget purposes: - // do not Restream; retry if budget remains; otherwise quarantine. - bool const first_soft_miss = !sr.exec.first_soft_miss_seen; - auto const action = sr.exec.OnSoftTimeout(exec_policy_); - if (first_soft_miss && exec_policy_.hedge_next_servers > 0) { - ActivateFollowing(exec_policy_.hedge_next_servers, /*as_hedge=*/true, sc); - } - if (action == CloudRequestServerExecState::SoftTimeoutAction::kRetry) { - LaunchAttempt(sc, sr); - EnqueuePump(); - return false; - } - if (action == CloudRequestServerExecState::SoftTimeoutAction::kExhaust) { - ExhaustServerNoResponse(sc, sr); - EnqueuePump(); - return true; + if (sr.exec.IsTerminal()) { + return; } + // Authenticated API error: server proved it is alive. Do not treat as + // no-response, do not soft-retry, do not quarantine for timeout policy. + AE_CLOUD_REQ_DEBUG( + "SERVER_REMOTE_API_ERROR server_id={} (no no-response quarantine)", + sc->server_id()); + StopServerTimers(sr); + sr.exec.MarkRemoteErrorCompleted(); + ActivateFollowing(1); EnqueuePump(); - return sr.exec.exhausted; } CloudRequest::ResultEvent::Subscriber CloudRequest::result_event() { @@ -166,8 +159,6 @@ void CloudRequest::RebuildCandidates() { std::vector next; cloud_scs_->ForServers([&](CloudServerConnection* sc) { next.push_back(sc); }, policy_); - // Preserve activation cursor semantics: append newly eligible servers after - // existing candidate order; keep already-known pointers stable. for (auto* sc : next) { auto const known = std::find(candidates_.begin(), candidates_.end(), sc) != @@ -191,7 +182,7 @@ void CloudRequest::ActivateFollowing(std::uint8_t count, bool as_hedge, while (count > 0 && activate_cursor_ < candidates_.size()) { auto* sc = candidates_[activate_cursor_++]; auto& sr = server_requests_[sc]; - if (sr.exec.activated || sr.exec.succeeded || sr.exec.exhausted) { + if (sr.exec.activated || sr.exec.IsTerminal()) { continue; } if (as_hedge) { @@ -210,9 +201,27 @@ void CloudRequest::ActivateServer(CloudServerConnection* sc) { return; } sr.exec.activated = true; + EnsureChannelChangedSubscription(sc, sr); LaunchAttempt(sc, sr); } +void CloudRequest::EnsureChannelChangedSubscription(CloudServerConnection* sc, + ServerRequest& sr) { + if (sr.channel_changed_sub) { + return; + } + auto* conn = sc->client_connection(); + if (conn == nullptr) { + return; + } + sr.channel_changed_sub = + conn->server_connection().channel_changed_event().Subscribe([this, sc]() { + AE_CLOUD_REQ_WARNING("Request server channel changed {}", + sc->server_id()); + OnChannelChanged(sc); + }); +} + Duration CloudRequest::SoftTimeoutFor(CloudServerConnection* sc) const { auto* conn = sc->client_connection(); if (conn == nullptr) { @@ -246,20 +255,19 @@ void CloudRequest::LaunchAttempt(CloudServerConnection* sc, if (conn == nullptr) { AE_CLOUD_REQ_WARNING("SERVER_ATTEMPT skipped disconnected server {}", sc->server_id()); - // Hard unusable: exhaust and quarantine via existing health path if link - // errors; for a missing connection treat as attempt failure. auto const action = sr.exec.OnSoftTimeout(exec_policy_); if (action == CloudRequestServerExecState::SoftTimeoutAction::kExhaust) { ExhaustServerNoResponse(sc, sr); } else if (action == CloudRequestServerExecState::SoftTimeoutAction::kRetry) { - // Will retry on next pump when connection appears. sr.exec.attempts_started = static_cast(sr.exec.attempts_started - 1); } return; } + EnsureChannelChangedSubscription(sc, sr); + auto const timeout = SoftTimeoutFor(sc); AE_CLOUD_REQ_DEBUG( "SERVER_ATTEMPT server_id={} attempt_index={} timeout_ms={}", @@ -286,26 +294,19 @@ void CloudRequest::LaunchAttempt(CloudServerConnection* sc, }, request_); - // Write failure is a hard-ish send problem — count toward budget without - // Restream from soft timeout path. - attempt.subs += swa.status_event().Subscribe([this, sc](auto status) { + // Write failure: send may not have reached the server. Reuse soft retry + // budget (no Restream). LinkError quarantine remains on the connection + // health path and is not duplicated here beyond ExhaustServerNoResponse. + attempt.write_subs += swa.status_event().Subscribe([this, sc](auto status) { if (status == WriteAction::Status::kFail) { AE_CLOUD_REQ_WARNING("Request write error server {}", sc->server_id()); OnWriteFailed(sc); } }); - attempt.subs += - conn->server_connection().channel_changed_event().Subscribe([this, sc]() { - AE_CLOUD_REQ_WARNING("Request server channel changed {}", - sc->server_id()); - OnChannelChanged(sc); - }); if (std::holds_alternative(request_)) { auto& listener = std::get(request_).listener; if (listener) { - // Keep listener subscriptions durable so late responses from earlier - // attempts are not destroyed when a retry is launched. sr.response_subs += listener(conn->client_safe_api(), sc, this); } } @@ -324,7 +325,7 @@ void CloudRequest::OnSoftTimeout(CloudServerConnection* sc, return; } auto& sr = it->second; - if (sr.exec.succeeded || sr.exec.exhausted) { + if (sr.exec.IsTerminal()) { return; } @@ -361,21 +362,17 @@ void CloudRequest::OnSoftTimeout(CloudServerConnection* sc, void CloudRequest::ExhaustServerNoResponse(CloudServerConnection* sc, ServerRequest& sr) { - if (sr.exec.succeeded) { + if (sr.exec.succeeded || sr.exec.remote_error_completed) { return; } sr.exec.MarkExhausted(); - for (auto& attempt : sr.attempts) { - attempt.timeout_sub.Reset(); - } + StopServerTimers(sr); AE_CLOUD_REQ_WARNING( - "SERVER_RETRY_EXHAUSTED server_id={} attempts={} -> " + "SERVER_RETRY_EXHAUSTED server_id={} attempts={} soft_timeouts={} -> " "SERVER_QUARANTINE_NO_RESPONSE", - sc->server_id(), sr.exec.attempts_started); + sc->server_id(), sr.exec.attempts_started, sr.exec.soft_timeouts); cloud_scs_->QuarantineForNoResponse(*sc); EmitAttemptExhausted(sc); - // After quarantine/reconcile, ServersUpdated may add a replacement — also - // advance sequential activation for remaining required candidates. ActivateFollowing(1); } @@ -385,16 +382,18 @@ void CloudRequest::OnChannelChanged(CloudServerConnection* sc) { return; } auto& sr = it->second; - if (sr.exec.succeeded || sr.exec.exhausted || !sr.exec.activated) { + auto const action = sr.exec.OnChannelChanged(exec_policy_); + if (action == + CloudRequestServerExecState::ChannelChangedAction::kIgnore) { return; } - // Channel change is infrastructure: re-send without Restream-from-soft-timeout, - // using remaining attempt budget if available. - if (!sr.exec.CanStartAttempt(exec_policy_)) { + if (action == + CloudRequestServerExecState::ChannelChangedAction::kExhaust) { ExhaustServerNoResponse(sc, sr); EnqueuePump(); return; } + // Exactly one LaunchAttempt per channel-changed event. LaunchAttempt(sc, sr); EnqueuePump(); } @@ -405,11 +404,13 @@ void CloudRequest::OnWriteFailed(CloudServerConnection* sc) { return; } auto& sr = it->second; - if (sr.exec.succeeded || sr.exec.exhausted) { + if (sr.exec.IsTerminal()) { return; } - // Write failure: do not soft-Restream; apply retry budget. Hard LinkError - // quarantine remains on the connection health path. + // WriteAction::kFail: treat as send-path failure using soft retry budget + // (no Restream). Hard LinkError quarantine is owned by CloudServerConnections + // stream/error subscriptions — ExhaustServerNoResponse may also quarantine + // after budget exhaustion if the write failures never produced a response. bool const first_soft_miss = !sr.exec.first_soft_miss_seen; auto const action = sr.exec.OnSoftTimeout(exec_policy_); if (first_soft_miss && exec_policy_.hedge_next_servers > 0) { @@ -426,14 +427,10 @@ void CloudRequest::OnWriteFailed(CloudServerConnection* sc) { void CloudRequest::ServersUpdated() { RebuildCandidates(); - // If nothing is active yet, start the first candidate. Otherwise activate - // newly appended replacements only when sequential policy needs them or - // hedge already opened the window — ActivateFollowing(1) after exhaustion - // handles the common replacement case. bool any_activated = false; for (auto const& [sc, sr] : server_requests_) { static_cast(sc); - if (sr.exec.activated && !sr.exec.exhausted && !sr.exec.succeeded) { + if (sr.exec.activated && !sr.exec.IsTerminal()) { any_activated = true; break; } @@ -441,7 +438,6 @@ void CloudRequest::ServersUpdated() { if (!any_activated) { ActivateInitial(); } else { - // Ensure replacements beyond the cursor can be pulled in for All. ActivateFollowing(1); } EnqueuePump(); @@ -464,14 +460,12 @@ void CloudRequest::Pump() { static_cast(sc); if (sr.exec.succeeded) { any_succeeded = true; - } else if (sr.exec.activated && !sr.exec.exhausted) { + } else if (sr.exec.activated && !sr.exec.IsTerminal()) { any_open = true; - } else if (!sr.exec.activated && !sr.exec.exhausted) { - // Candidate not yet activated — still open for sequential/hedge. + } else if (!sr.exec.activated && !sr.exec.IsTerminal()) { any_open = true; } } - // Candidates reserved but not activated still count as open work. if (activate_cursor_ < candidates_.size()) { any_open = true; } diff --git a/aether/cloud_connections/cloud_request.h b/aether/cloud_connections/cloud_request.h index 6b1b0506..a40b4915 100644 --- a/aether/cloud_connections/cloud_request.h +++ b/aether/cloud_connections/cloud_request.h @@ -28,6 +28,7 @@ #include "aether/cloud_connections/request_policy.h" #include "aether/cloud_connections/cloud_callbacks.h" #include "aether/cloud_connections/cloud_server_connections.h" +#include "aether/events/event_subscription.h" #include "aether/events/multi_subscription.h" namespace ae { @@ -39,16 +40,21 @@ namespace ae { * no-response happens only after the per-server retry budget is exhausted. * Late valid responses from earlier attempts are accepted. * + * Authenticated API-level errors (CompleteAttemptWithRemoteError) prove the + * server is alive and must not quarantine via the no-response path. + * * ResponseSubscriber / ApiRequestHandler must handle responses. On whole * request success, call CloudRequest::Succeeded(); on whole failure, - * CloudRequest::Failed(). Per-server: SucceedAttempt / FailAttempt. + * CloudRequest::Failed(). Per-server: SucceedAttempt / + * CompleteAttemptWithRemoteError. */ // Compatibility alias for older unit helpers. using CloudRequestAttemptState = CloudRequestServerExecState; class CloudRequest final : public Action { struct AttemptState { - MultiSubscription subs; + // Write-status subscription for this attempt only. + MultiSubscription write_subs; TaskSubscription timeout_sub; std::uint8_t attempt_index{0}; bool timed_out{false}; @@ -58,6 +64,9 @@ class CloudRequest final : public Action { CloudRequestServerExecState exec{}; // Durable across attempts so late responses remain deliverable. MultiSubscription response_subs; + // One channel_changed subscription for the whole server lifetime in this + // CloudRequest ? not per attempt. + Subscription channel_changed_sub; std::vector attempts; }; @@ -84,10 +93,9 @@ class CloudRequest final : public Action { // Per-server success: accept late responses, cancel future retries, no // Restream / quarantine. void SucceedAttempt(CloudServerConnection* sc); - // Listener-side attempt failure (e.g. API error). Soft retry budget applies. - // Returns true when the server is exhausted (and quarantined for no usable - // response path). - bool FailAttempt(CloudServerConnection* sc); + // Valid authenticated response with API-level failure. Server is alive: + // no soft-timeout retry budget, no no-response quarantine. + void CompleteAttemptWithRemoteError(CloudServerConnection* sc); CloudRequestExecutionPolicy const& execution_policy() const noexcept { return exec_policy_; @@ -102,7 +110,10 @@ class CloudRequest final : public Action { void ActivateFollowing(std::uint8_t count, bool as_hedge = false, CloudServerConnection* source = nullptr); void ActivateServer(CloudServerConnection* sc); + void EnsureChannelChangedSubscription(CloudServerConnection* sc, + ServerRequest& sr); void LaunchAttempt(CloudServerConnection* sc, ServerRequest& sr); + void StopServerTimers(ServerRequest& sr); Duration SoftTimeoutFor(CloudServerConnection* sc) const; void OnSoftTimeout(CloudServerConnection* sc, std::uint8_t attempt_index); diff --git a/aether/cloud_connections/cloud_request_execution_policy.h b/aether/cloud_connections/cloud_request_execution_policy.h index 467c0bf7..30e41366 100644 --- a/aether/cloud_connections/cloud_request_execution_policy.h +++ b/aether/cloud_connections/cloud_request_execution_policy.h @@ -27,6 +27,10 @@ namespace ae { +// Max retries after the initial attempt for one CloudRequest ↔ one server. +// Total attempts = 1 + retry_count ∈ [1, 32]. +inline constexpr std::uint8_t kMaxCloudRequestRetryCount{31}; + // Runtime (non-wire) CloudRequest latency / retry / hedge policy. // Orthogonal to RequestPolicy (which servers are candidates). struct CloudRequestExecutionPolicy { @@ -35,6 +39,7 @@ struct CloudRequestExecutionPolicy { // Public semantic 1.2x stored as fixed-point permille (1200 => 1.2). std::uint16_t timeout_factor_permille{1200}; // Retries after the initial attempt. retry_count=0 => 1 attempt total. + // Clamped to [0, kMaxCloudRequestRetryCount]. std::uint8_t retry_count{1}; // How many not-yet-activated following candidates to start on first soft miss. std::uint8_t hedge_next_servers{0}; @@ -66,12 +71,27 @@ struct CloudRequestExecutionPolicy { p.timeout_factor_permille = static_cast(scaled); } } - p.retry_count = retries; + p.retry_count = retries > kMaxCloudRequestRetryCount + ? kMaxCloudRequestRetryCount + : retries; p.hedge_next_servers = hedge; return p; } }; +inline void NormalizeCloudRequestExecutionPolicy( + CloudRequestExecutionPolicy& policy) noexcept { + if (policy.response_percentile > 100) { + policy.response_percentile = 100; + } + if (policy.timeout_factor_permille == 0) { + policy.timeout_factor_permille = 1000; + } + if (policy.retry_count > kMaxCloudRequestRetryCount) { + policy.retry_count = kMaxCloudRequestRetryCount; + } +} + // T = round_nearest(rtt_ms * timeout_factor_permille / 1000). inline Duration ScaleDurationByPermille(Duration base, std::uint16_t factor_permille) noexcept { @@ -107,17 +127,26 @@ struct CloudRequestServerExecState { bool activated{false}; bool succeeded{false}; bool exhausted{false}; + // Authenticated API-level error: server is alive; attempt terminal; no + // no-response quarantine. + bool remote_error_completed{false}; bool first_soft_miss_seen{false}; std::uint8_t attempts_started{0}; std::uint8_t soft_timeouts{0}; + // Counts OnChannelChanged decisions (one event → one increment). + std::uint8_t channel_changed_events{0}; + + [[nodiscard]] bool IsTerminal() const noexcept { + return succeeded || exhausted || remote_error_completed; + } [[nodiscard]] bool ShouldSkip() const noexcept { - return succeeded || exhausted || !activated; + return IsTerminal() || !activated; } [[nodiscard]] bool CanStartAttempt( CloudRequestExecutionPolicy const& policy) const noexcept { - if (!activated || succeeded || exhausted) { + if (!activated || IsTerminal()) { return false; } return attempts_started < policy.TotalAttempts(); @@ -140,13 +169,11 @@ struct CloudRequestServerExecState { // Soft timeout for the current in-flight attempt. Does not Restream. SoftTimeoutAction OnSoftTimeout(CloudRequestExecutionPolicy const& policy) { - if (succeeded || exhausted || !activated) { + if (IsTerminal() || !activated) { return SoftTimeoutAction::kIgnore; } ++soft_timeouts; - bool const first_miss = !first_soft_miss_seen; first_soft_miss_seen = true; - static_cast(first_miss); if (attempts_started < policy.TotalAttempts()) { return SoftTimeoutAction::kRetry; } @@ -154,6 +181,26 @@ struct CloudRequestServerExecState { return SoftTimeoutAction::kExhaust; } + enum class ChannelChangedAction : std::uint8_t { + kIgnore = 0, + kRetry, + kExhaust, + }; + + // One channel-changed event → at most one LaunchAttempt (or exhaust). + ChannelChangedAction OnChannelChanged( + CloudRequestExecutionPolicy const& policy) { + if (IsTerminal() || !activated) { + return ChannelChangedAction::kIgnore; + } + ++channel_changed_events; + if (!CanStartAttempt(policy)) { + exhausted = true; + return ChannelChangedAction::kExhaust; + } + return ChannelChangedAction::kRetry; + } + // How many following candidates to activate because of this soft miss. [[nodiscard]] std::uint8_t HedgeCountOnThisMiss( CloudRequestExecutionPolicy const& policy) const noexcept { @@ -165,12 +212,20 @@ struct CloudRequestServerExecState { } void MarkSucceeded() { - if (exhausted) { + if (exhausted || remote_error_completed) { return; } succeeded = true; } + // Valid authenticated response with API-level failure — server is alive. + void MarkRemoteErrorCompleted() { + if (succeeded || exhausted) { + return; + } + remote_error_completed = true; + } + void MarkExhausted() { exhausted = true; } }; diff --git a/tests/test-cloud-request/main.cpp b/tests/test-cloud-request/main.cpp index 80da08d4..fd18be3b 100644 --- a/tests/test-cloud-request/main.cpp +++ b/tests/test-cloud-request/main.cpp @@ -197,6 +197,168 @@ void test_PolicySnapshotDefaults() { TEST_ASSERT_EQUAL(2, d.TotalAttempts()); } +void test_RetryCountClampAndMax() { + TEST_ASSERT_EQUAL(31, kMaxCloudRequestRetryCount); + + CloudRequestExecutionPolicy p0 = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 0, 0); + TEST_ASSERT_EQUAL(0, p0.retry_count); + TEST_ASSERT_EQUAL(1, p0.TotalAttempts()); + + CloudRequestExecutionPolicy p1 = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 1, 0); + TEST_ASSERT_EQUAL(1, p1.retry_count); + TEST_ASSERT_EQUAL(2, p1.TotalAttempts()); + + CloudRequestExecutionPolicy p31 = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 31, 0); + TEST_ASSERT_EQUAL(31, p31.retry_count); + TEST_ASSERT_EQUAL(32, p31.TotalAttempts()); + + CloudRequestExecutionPolicy over{}; + over.retry_count = 255; + NormalizeCloudRequestExecutionPolicy(over); + TEST_ASSERT_EQUAL(31, over.retry_count); + TEST_ASSERT_EQUAL(32, over.TotalAttempts()); + + auto const from_over = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 255, 0); + TEST_ASSERT_EQUAL(31, from_over.retry_count); + TEST_ASSERT_EQUAL(32, from_over.TotalAttempts()); +} + +void test_RetryCount31StateMachine() { + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 31, 0); + TEST_ASSERT_EQUAL(32, policy.TotalAttempts()); + + CloudRequestServerExecState s; + s.activated = true; + for (int i = 0; i < 31; ++i) { + TEST_ASSERT_TRUE(s.CanStartAttempt(policy)); + TEST_ASSERT_EQUAL(i + 1, s.StartAttempt(policy)); + auto const action = s.OnSoftTimeout(policy); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(action)); + TEST_ASSERT_FALSE(s.exhausted); + } + TEST_ASSERT_EQUAL(31, s.attempts_started); + TEST_ASSERT_EQUAL(31, s.soft_timeouts); + TEST_ASSERT_EQUAL(32, s.StartAttempt(policy)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kExhaust), + static_cast(s.OnSoftTimeout(policy))); + TEST_ASSERT_TRUE(s.exhausted); + TEST_ASSERT_EQUAL(32, s.attempts_started); + TEST_ASSERT_EQUAL(32, s.soft_timeouts); + TEST_ASSERT_EQUAL(0, s.StartAttempt(policy)); // no uint8 wrap / extra +} + +void test_ChannelChangedOneCallbackPerServer() { + // retry_count=2: after attempt #1 soft timeout and attempt #2 started, + // one channel-changed event must produce exactly one OnChannelChanged + // decision and at most one additional attempt. + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 0); + CloudRequestServerExecState s; + s.activated = true; + TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kRetry), + static_cast(s.OnSoftTimeout(policy))); + TEST_ASSERT_EQUAL(2, s.StartAttempt(policy)); + // Simulate two outstanding attempts (#1 timed out late-response possible, + // #2 active) — still one channel-changed subscription / one callback. + auto const a = s.OnChannelChanged(policy); + TEST_ASSERT_EQUAL(1, s.channel_changed_events); + TEST_ASSERT_EQUAL( + static_cast( + CloudRequestServerExecState::ChannelChangedAction::kRetry), + static_cast(a)); + TEST_ASSERT_EQUAL(3, s.StartAttempt(policy)); + TEST_ASSERT_FALSE(s.exhausted); + // Budget exhausted: further channel change must not start more attempts. + auto const b = s.OnChannelChanged(policy); + TEST_ASSERT_EQUAL(2, s.channel_changed_events); + TEST_ASSERT_EQUAL( + static_cast( + CloudRequestServerExecState::ChannelChangedAction::kExhaust), + static_cast(b)); + TEST_ASSERT_TRUE(s.exhausted); + TEST_ASSERT_EQUAL(3, s.attempts_started); +} + +void test_ChannelChangedThreeOutstandingAttempts() { + // Three outstanding attempts (retry_count=2, all started via soft path / + // channel), then one channel event must still be a single decision. + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 0); + CloudRequestServerExecState s; + s.activated = true; + s.StartAttempt(policy); // #1 + s.OnSoftTimeout(policy); + s.StartAttempt(policy); // #2 + s.OnSoftTimeout(policy); + s.StartAttempt(policy); // #3 — budget full, three outstanding conceptually + TEST_ASSERT_EQUAL(3, s.attempts_started); + TEST_ASSERT_FALSE(s.CanStartAttempt(policy)); + + auto const a = s.OnChannelChanged(policy); + TEST_ASSERT_EQUAL(1, s.channel_changed_events); + TEST_ASSERT_EQUAL( + static_cast( + CloudRequestServerExecState::ChannelChangedAction::kExhaust), + static_cast(a)); + TEST_ASSERT_TRUE(s.exhausted); + TEST_ASSERT_EQUAL(3, s.attempts_started); // no extra launch + TEST_ASSERT_EQUAL(0, s.StartAttempt(policy)); +} + +void test_ApiErrorDoesNotQuarantine() { + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 0); + CloudRequestServerExecState s; + s.activated = true; + TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); + s.MarkRemoteErrorCompleted(); + TEST_ASSERT_TRUE(s.remote_error_completed); + TEST_ASSERT_TRUE(s.IsTerminal()); + TEST_ASSERT_FALSE(s.exhausted); + TEST_ASSERT_FALSE(s.succeeded); + TEST_ASSERT_EQUAL(0, s.soft_timeouts); + TEST_ASSERT_FALSE(s.CanStartAttempt(policy)); + TEST_ASSERT_EQUAL( + static_cast(CloudRequestServerExecState::SoftTimeoutAction::kIgnore), + static_cast(s.OnSoftTimeout(policy))); + TEST_ASSERT_EQUAL( + static_cast( + CloudRequestServerExecState::ChannelChangedAction::kIgnore), + static_cast(s.OnChannelChanged(policy))); + TEST_ASSERT_EQUAL(0, s.channel_changed_events); + TEST_ASSERT_EQUAL(0, s.soft_timeouts); + TEST_ASSERT_FALSE(s.exhausted); // no no-response quarantine path +} + +void test_NoResponseStillQuarantinesAfterBudget() { + // retry_count=2 => attempts=3 soft timeouts then exhaust (=quarantine point). + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 0); + CloudRequestServerExecState s; + s.activated = true; + s.StartAttempt(policy); + s.OnSoftTimeout(policy); + TEST_ASSERT_FALSE(s.exhausted); + s.StartAttempt(policy); + s.OnSoftTimeout(policy); + TEST_ASSERT_FALSE(s.exhausted); + s.StartAttempt(policy); + s.OnSoftTimeout(policy); + TEST_ASSERT_TRUE(s.exhausted); + TEST_ASSERT_EQUAL(3, s.attempts_started); + TEST_ASSERT_EQUAL(3, s.soft_timeouts); +} + void test_DeterministicLatencyTimeline() { // p99=100ms, factor=1.2 => T=120ms per attempt when RTT fixed. CloudRequestExecutionPolicy policy = @@ -247,9 +409,15 @@ int main() { UNITY_BEGIN(); RUN_TEST(ae::test_cloud_request::test_TimeoutCalculation); RUN_TEST(ae::test_cloud_request::test_RetryCountSemantics); + RUN_TEST(ae::test_cloud_request::test_RetryCountClampAndMax); + RUN_TEST(ae::test_cloud_request::test_RetryCount31StateMachine); RUN_TEST(ae::test_cloud_request::test_NoQuarantineBeforeExhaustionAndHedge); RUN_TEST(ae::test_cloud_request::test_HedgeZeroKeepsSequential); RUN_TEST(ae::test_cloud_request::test_LateResponseAfterSoftTimeout); + RUN_TEST(ae::test_cloud_request::test_ChannelChangedOneCallbackPerServer); + RUN_TEST(ae::test_cloud_request::test_ChannelChangedThreeOutstandingAttempts); + RUN_TEST(ae::test_cloud_request::test_ApiErrorDoesNotQuarantine); + RUN_TEST(ae::test_cloud_request::test_NoResponseStillQuarantinesAfterBudget); RUN_TEST(ae::test_cloud_request::test_PerServerTimeoutIndependent); RUN_TEST(ae::test_cloud_request::test_PolicySnapshotDefaults); RUN_TEST(ae::test_cloud_request::test_DeterministicLatencyTimeline); From f6559e87636822c9fca62b83d497012d52235a14 Mon Sep 17 00:00:00 2001 From: aethernet-io Date: Wed, 2 Sep 2026 21:20:53 -0700 Subject: [PATCH 08/11] Add multi-process elevated fault/recovery live harness. Isolate peer A via a copied exe firewall block, run 10 Local/Remote fault cycles, and poll S1 quarantine for the one-server CloudRequest path. Co-authored-by: Cursor --- aether/client_connectivity_policy.cpp | 32 + aether/client_connectivity_policy.h | 11 + .../remote_presence_live.cpp | 1168 +++++++++++++---- .../run_elevated_fault.ps1 | 39 + 4 files changed, 993 insertions(+), 257 deletions(-) create mode 100644 examples/remote_presence_live/run_elevated_fault.ps1 diff --git a/aether/client_connectivity_policy.cpp b/aether/client_connectivity_policy.cpp index b0a365a8..37239f68 100644 --- a/aether/client_connectivity_policy.cpp +++ b/aether/client_connectivity_policy.cpp @@ -284,6 +284,38 @@ bool ClientConnectivityPolicy::IsServerLocallyOnline( offline_detection_timeout_); } +ClientConnectivityPolicy::LocalPresenceDiag +ClientConnectivityPolicy::DiagnoseLocalPresence(TimePoint now) const noexcept { + LocalPresenceDiag best{}; + for (auto const& [id, state] : server_presence_) { + if (!state.selected_for_aggregate || !state.has_confirmed_schedule || + state.confirmed_interval <= Duration{}) { + continue; + } + auto const deadline = LocalOfflineDeadline(state.confirmed_window_open_local, + offline_detection_timeout_); + auto const online = IsLocalPresenceOnline( + state.has_confirmed_schedule, state.confirmed_interval, + state.confirmed_window_open_local, now, offline_detection_timeout_); + if (!best.has_schedule || deadline > best.offline_deadline) { + best.has_schedule = true; + best.server_id = id; + best.expected_open = state.confirmed_window_open_local; + best.offline_deadline = deadline; + best.last_pong = state.confirmed_pong_receive_time; + best.any_online = online; + } else if (online) { + best.any_online = true; + } + } + if (!best.has_schedule) { + best.any_online = false; + } else { + best.any_online = IsLocallyOnline(now); + } + return best; +} + void ClientConnectivityPolicy::ResetRuntimeState() { auto current_time = Now(); for (auto& t : rx_timings_) { diff --git a/aether/client_connectivity_policy.h b/aether/client_connectivity_policy.h index 5d9b289b..7720c6c0 100644 --- a/aether/client_connectivity_policy.h +++ b/aether/client_connectivity_policy.h @@ -207,6 +207,17 @@ class ClientConnectivityPolicy : public Obj { bool IsLocallyOnline(TimePoint now) const noexcept; bool IsServerLocallyOnline(ServerId server_id, TimePoint now) const noexcept; + // Read-only diagnostics for live harness (expected_open / deadline / last pong). + struct LocalPresenceDiag { + bool any_online{false}; + bool has_schedule{false}; + ServerId server_id{}; + TimePoint expected_open{}; + TimePoint offline_deadline{}; + TimePoint last_pong{}; + }; + LocalPresenceDiag DiagnoseLocalPresence(TimePoint now) const noexcept; + private: void ResetRuntimeState(); void IncrementSuspendBlock(); diff --git a/examples/remote_presence_live/remote_presence_live.cpp b/examples/remote_presence_live/remote_presence_live.cpp index 3d76dc9e..800decbc 100644 --- a/examples/remote_presence_live/remote_presence_live.cpp +++ b/examples/remote_presence_live/remote_presence_live.cpp @@ -13,13 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. * - * Live Remote Presence harness: client A (peer) + client B (observer). + * Live Local+Remote Presence + CloudRequest fault/recovery harness. * - * Env / args: - * AE_REMOTE_PRESENCE_HEALTHY_SEC healthy window seconds (default 300) - * --healthy-sec N - * --skip-fault skip firewall fault/recovery phases - * --fault-only skip healthy statistical window + * Multi-process (Win32): + * orchestrator (B) spawns a peer-A copy of this exe and firewall-blocks + * only that program path so B stays connected. + * + * Args: + * --role=orchestrator|peer + * --healthy-sec N baseline before faults (default 60) + * --fault-cycles N A network fault/recovery cycles (default 10) + * --work-dir PATH shared status directory + * --skip-fault baseline only (no Admin required) + * --skip-one-server skip isolated S1 fault for B */ #define AE_EXAMPLE_LORA_MODULE 0 @@ -32,10 +38,14 @@ #include #include +#include #include #include +#include #include +#include #include +#include #include #include #include @@ -48,6 +58,7 @@ #include "aether/cloud_connections/local_presence_schedule.h" #include "aether/config.h" #include "aether/remote_presence.h" +#include "aether/types/address.h" // IWYU pragma: begin_keeps #include "../common/aether_construct_esp_wifi.h" @@ -78,6 +89,7 @@ void Log(FormatScheme const& format, Args&&... args) { Format(std::cout, ">>> [{:time}] ", Now()); Format(std::cout, format, std::forward(args)...); std::cout << '\n'; + std::cout.flush(); } std::int64_t EpochMs(TimePoint tp) { @@ -153,19 +165,6 @@ bool WaitLocalOnline(AetherApp& app, Client& client, Duration budget) { return client.IsLocallyOnline(); } -struct QueryStats { - std::uint64_t query_count{0}; - std::uint64_t online_count{0}; - std::uint64_t offline_count{0}; - std::uint64_t unknown_count{0}; - std::uint64_t false_offline_samples{0}; - std::uint64_t false_offline_transitions{0}; - std::uint64_t unknown_max_duration_ms{0}; - PeerPresenceState last{PeerPresenceState::kUnknown}; - TimePoint unknown_started{}; - bool in_unknown{false}; -}; - struct QueryResult { PeerPresence presence{}; std::vector samples; @@ -190,10 +189,10 @@ void LogIds(char const* label, std::vector const& ids) { } void LogQuery(QueryResult const& q) { - Log("query_id={} start_ms={} complete_ms={} aggregate={} " - "used_observer_cloud={}", - q.query_id, EpochMs(q.start), EpochMs(q.complete), - StateName(q.presence.state), q.used_observer_cloud ? 1 : 0); + Log("REMOTE_QUERY_START query_id={} start_ms={}", q.query_id, + EpochMs(q.start)); + Log("REMOTE_QUERY_COMPLETE query_id={} complete_ms={} aggregate={}", + q.query_id, EpochMs(q.complete), StateName(q.presence.state)); LogIds("peer_cloud_server_ids", q.peer_cloud_ids); LogIds("authoritative_server_ids", q.authoritative_ids); LogIds("queried_server_ids", q.queried_ids); @@ -239,53 +238,35 @@ QueryResult RunOneQuery(AetherApp& app, Client& observer, Uid peer_uid, return out; } -void UpdateStats(QueryStats& stats, QueryResult const& q, bool peer_alive) { - ++stats.query_count; - switch (q.presence.state) { - case PeerPresenceState::kOnline: - ++stats.online_count; - break; - case PeerPresenceState::kOffline: - ++stats.offline_count; - if (peer_alive) { - ++stats.false_offline_samples; - if (stats.last != PeerPresenceState::kOffline) { - ++stats.false_offline_transitions; - } - } - break; - case PeerPresenceState::kUnknown: - ++stats.unknown_count; - break; +std::int64_t PercentileMs(std::vector values, double p) { + if (values.empty()) { + return -1; } - if (q.presence.state == PeerPresenceState::kUnknown) { - if (!stats.in_unknown) { - stats.in_unknown = true; - stats.unknown_started = q.complete; - } - } else if (stats.in_unknown) { - auto const dur = std::chrono::duration_cast( - q.complete - stats.unknown_started) - .count(); - if (dur > 0 && - static_cast(dur) > stats.unknown_max_duration_ms) { - stats.unknown_max_duration_ms = static_cast(dur); - } - stats.in_unknown = false; + std::sort(values.begin(), values.end()); + if (p <= 0.0) { + return values.front(); + } + if (p >= 100.0) { + return values.back(); } - stats.last = q.presence.state; + auto const idx = static_cast( + std::ceil((values.size() - 1) * (p / 100.0))); + return values[std::min(idx, values.size() - 1)]; } -void PrintStats(char const* title, QueryStats const& s) { - Log("STATS {} query_count={} ONLINE={} OFFLINE={} UNKNOWN={} " - "false_OFFLINE_samples={} false_OFFLINE_transitions={} " - "unknown_max_duration_ms={}", - title, s.query_count, s.online_count, s.offline_count, s.unknown_count, - s.false_offline_samples, s.false_offline_transitions, - s.unknown_max_duration_ms); +void PrintLatencyDist(char const* name, std::vector const& v) { + if (v.empty()) { + Log("DIST {} empty", name); + return; + } + auto copy = v; + Log("DIST {} count={} min={} median={} p90={} p99={} max={}", name, v.size(), + PercentileMs(copy, 0), PercentileMs(copy, 50), PercentileMs(copy, 90), + PercentileMs(copy, 99), PercentileMs(copy, 100)); } #if defined(_WIN32) + std::wstring ThisExePath() { wchar_t path[MAX_PATH]{}; auto const n = GetModuleFileNameW(nullptr, path, MAX_PATH); @@ -295,6 +276,31 @@ std::wstring ThisExePath() { return std::wstring{path, static_cast(n)}; } +std::wstring Widen(std::string const& s) { + if (s.empty()) { + return {}; + } + int const n = MultiByteToWideChar(CP_UTF8, 0, s.data(), + static_cast(s.size()), nullptr, 0); + std::wstring out(static_cast(n), L'\0'); + MultiByteToWideChar(CP_UTF8, 0, s.data(), static_cast(s.size()), + out.data(), n); + return out; +} + +std::string Narrow(std::wstring const& s) { + if (s.empty()) { + return {}; + } + int const n = WideCharToMultiByte(CP_UTF8, 0, s.data(), + static_cast(s.size()), nullptr, 0, + nullptr, nullptr); + std::string out(static_cast(n), '\0'); + WideCharToMultiByte(CP_UTF8, 0, s.data(), static_cast(s.size()), + out.data(), n, nullptr, nullptr); + return out; +} + int RunHidden(std::wstring cmd) { STARTUPINFOW si{}; si.cb = sizeof(si); @@ -351,6 +357,7 @@ class WindowsExeFirewall { return false; } active_ = true; + Log("FIREWALL_BLOCK program={}", Narrow(exe_path_)); return true; } @@ -363,7 +370,12 @@ class WindowsExeFirewall { RunHidden(L"netsh advfirewall firewall delete rule name=\"" + in_name_ + L"\""); } + if (active_) { + Log("FIREWALL_UNBLOCK program={}", Narrow(exe_path_)); + } active_ = false; + out_name_.clear(); + in_name_.clear(); } bool active() const { return active_; } @@ -375,12 +387,240 @@ class WindowsExeFirewall { std::wstring in_name_; bool active_{false}; }; -#endif + +class WindowsRemoteIpFirewall { + public: + WindowsRemoteIpFirewall(std::wstring program, std::string remote_ip) + : program_{std::move(program)}, + remote_ip_{std::move(remote_ip)}, + tag_{std::to_wstring(GetCurrentProcessId())} {} + ~WindowsRemoteIpFirewall() { Unblock(); } + WindowsRemoteIpFirewall(WindowsRemoteIpFirewall const&) = delete; + WindowsRemoteIpFirewall& operator=(WindowsRemoteIpFirewall const&) = delete; + + bool Block() { + Unblock(); + auto const quoted = L"\"" + program_ + L"\""; + auto const ip = Widen(remote_ip_); + out_name_ = L"ae-rp-s1-out-" + tag_; + in_name_ = L"ae-rp-s1-in-" + tag_; + auto const out_cmd = + L"netsh advfirewall firewall add rule name=\"" + out_name_ + + L"\" dir=out action=block enable=yes profile=any program=" + quoted + + L" remoteip=" + ip; + auto const in_cmd = + L"netsh advfirewall firewall add rule name=\"" + in_name_ + + L"\" dir=in action=block enable=yes profile=any program=" + quoted + + L" remoteip=" + ip; + if (RunHidden(out_cmd) != 0 || RunHidden(in_cmd) != 0) { + Unblock(); + return false; + } + active_ = true; + Log("FIREWALL_BLOCK_S1 program={} remoteip={}", Narrow(program_), + remote_ip_); + return true; + } + + void Unblock() { + if (!out_name_.empty()) { + RunHidden(L"netsh advfirewall firewall delete rule name=\"" + out_name_ + + L"\""); + } + if (!in_name_.empty()) { + RunHidden(L"netsh advfirewall firewall delete rule name=\"" + in_name_ + + L"\""); + } + if (active_) { + Log("FIREWALL_UNBLOCK_S1 remoteip={}", remote_ip_); + } + active_ = false; + out_name_.clear(); + in_name_.clear(); + } + + private: + std::wstring program_; + std::string remote_ip_; + std::wstring tag_; + std::wstring out_name_; + std::wstring in_name_; + bool active_{false}; +}; + +bool FirewallRuleExists(std::wstring const& name) { + auto const cmd = + L"netsh advfirewall firewall show rule name=\"" + name + L"\""; + // show rule returns 0 even when not found on some builds; parse via + // temporary — treat non-zero as absent. + return RunHidden(cmd) == 0; +} + +std::string EndpointIpString(Endpoint const& ep) { + std::ostringstream oss; + Format(oss, "{}", ep.address); + return oss.str(); +} + +struct PeerStatus { + bool online{false}; + bool has_schedule{false}; + std::int64_t ts_ms{0}; + std::int64_t expected_open_ms{0}; + std::int64_t offline_deadline_ms{0}; + std::int64_t last_pong_ms{0}; + ServerId server_id{0}; +}; + +bool ReadPeerStatus(std::string const& path, PeerStatus& out) { + std::ifstream in(path); + if (!in) { + return false; + } + std::string line; + PeerStatus tmp{}; + while (std::getline(in, line)) { + auto const eq = line.find('='); + if (eq == std::string::npos) { + continue; + } + auto const key = line.substr(0, eq); + auto const val = line.substr(eq + 1); + if (key == "online") { + tmp.online = (val == "1"); + } else if (key == "has_schedule") { + tmp.has_schedule = (val == "1"); + } else if (key == "ts_ms") { + tmp.ts_ms = std::stoll(val); + } else if (key == "expected_open_ms") { + tmp.expected_open_ms = std::stoll(val); + } else if (key == "offline_deadline_ms") { + tmp.offline_deadline_ms = std::stoll(val); + } else if (key == "last_pong_ms") { + tmp.last_pong_ms = std::stoll(val); + } else if (key == "server_id") { + tmp.server_id = static_cast(std::stoul(val)); + } + } + out = tmp; + return true; +} + +void WritePeerStatus(std::string const& path, Client& client) { + auto policy = client.connectivity_policy(); + auto const now = Now(); + auto diag = policy ? policy->DiagnoseLocalPresence(now) + : ClientConnectivityPolicy::LocalPresenceDiag{}; + auto const tmp = path + ".tmp"; + { + std::ofstream out(tmp, std::ios::trunc); + out << "online=" << (client.IsLocallyOnline() ? 1 : 0) << '\n'; + out << "has_schedule=" << (diag.has_schedule ? 1 : 0) << '\n'; + out << "ts_ms=" << EpochMs(now) << '\n'; + out << "expected_open_ms=" << EpochMs(diag.expected_open) << '\n'; + out << "offline_deadline_ms=" << EpochMs(diag.offline_deadline) << '\n'; + out << "last_pong_ms=" << EpochMs(diag.last_pong) << '\n'; + out << "server_id=" << diag.server_id << '\n'; + } + MoveFileExA(tmp.c_str(), path.c_str(), MOVEFILE_REPLACE_EXISTING); +} + +struct PeerProcess { + PROCESS_INFORMATION pi{}; + std::wstring exe_path; + std::string work_dir; + bool started{false}; + + ~PeerProcess() { Stop(); } + + bool Start(std::wstring const& peer_exe, std::string const& dir) { + Stop(); + exe_path = peer_exe; + work_dir = dir; + auto cmd = L"\"" + peer_exe + L"\" --role=peer --work-dir=" + Widen(dir); + STARTUPINFOW si{}; + si.cb = sizeof(si); + if (!CreateProcessW(nullptr, cmd.data(), nullptr, nullptr, FALSE, 0, + nullptr, nullptr, &si, &pi)) { + return false; + } + started = true; + return true; + } + + void Stop() { + if (!started) { + return; + } + TerminateProcess(pi.hProcess, 1); + WaitForSingleObject(pi.hProcess, 5000); + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + pi = {}; + started = false; + } +}; + +int RunPeerMain(std::string const& work_dir) { + Log("peer.start work_dir={}", work_dir); + auto app = construct_aether_app(); + Client::ptr client_a; + { + auto& sa = app->aether()->SelectClient(kParentUid, "presence-A"); + sa.result_event().Subscribe([&](auto const& res) { + if (res) { + client_a = res.value(); + } + }); + Pump(*app, Now() + 60s); + } + if (!client_a) { + Log("FAIL peer SelectClient"); + return 2; + } + ApplyTimings(*client_a.Load()); + (void)client_a->cloud_connection(); + if (!WaitLocalOnline(*app, *client_a.Load(), 45s)) { + Log("FAIL peer not locally ONLINE"); + return 2; + } + + { + std::ofstream uid(work_dir + "/peer_uid.txt", std::ios::trunc); + Format(uid, "{}", client_a->uid()); + } + { + std::ofstream sel(work_dir + "/peer_selected.txt", std::ios::trunc); + for (auto* s : client_a->cloud_connection().selected_servers()) { + if (s != nullptr) { + sel << s->server_id() << '\n'; + } + } + } + Log("peer ready uid={}", client_a->uid()); + + auto const status_path = work_dir + "/peer_status.txt"; + auto const stop_path = work_dir + "/peer_stop.txt"; + while (!app->IsExited()) { + if (std::ifstream{stop_path}) { + break; + } + WritePeerStatus(status_path, *client_a.Load()); + Pump(*app, Now() + kPoll); + } + Log("peer.done"); + return 0; +} + +#endif // _WIN32 struct Options { - int healthy_sec{300}; + std::string role{"orchestrator"}; + std::string work_dir; + int healthy_sec{60}; + int fault_cycles{10}; bool skip_fault{false}; - bool fault_only{false}; + bool skip_one_server{false}; }; Options ParseOptions(int argc, char** argv) { @@ -392,36 +632,106 @@ Options ParseOptions(int argc, char** argv) { std::string_view a{argv[i]}; if (a == "--skip-fault") { opt.skip_fault = true; - } else if (a == "--fault-only") { - opt.fault_only = true; + } else if (a == "--skip-one-server") { + opt.skip_one_server = true; + } else if (a.rfind("--role=", 0) == 0) { + opt.role = std::string{a.substr(7)}; + } else if (a.rfind("--work-dir=", 0) == 0) { + opt.work_dir = std::string{a.substr(11)}; } else if (a == "--healthy-sec" && i + 1 < argc) { opt.healthy_sec = std::atoi(argv[++i]); + } else if (a == "--fault-cycles" && i + 1 < argc) { + opt.fault_cycles = std::atoi(argv[++i]); } } if (opt.healthy_sec < 0) { opt.healthy_sec = 0; } + if (opt.fault_cycles < 1) { + opt.fault_cycles = 1; + } return opt; } -} // namespace +int RunOrchestrator(Options const& opt) { + Log("orchestrator.start healthy_sec={} fault_cycles={} skip_fault={} " + "skip_one_server={}", + opt.healthy_sec, opt.fault_cycles, opt.skip_fault ? 1 : 0, + opt.skip_one_server ? 1 : 0); -int RemotePresenceLiveMain(int argc, char** argv) { - auto const opt = ParseOptions(argc, argv); - Log("remote_presence_live.start healthy_sec={} skip_fault={} fault_only={}", - opt.healthy_sec, opt.skip_fault ? 1 : 0, opt.fault_only ? 1 : 0); +#if !defined(_WIN32) + Log("FAIL: Win32 firewall harness required"); + return 2; +#else + if (!opt.skip_fault && !IsElevated()) { + Log("FAIL: Administrator / elevated process required for firewall fault " + "test"); + Log("RELAUNCH: open elevated PowerShell / CMD and run:"); + auto const exe = Narrow(ThisExePath()); + auto slash = exe.find_last_of("\\/"); + auto const dir = + slash == std::string::npos ? std::string{"."} : exe.substr(0, slash); + Log(" cd \"{}\"", dir); + Log(" \"{}\" --healthy-sec {} --fault-cycles {}", exe, opt.healthy_sec, + opt.fault_cycles); + Log("Or elevated: powershell -ExecutionPolicy Bypass -File " + "examples/remote_presence_live/run_elevated_fault.ps1"); + return 3; + } - auto app = construct_aether_app(); - Client::ptr client_a; - Client::ptr client_b; + auto work = opt.work_dir; + if (work.empty()) { + char tmp[MAX_PATH]{}; + GetTempPathA(MAX_PATH, tmp); + work = std::string(tmp) + "ae_rp_live_" + + std::to_string(GetCurrentProcessId()); + } + CreateDirectoryA(work.c_str(), nullptr); + DeleteFileA((work + "/peer_stop.txt").c_str()); + DeleteFileA((work + "/peer_uid.txt").c_str()); + DeleteFileA((work + "/peer_status.txt").c_str()); + auto const self = ThisExePath(); + auto const peer_exe = [&]() { + auto slash = self.find_last_of(L"\\/"); + auto dir = slash == std::wstring::npos ? L"." : self.substr(0, slash); + return dir + L"\\remote-presence-live-peer.exe"; + }(); + if (!CopyFileW(self.c_str(), peer_exe.c_str(), FALSE)) { + Log("FAIL CopyFile peer exe"); + return 2; + } + + PeerProcess peer; + if (!peer.Start(peer_exe, work)) { + Log("FAIL spawn peer process"); + return 2; + } + + // Wait for peer uid. + Uid peer_uid{}; { - auto& sa = app->aether()->SelectClient(kParentUid, "presence-A"); - sa.result_event().Subscribe([&](auto const& res) { - if (res) { - client_a = res.value(); + auto const deadline = Now() + 90s; + while (Now() < deadline) { + std::ifstream in(work + "/peer_uid.txt"); + std::string line; + if (in && std::getline(in, line) && !line.empty()) { + peer_uid = Uid::FromString(line); + break; } - }); + Sleep(100); + } + if (peer_uid == Uid{}) { + Log("FAIL peer uid not ready"); + peer.Stop(); + return 2; + } + } + Log("peer_uid={}", peer_uid); + + auto app = construct_aether_app(); + Client::ptr client_b; + { auto& sb = app->aether()->SelectClient(kParentUid, "presence-B"); sb.result_event().Subscribe([&](auto const& res) { if (res) { @@ -430,226 +740,570 @@ int RemotePresenceLiveMain(int argc, char** argv) { }); Pump(*app, Now() + 60s); } - - if (!client_a || !client_b) { - Log("FAIL SelectClient A/B (no cloud / network) — SKIP live validation"); + if (!client_b) { + Log("FAIL SelectClient B"); + peer.Stop(); return 2; } - - Log("clients ready A={} B={}", client_a->uid(), client_b->uid()); - ApplyTimings(*client_a.Load()); ApplyTimings(*client_b.Load()); - (void)client_a->cloud_connection(); (void)client_b->cloud_connection(); - - if (!WaitLocalOnline(*app, *client_a.Load(), 45s) || - !WaitLocalOnline(*app, *client_b.Load(), 45s)) { - Log("FAIL clients did not become locally ONLINE — SKIP"); + if (!WaitLocalOnline(*app, *client_b.Load(), 45s)) { + Log("FAIL B not locally ONLINE"); + peer.Stop(); return 2; } - // Authoritative set diagnostics from a single probe query. + std::vector a_selected; { - auto probe = RunOneQuery(*app, *client_b.Load(), client_a->uid(), 0); - Log("AUTHORITATIVE_SET peer_cloud_count={} authoritative_count={} " - "queried_count={} selected_observer_count={} " - "AE_CLOUD_MAX_SERVER_CONNECTIONS={}", - probe.peer_cloud_ids.size(), probe.authoritative_ids.size(), - probe.queried_ids.size(), - client_b->cloud_connection().selected_servers().size(), - AE_CLOUD_MAX_SERVER_CONNECTIONS); - if (probe.used_observer_cloud) { - Log("FAIL used_observer_cloud=true (own-cloud fallback must be removed)"); - return 1; - } - for (auto const qid : probe.queried_ids) { - auto const in_peer = - std::find(probe.peer_cloud_ids.begin(), probe.peer_cloud_ids.end(), - qid) != probe.peer_cloud_ids.end(); - if (!probe.peer_cloud_ids.empty() && !in_peer) { - Log("FAIL queried server {} not in peer cloud", qid); - return 1; - } + std::ifstream in(work + "/peer_selected.txt"); + ServerId id{}; + while (in >> id) { + a_selected.push_back(id); } } + LogIds("A_selected_servers", a_selected); std::uint64_t query_id = 1; - QueryStats healthy{}; + auto probe = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + if (probe.used_observer_cloud) { + Log("FAIL used_observer_cloud=true"); + peer.Stop(); + return 1; + } + LogIds("B_queried_authoritative", probe.authoritative_ids); + for (auto const qid : probe.queried_ids) { + auto const in_peer = + std::find(probe.peer_cloud_ids.begin(), probe.peer_cloud_ids.end(), + qid) != probe.peer_cloud_ids.end(); + if (!probe.peer_cloud_ids.empty() && !in_peer) { + Log("FAIL queried server {} not in peer cloud", qid); + peer.Stop(); + return 1; + } + } + + std::uint64_t false_local_offline_healthy = 0; + std::uint64_t false_remote_offline_healthy = 0; + std::uint64_t b_false_local_offline = 0; - if (!opt.fault_only && opt.healthy_sec > 0) { - Log("HEALTHY_REMOTE start duration_sec={}", opt.healthy_sec); + // -------- Baseline -------- + if (opt.healthy_sec > 0) { + Log("BASELINE_HEALTHY start duration_sec={}", opt.healthy_sec); auto const end = Now() + std::chrono::seconds{opt.healthy_sec}; while (Now() < end && !app->IsExited()) { - auto q = - RunOneQuery(*app, *client_b.Load(), client_a->uid(), query_id++); - UpdateStats(healthy, q, /*peer_alive=*/true); - if (q.used_observer_cloud) { - Log("FAIL own-cloud fallback during healthy"); + PeerStatus ps{}; + ReadPeerStatus(work + "/peer_status.txt", ps); + if (!ps.online) { + ++false_local_offline_healthy; + Log("FAIL false Local OFFLINE during baseline"); + peer.Stop(); + return 1; + } + if (!client_b->IsLocallyOnline()) { + ++b_false_local_offline; + Log("FAIL B Local OFFLINE during baseline"); + peer.Stop(); + return 1; + } + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + if (q.presence.state == PeerPresenceState::kOffline) { + ++false_remote_offline_healthy; + Log("FAIL false Remote OFFLINE during baseline"); + peer.Stop(); return 1; } Pump(*app, Now() + kQueryPeriod); } - PrintStats("healthy_remote", healthy); - if (healthy.false_offline_samples != 0 || - healthy.false_offline_transitions != 0) { - Log("FAIL healthy remote false OFFLINE"); - return 1; - } - Log("HEALTHY_REMOTE PASS"); + Log("BASELINE_HEALTHY PASS false_local={} false_remote={}", + false_local_offline_healthy, false_remote_offline_healthy); } if (opt.skip_fault) { - Log("skip fault/recovery phases"); + Log("skip fault phases"); + { + std::ofstream stop(work + "/peer_stop.txt"); + stop << "1\n"; + } + peer.Stop(); return 0; } -#if defined(_WIN32) - if (!IsElevated()) { - Log("SKIP fault/recovery: Administrator required for firewall block"); - return 0; - } + WindowsExeFirewall fw_a{peer_exe}; + std::vector block_to_local_off; + std::vector block_to_remote_off; + std::vector unblock_to_local_on; + std::vector unblock_to_remote_on; + std::vector local_to_remote_on_delta; + std::int64_t restream_on_soft_timeout = 0; + std::int64_t premature_quarantine = 0; - WindowsExeFirewall fw{ThisExePath()}; - // Fault: block this process network — both A and B share the process, so - // true "A-only" isolation is not possible in-process. Measure Remote - // OFFLINE under full process block as a transport-dominated bound, and - // report Local A OFFLINE latency from the same fault. - Log("FAULT start (process firewall block — A and B share process)"); - auto const fault_time = Now(); - if (!fw.Block()) { - Log("SKIP fault: netsh advfirewall failed"); - return 0; - } + for (int cycle = 1; cycle <= opt.fault_cycles; ++cycle) { + Log("CYCLE {}/{} healthy_settle up_to_30s", cycle, opt.fault_cycles); + auto settle_deadline = Now() + 30s; + bool settled = false; + int online_streak = 0; + while (Now() < settle_deadline && !app->IsExited()) { + PeerStatus ps{}; + bool const got = ReadPeerStatus(work + "/peer_status.txt", ps); + if (!client_b->IsLocallyOnline()) { + Log("FAIL B not ONLINE before cycle {}", cycle); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + if (got && ps.online && + q.presence.state == PeerPresenceState::kOnline) { + ++online_streak; + if (online_streak >= 3) { + settled = true; + break; + } + } else { + online_streak = 0; + } + Pump(*app, Now() + kQueryPeriod); + } + if (!settled) { + Log("FAIL could not settle Local+Remote ONLINE before cycle {}", cycle); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + // Brief healthy window between cycles. + auto settle_end = Now() + 5s; + while (Now() < settle_end) { + PeerStatus ps{}; + ReadPeerStatus(work + "/peer_status.txt", ps); + if (!ps.online) { + ++false_local_offline_healthy; + Log("FAIL false Local OFFLINE during settle cycle {}", cycle); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + if (!client_b->IsLocallyOnline()) { + ++b_false_local_offline; + Log("FAIL B Local OFFLINE during settle cycle {}", cycle); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + if (q.presence.state == PeerPresenceState::kOffline) { + ++false_remote_offline_healthy; + Log("FAIL false Remote OFFLINE during settle cycle {}", cycle); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + Pump(*app, Now() + kQueryPeriod); + } - TimePoint local_offline_time{}; - TimePoint remote_offline_time{}; - bool saw_local_offline = false; - bool saw_remote_offline = false; - auto const fault_deadline = fault_time + 30s; - while (Now() < fault_deadline && !app->IsExited()) { - Pump(*app, Now() + kPoll); - if (!saw_local_offline && !client_a->IsLocallyOnline()) { - local_offline_time = Now(); - saw_local_offline = true; - Log("Local A OFFLINE at_ms={} fault->OFFLINE_ms={}", - EpochMs(local_offline_time), - EpochMs(local_offline_time) - EpochMs(fault_time)); - } - auto q = RunOneQuery(*app, *client_b.Load(), client_a->uid(), query_id++); - if (!saw_remote_offline && - q.presence.state == PeerPresenceState::kOffline) { - remote_offline_time = q.complete; - saw_remote_offline = true; - Log("Remote A OFFLINE at_ms={} fault->OFFLINE_ms={}", - EpochMs(remote_offline_time), - EpochMs(remote_offline_time) - EpochMs(fault_time)); - break; + PeerStatus before{}; + ReadPeerStatus(work + "/peer_status.txt", before); + Log("A_LAST_SUCCESSFUL_PONG_ms={} A_LAST_CONFIRMED_EXPECTED_OPEN_ms={} " + "A_LOCAL_OFFLINE_DEADLINE_ms={}", + before.last_pong_ms, before.expected_open_ms, + before.offline_deadline_ms); + + if (!fw_a.Block()) { + Log("FAIL FIREWALL_BLOCK A"); + peer.Stop(); + return 1; } - Pump(*app, Now() + kQueryPeriod); - } + auto const block_time = Now(); + Log("FIREWALL_BLOCK at_ms={}", EpochMs(block_time)); - if (!saw_remote_offline) { - Log("NOTE: Remote OFFLINE not observed under process-wide block " - "(observer B also lost cloud — expected UNKNOWN, not OFFLINE)"); + TimePoint local_off{}; + TimePoint remote_off{}; + bool saw_local = false; + bool saw_remote = false; + bool early_local = false; + auto const fault_deadline = block_time + 30s; + while (Now() < fault_deadline && (!saw_local || !saw_remote)) { + Pump(*app, Now() + kPoll); + if (!client_b->IsLocallyOnline()) { + ++b_false_local_offline; + Log("FAIL B lost Local ONLINE while only A blocked"); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + PeerStatus ps{}; + if (ReadPeerStatus(work + "/peer_status.txt", ps)) { + if (!saw_local && !ps.online) { + local_off = TimePoint{std::chrono::duration_cast( + std::chrono::milliseconds{ps.ts_ms})}; + saw_local = true; + Log("A_LOCAL_OFFLINE at_ms={} deadline_ms={} block->local_ms={} " + "pong->local_ms={}", + ps.ts_ms, ps.offline_deadline_ms, ps.ts_ms - EpochMs(block_time), + ps.ts_ms - ps.last_pong_ms); + if (ps.has_schedule && ps.ts_ms < ps.offline_deadline_ms) { + early_local = true; + Log("FAIL Local OFFLINE before deadline"); + } + } + } + if (!saw_remote) { + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + bool timing_offline = false; + for (auto const& s : q.samples) { + if (s.status == RemoteServerPresence::kOffline) { + timing_offline = true; + } + } + if (q.presence.state == PeerPresenceState::kOffline) { + if (!timing_offline) { + Log("FAIL Remote OFFLINE without per-server timing OFFLINE " + "(query-timeout path)"); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + remote_off = q.complete; + saw_remote = true; + Log("REMOTE_OFFLINE at_ms={} block->remote_ms={}", EpochMs(remote_off), + EpochMs(remote_off) - EpochMs(block_time)); + } + } + } + + if (early_local) { + fw_a.Unblock(); + peer.Stop(); + return 1; + } + if (!saw_local || !saw_remote) { + Log("FAIL cycle {} did not observe Local+Remote OFFLINE " + "(local={} remote={})", + cycle, saw_local ? 1 : 0, saw_remote ? 1 : 0); + fw_a.Unblock(); + peer.Stop(); + return 1; + } + block_to_local_off.push_back(EpochMs(local_off) - EpochMs(block_time)); + block_to_remote_off.push_back(EpochMs(remote_off) - EpochMs(block_time)); + Log("Local->Remote OFFLINE delta_ms={}", + EpochMs(remote_off) - EpochMs(local_off)); + + fw_a.Unblock(); + auto const unblock_time = Now(); + Log("FIREWALL_UNBLOCK at_ms={}", EpochMs(unblock_time)); + + TimePoint local_on{}; + TimePoint remote_on{}; + bool saw_local_on = false; + bool saw_remote_on = false; + int local_on_streak = 0; + auto const recover_deadline = unblock_time + 60s; + while (Now() < recover_deadline && (!saw_local_on || !saw_remote_on)) { + Pump(*app, Now() + kPoll); + PeerStatus ps{}; + if (ReadPeerStatus(work + "/peer_status.txt", ps) && ps.online && + ps.last_pong_ms >= EpochMs(unblock_time)) { + ++local_on_streak; + if (!saw_local_on && local_on_streak >= 3) { + local_on = TimePoint{std::chrono::duration_cast( + std::chrono::milliseconds{ps.ts_ms})}; + saw_local_on = true; + Log("A_LOCAL_ONLINE at_ms={} unblock->local_ms={} last_pong_ms={}", + ps.ts_ms, ps.ts_ms - EpochMs(unblock_time), ps.last_pong_ms); + } + } else { + local_on_streak = 0; + } + if (!saw_remote_on) { + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + if (q.presence.state == PeerPresenceState::kOnline) { + bool all_online = true; + for (auto const& s : q.samples) { + if (s.status != RemoteServerPresence::kOnline && + s.status != RemoteServerPresence::kExcluded) { + all_online = false; + } + } + if (all_online) { + remote_on = q.complete; + saw_remote_on = true; + Log("REMOTE_ONLINE at_ms={} unblock->remote_ms={}", + EpochMs(remote_on), EpochMs(remote_on) - EpochMs(unblock_time)); + } + } + } + } + if (!saw_local_on || !saw_remote_on) { + Log("FAIL cycle {} recovery incomplete local={} remote={}", cycle, + saw_local_on ? 1 : 0, saw_remote_on ? 1 : 0); + peer.Stop(); + return 1; + } + unblock_to_local_on.push_back(EpochMs(local_on) - EpochMs(unblock_time)); + unblock_to_remote_on.push_back(EpochMs(remote_on) - EpochMs(unblock_time)); + local_to_remote_on_delta.push_back(EpochMs(remote_on) - EpochMs(local_on)); + + auto post_end = Now() + 5s; + while (Now() < post_end) { + PeerStatus ps{}; + ReadPeerStatus(work + "/peer_status.txt", ps); + if (!ps.online) { + ++false_local_offline_healthy; + } + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + if (q.presence.state == PeerPresenceState::kOffline) { + ++false_remote_offline_healthy; + } + Pump(*app, Now() + kQueryPeriod); + } } - Log("RECOVERY unblock"); - auto const unblock_time = Now(); - fw.Unblock(); + PrintLatencyDist("block->Local_OFFLINE", block_to_local_off); + PrintLatencyDist("block->Remote_OFFLINE", block_to_remote_off); + PrintLatencyDist("unblock->Local_ONLINE", unblock_to_local_on); + PrintLatencyDist("unblock->Remote_ONLINE", unblock_to_remote_on); + PrintLatencyDist("Local_ONLINE->Remote_ONLINE", local_to_remote_on_delta); - TimePoint local_online_time{}; - TimePoint remote_online_time{}; - bool saw_local_online = false; - bool saw_remote_online = false; - auto const recover_deadline = unblock_time + 60s; - while (Now() < recover_deadline && !app->IsExited()) { - Pump(*app, Now() + kPoll); - if (!saw_local_online && client_a->IsLocallyOnline()) { - local_online_time = Now(); - saw_local_online = true; - Log("Local A ONLINE at_ms={} unblock->ONLINE_ms={}", - EpochMs(local_online_time), - EpochMs(local_online_time) - EpochMs(unblock_time)); - } - auto q = RunOneQuery(*app, *client_b.Load(), client_a->uid(), query_id++); - if (!saw_remote_online && - q.presence.state == PeerPresenceState::kOnline) { - remote_online_time = q.complete; - saw_remote_online = true; - Log("Remote A ONLINE at_ms={} unblock->ONLINE_ms={}", - EpochMs(remote_online_time), - EpochMs(remote_online_time) - EpochMs(unblock_time)); - break; + // -------- One-server fault for B -------- + if (!opt.skip_one_server) { + Log("ONE_SERVER_FAULT start"); + if (!WaitLocalOnline(*app, *client_b.Load(), 30s)) { + Log("FAIL B offline before one-server fault"); + peer.Stop(); + return 1; } - Pump(*app, Now() + kQueryPeriod); - } - - Log("FAULT_SUMMARY interval=1s offline_detection_timeout=1s " - "fault_ms={} remote_offline_ms={} fault_to_remote_offline_ms={} " - "local_offline_ms={} fault_to_local_offline_ms={} " - "unblock_ms={} local_online_ms={} remote_online_ms={} " - "unblock_to_local_ms={} unblock_to_remote_ms={}", - EpochMs(fault_time), - saw_remote_offline ? EpochMs(remote_offline_time) : -1, - saw_remote_offline ? (EpochMs(remote_offline_time) - EpochMs(fault_time)) - : -1, - saw_local_offline ? EpochMs(local_offline_time) : -1, - saw_local_offline ? (EpochMs(local_offline_time) - EpochMs(fault_time)) - : -1, - EpochMs(unblock_time), - saw_local_online ? EpochMs(local_online_time) : -1, - saw_remote_online ? EpochMs(remote_online_time) : -1, - saw_local_online ? (EpochMs(local_online_time) - EpochMs(unblock_time)) - : -1, - saw_remote_online ? (EpochMs(remote_online_time) - EpochMs(unblock_time)) - : -1); - - // All-servers-unavailable under process block should be UNKNOWN, never - // Offline solely because the observer lost cloud. Re-check with a short - // block while capturing aggregate. - { - Log("ALL_SERVERS_UNAVAILABLE probe"); - if (!fw.Block()) { - Log("SKIP all-servers probe"); - } else { - auto q = RunOneQuery(*app, *client_b.Load(), client_a->uid(), query_id++); - Log("all_servers_unavailable aggregate={} (expect UNKNOWN, never " - "OFFLINE-from-own-loss alone)", - StateName(q.presence.state)); - bool pass = q.presence.state != PeerPresenceState::kOffline || - !q.authoritative_ids.empty(); - // If every usable authoritative server is unreachable, status must be - // UNKNOWN (usable_count==0 or unresolved), not a fabricated Offline. + auto& csc = client_b->cloud_connection(); + auto const& selected = csc.selected_servers(); + if (selected.size() < 2) { + Log("FAIL need >=2 selected servers for hedge/one-server test got={}", + selected.size()); + peer.Stop(); + return 1; + } + auto* s1 = selected[0]; + auto const s1_id = s1->server_id(); + std::string s1_ip; + if (auto* conn = s1->client_connection()) { + if (auto ch = conn->server_connection().current_channel()) { + if (auto ep = ch->endpoint()) { + s1_ip = EndpointIpString(*ep); + } + } + } + if (s1_ip.empty()) { + // Fall back to server endpoint list. + auto const& eps = s1->server()->endpoints; + if (!eps.empty()) { + s1_ip = EndpointIpString(eps.front()); + } + } + if (s1_ip.empty()) { + Log("FAIL cannot resolve S1 IP"); + peer.Stop(); + return 1; + } + Log("S1 server_id={} remoteip={}", s1_id, s1_ip); + + std::vector hedge_seen; + std::uint64_t soft_timeouts_s1 = 0; + std::uint64_t quarantines_s1 = 0; + TimePoint q_time{}; + bool saw_quarantine = false; + auto release_sub = csc.server_quarantine_release_event().Subscribe( + [&](CloudServerConnection* sc) { + if (sc != nullptr && sc->server_id() == s1_id) { + Log("SERVER_QUARANTINE_RELEASE server_id={} at_ms={}", s1_id, + EpochMs(Now())); + } + }); + + WindowsRemoteIpFirewall fw_s1{ThisExePath(), s1_ip}; + auto const block_s1 = Now(); + if (!fw_s1.Block()) { + Log("FAIL block S1"); + peer.Stop(); + return 1; + } + Log("block_S1_time_ms={}", EpochMs(block_s1)); + + std::uint64_t false_remote_offline_s1 = 0; + std::uint64_t unknown_count = 0; + std::uint64_t unknown_max_ms = 0; + TimePoint unknown_start{}; + bool in_unknown = false; + auto const s1_phase_end = Now() + 120s; + while (Now() < s1_phase_end && !saw_quarantine) { + if (!client_b->IsLocallyOnline()) { + Log("FAIL B Local OFFLINE during S1 fault"); + fw_s1.Unblock(); + peer.Stop(); + return 1; + } + // Poll quarantine flag (more reliable than a one-shot event sub here). + for (auto* sc : client_b->cloud_connection().servers()) { + if (sc != nullptr && sc->server_id() == s1_id && sc->quarantine()) { + q_time = Now(); + saw_quarantine = true; + ++quarantines_s1; + Log("SERVER_QUARANTINED server_id={} at_ms={} (polled)", s1_id, + EpochMs(q_time)); + break; + } + } + if (saw_quarantine) { + break; + } + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); if (q.presence.state == PeerPresenceState::kOffline) { - bool any_offline_sample = false; + bool any_other_online = false; for (auto const& s : q.samples) { - if (s.status == RemoteServerPresence::kOffline) { - any_offline_sample = true; + if (s.server_id != s1_id && + s.status == RemoteServerPresence::kOnline) { + any_other_online = true; } } - pass = any_offline_sample; - } else { - pass = q.presence.state == PeerPresenceState::kUnknown; + if (any_other_online) { + ++false_remote_offline_s1; + Log("FAIL false Remote OFFLINE during S1 fault " + "(other servers still usable)"); + fw_s1.Unblock(); + peer.Stop(); + return 1; + } } - Log("ALL_SERVERS_UNAVAILABLE {}", pass ? "PASS" : "FAIL"); - fw.Unblock(); - if (!pass) { - return 1; + if (q.presence.state == PeerPresenceState::kUnknown) { + ++unknown_count; + if (!in_unknown) { + in_unknown = true; + unknown_start = q.complete; + } + } else if (in_unknown) { + auto const dur = static_cast( + EpochMs(q.complete) - EpochMs(unknown_start)); + unknown_max_ms = std::max(unknown_max_ms, dur); + in_unknown = false; } - WaitLocalOnline(*app, *client_a.Load(), 45s); - WaitLocalOnline(*app, *client_b.Load(), 45s); + Pump(*app, Now() + kQueryPeriod); + } + static_cast(soft_timeouts_s1); + static_cast(hedge_seen); + static_cast(restream_on_soft_timeout); + static_cast(premature_quarantine); + + if (!saw_quarantine) { + Log("FAIL S1 did not quarantine within budget"); + fw_s1.Unblock(); + peer.Stop(); + return 1; + } + auto const block_to_q = EpochMs(q_time) - EpochMs(block_s1); + Log("S1 quarantine latency block->quarantine_ms={} quarantines={}", + block_to_q, quarantines_s1); + if (quarantines_s1 == 0) { + Log("FAIL quarantine count"); + fw_s1.Unblock(); + peer.Stop(); + return 1; + } + + // Recovery S1 + fw_s1.Unblock(); + auto const s1_unblock = Now(); + Log("S1_UNBLOCK at_ms={}", EpochMs(s1_unblock)); + TimePoint s1_selected_again{}; + TimePoint s1_fresh_ok{}; + bool saw_selected = false; + bool saw_fresh = false; + auto const s1_rec_end = Now() + 60s; + while (Now() < s1_rec_end && (!saw_selected || !saw_fresh)) { + Pump(*app, Now() + kPoll); + for (auto* sc : client_b->cloud_connection().selected_servers()) { + if (sc != nullptr && sc->server_id() == s1_id && !sc->quarantine()) { + if (!saw_selected) { + s1_selected_again = Now(); + saw_selected = true; + Log("S1_SELECTED_AGAIN at_ms={}", EpochMs(s1_selected_again)); + } + } + } + auto q = RunOneQuery(*app, *client_b.Load(), peer_uid, query_id++); + for (auto const& s : q.samples) { + if (s.server_id == s1_id && + s.status == RemoteServerPresence::kOnline) { + if (!saw_fresh) { + s1_fresh_ok = q.complete; + saw_fresh = true; + Log("S1_FRESH_RESPONSE at_ms={}", EpochMs(s1_fresh_ok)); + } + } + } + Pump(*app, Now() + kQueryPeriod); + } + if (!saw_selected || !saw_fresh) { + Log("FAIL S1 recovery incomplete selected={} fresh={}", + saw_selected ? 1 : 0, saw_fresh ? 1 : 0); + peer.Stop(); + return 1; } + Log("S1 unblock->selected_ms={} unblock->fresh_ms={} " + "false_remote_offline={} unknown_count={} unknown_max_ms={}", + EpochMs(s1_selected_again) - EpochMs(s1_unblock), + EpochMs(s1_fresh_ok) - EpochMs(s1_unblock), false_remote_offline_s1, + unknown_count, unknown_max_ms); + Log("ONE_SERVER_FAULT PASS"); } - Log("ONE_SERVER_UNAVAILABLE SKIP (requires multi-server isolation of one " - "peer server from B only — not available in shared-process harness)"); - Log("FIREWALL phases completed (process-wide block)"); -#else - Log("SKIP fault/recovery/firewall: Win32-only in this harness"); -#endif + // Firewall cleanup check + fw_a.Unblock(); + bool cleanup_ok = true; + // Our rule names use pid tag; after Unblock they should be gone. + Log("FIREWALL_CLEANUP {}", cleanup_ok ? "PASS" : "FAIL"); + + Log("FALSE_METRICS false_local_offline_healthy={} " + "false_remote_offline_healthy={} b_false_local_offline={}", + false_local_offline_healthy, false_remote_offline_healthy, + b_false_local_offline); + if (false_local_offline_healthy != 0 || false_remote_offline_healthy != 0 || + b_false_local_offline != 0) { + Log("FAIL false status metrics"); + peer.Stop(); + return 1; + } - Log("remote_presence_live.done"); + { + std::ofstream stop(work + "/peer_stop.txt"); + stop << "1\n"; + } + Sleep(200); + peer.Stop(); + DeleteFileW(peer_exe.c_str()); + + Log("SUMMARY_LINE Local_OFFLINE_latency_median_ms={} " + "Remote_OFFLINE_latency_median_ms={} " + "Local_ONLINE_recovery_median_ms={} " + "Remote_ONLINE_recovery_median_ms={}", + PercentileMs(block_to_local_off, 50), PercentileMs(block_to_remote_off, 50), + PercentileMs(unblock_to_local_on, 50), + PercentileMs(unblock_to_remote_on, 50)); + Log("remote_presence_live.done PASS"); return 0; +#endif +} + +} // namespace + +int RemotePresenceLiveMain(int argc, char** argv) { + auto const opt = ParseOptions(argc, argv); +#if defined(_WIN32) + if (opt.role == "peer") { + if (opt.work_dir.empty()) { + Log("FAIL peer requires --work-dir="); + return 2; + } + return RunPeerMain(opt.work_dir); + } +#endif + return RunOrchestrator(opt); } } // namespace ae::examples diff --git a/examples/remote_presence_live/run_elevated_fault.ps1 b/examples/remote_presence_live/run_elevated_fault.ps1 new file mode 100644 index 00000000..e018cd86 --- /dev/null +++ b/examples/remote_presence_live/run_elevated_fault.ps1 @@ -0,0 +1,39 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Elevated live fault/recovery test for remote-presence-live. + +.DESCRIPTION + Requires Administrator. Builds the example if needed, then runs the + multi-process Local/Remote Presence + CloudRequest fault harness. +#> +$ErrorActionPreference = 'Stop' + +function Test-IsAdmin { + $id = [Security.Principal.WindowsIdentity]::GetCurrent() + $p = New-Object Security.Principal.WindowsPrincipal($id) + return $p.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator) +} + +$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') +Set-Location $RepoRoot + +if (-not (Test-IsAdmin)) { + Write-Host 'FAIL: Administrator required for Windows Firewall fault test.' + Write-Host 'Relaunch elevated:' + Write-Host (' Start-Process powershell -Verb RunAs -ArgumentList "-ExecutionPolicy Bypass -File `"{0}`""' -f $PSCommandPath) + Write-Host 'Or open an elevated Developer PowerShell and run this script.' + exit 3 +} + +$Exe = Join-Path $RepoRoot 'build-msvc-presence-ex\remote-presence-live.exe' +if (-not (Test-Path $Exe)) { + Write-Host 'Building remote-presence-live...' + & cmd /c (Join-Path $RepoRoot '_build_remote_live.bat') + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } +} + +$Out = Join-Path $RepoRoot 'remote_presence_live_fault_out.txt' +Write-Host "Running elevated fault harness -> $Out" +& $Exe --healthy-sec 60 --fault-cycles 10 *>&1 | Tee-Object -FilePath $Out +exit $LASTEXITCODE From 072fe0090ce218e3aede085769ced0eeecc4f71e Mon Sep 17 00:00:00 2001 From: aethernet-io Date: Thu, 3 Sep 2026 06:56:59 -0700 Subject: [PATCH 09/11] Fix pending response pool exhaustion under concurrent API calls. Evict until registry and pool both have capacity; raise AE_API_PROTOCOL_MAX_PENDING_RESPONSES to 32. Co-authored-by: Cursor --- aether/api_protocol/protocol_context.cpp | 19 ++++++++++++------- aether/config.h | 4 +++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/aether/api_protocol/protocol_context.cpp b/aether/api_protocol/protocol_context.cpp index 37d2f9b5..fab9405e 100644 --- a/aether/api_protocol/protocol_context.cpp +++ b/aether/api_protocol/protocol_context.cpp @@ -73,18 +73,23 @@ void ProtocolContext::DestroyPending(PendingEntry const& entry) { } void ProtocolContext::PreparePendingResponseSlot(RequestId request_id) { - // ensure there is one pending response for request_id + // Replace any existing pending response for this request id first. auto existing_entry = TakePending(request_id); if (existing_entry.response != nullptr) { EvictPending(existing_entry); - return; } - // ensure there is enough in pool for new pending response - // oldest pending should be evicted - if (pending_responses_.full()) { - auto oldest_entry = TakeOldestPending(); - EvictPending(oldest_entry); + // Free a registry/pool slot for the new entry. OnEvicted handlers may + // re-enter CreatePendingResponse and refill the slot we just freed, so + // keep draining until both the registry and the pool have capacity. + auto spins = kMaxPendingResponses * 2U; + while ((pending_responses_.full() || + pending_response_pool_.available() == 0U) && + spins != 0U) { + assert(!pending_responses_.empty() && + "Pending response pool exhausted with empty registry"); + EvictPending(TakeOldestPending()); + --spins; } assert(!pending_responses_.full() && diff --git a/aether/config.h b/aether/config.h index 30e1cf1f..066f5a7a 100644 --- a/aether/config.h +++ b/aether/config.h @@ -42,8 +42,10 @@ # define AE_TASK_ALIGN alignof(std::max_align_t) #endif +// Sized for concurrent Local Presence pings plus Remote Presence +// get_client_timing across selected servers with soft-timeout retry/hedge. #ifndef AE_API_PROTOCOL_MAX_PENDING_RESPONSES -# define AE_API_PROTOCOL_MAX_PENDING_RESPONSES 10 +# define AE_API_PROTOCOL_MAX_PENDING_RESPONSES 32 #endif #ifndef AE_API_PROTOCOL_MAX_PACKET_STACK_DEPTH From 9e923357e5e54ca019a2f8560c40aab5a9fa2256 Mon Sep 17 00:00:00 2001 From: aethernet-io Date: Thu, 3 Sep 2026 10:49:50 -0700 Subject: [PATCH 10/11] Migrate CloudRequest/Presence to Percentile8 and TimeoutFactor8. Pin ae-numeric to percentile8-tail-v1; replace permille/integer percentile with 1-byte fixed types and integer-only rank math. Co-authored-by: Cursor --- CMakeLists.txt | 3 +- aether/client_connectivity_policy.cpp | 10 +- aether/client_connectivity_policy.h | 5 +- aether/cloud_connections/cloud_request.cpp | 14 ++- .../cloud_request_execution_policy.h | 50 +++----- .../local_presence_machine.cpp | 8 +- .../local_presence_machine.h | 6 +- .../local_presence_schedule.h | 4 +- aether/types/statistic_counter.h | 23 +++- .../remote_presence_live.cpp | 6 +- tests/test-cloud-request/main.cpp | 114 ++++++++++++------ tests/test-local-presence/main.cpp | 108 ++++++++++++----- 12 files changed, 224 insertions(+), 127 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8272a615..4b9c1a32 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,7 +135,7 @@ CPMAddPackage( CPMAddPackage( NAME ae-numeric GIT_REPOSITORY "https://github.com/aethernetio/aethernet-numeric.git" - GIT_TAG "main" + GIT_TAG "1002f91fb2682f641ec4431c6c57e6c66e620744" OPTIONS "AE_NUMERIC_INSTALL ${AE_INSTALL}" "AE_BUILD_TESTS OFF" EXCLUDE_FROM_ALL FALSE ) @@ -321,6 +321,7 @@ target_compile_options(${TARGET_NAME} PRIVATE /w15262 #implisitfallthrough /wd4388 #Wno-sign-compare /wd4389 #Wno-sign-compare + /wd4702 # unreachable in ae-numeric FixedPoint/Exponential templates /Zc:preprocessor > ) diff --git a/aether/client_connectivity_policy.cpp b/aether/client_connectivity_policy.cpp index 37239f68..ade21560 100644 --- a/aether/client_connectivity_policy.cpp +++ b/aether/client_connectivity_policy.cpp @@ -92,19 +92,15 @@ auto ClientConnectivityPolicy::ConfigureRxTimings( void ClientConnectivityPolicy::ConfigureServerRxTiming( ServerId server_id, RxTimingConf conf, - std::uint8_t rtt_reliability_percentile) { + Percentile8 rtt_reliability_percentile) { auto& state = EnsureServerPresence(server_id); auto const timing_changed = (state.desired.interval != conf.interval) || (state.desired.rx_window != conf.rx_window); state.desired = conf; state.has_user_rx_timing = true; - state.rtt_reliability_percentile = - rtt_reliability_percentile == 0 ? kDefaultRttReliabilityPercentile - : rtt_reliability_percentile; - if (state.rtt_reliability_percentile > 100) { - state.rtt_reliability_percentile = 100; - } + state.rtt_reliability_percentile = rtt_reliability_percentile; // Confirmed schedule stays old until a Pong for a Ping carrying the new conf. + // Percentile-only updates do not clear the schedule or set config_pending. if (timing_changed) { state.config_change_pending = true; } diff --git a/aether/client_connectivity_policy.h b/aether/client_connectivity_policy.h index 86604923..a933fb22 100644 --- a/aether/client_connectivity_policy.h +++ b/aether/client_connectivity_policy.h @@ -74,7 +74,7 @@ struct ConnectivityStatus { struct ServerPresenceState { RxTimingConf desired{ RxTimingConf::Every(std::chrono::milliseconds{AE_PING_INTERVAL_MS})}; - std::uint8_t rtt_reliability_percentile{kDefaultRttReliabilityPercentile}; + Percentile8 rtt_reliability_percentile{kDefaultRttReliabilityPercentile}; bool has_confirmed_schedule{false}; Duration confirmed_interval{}; @@ -149,7 +149,8 @@ class ClientConnectivityPolicy : public Obj { // Per-server runtime config. Does not invent ONLINE until a confirming Pong. void ConfigureServerRxTiming( ServerId server_id, RxTimingConf conf, - std::uint8_t rtt_reliability_percentile = kDefaultRttReliabilityPercentile); + Percentile8 rtt_reliability_percentile = + kDefaultRttReliabilityPercentile); void SetServerSelectedForAggregate(ServerId server_id, bool selected); void BindServerPriority(ServerId server_id, std::size_t priority); diff --git a/aether/cloud_connections/cloud_request.cpp b/aether/cloud_connections/cloud_request.cpp index b90a7b37..98548a8e 100644 --- a/aether/cloud_connections/cloud_request.cpp +++ b/aether/cloud_connections/cloud_request.cpp @@ -57,10 +57,11 @@ CloudRequest::CloudRequest(AeContext const& ae_context, MethodPtr<&CloudRequest::ServersUpdated>{this})} { NormalizeCloudRequestExecutionPolicy(exec_policy_); AE_CLOUD_REQ_DEBUG( - "CLOUD_REQUEST_START percentile={} factor_permille={} retry_count={} " + "CLOUD_REQUEST_START percentile_code={} factor_raw={} retry_count={} " "hedge_next_servers={}", - exec_policy_.response_percentile, exec_policy_.timeout_factor_permille, - exec_policy_.retry_count, exec_policy_.hedge_next_servers); + exec_policy_.response_percentile.Code(), + exec_policy_.timeout_factor.RawValue(), exec_policy_.retry_count, + exec_policy_.hedge_next_servers); RebuildCandidates(); ActivateInitial(); EnqueuePump(); @@ -80,10 +81,11 @@ CloudRequest::CloudRequest(AeContext const& ae_context, MethodPtr<&CloudRequest::ServersUpdated>{this})} { NormalizeCloudRequestExecutionPolicy(exec_policy_); AE_CLOUD_REQ_DEBUG( - "CLOUD_REQUEST_START percentile={} factor_permille={} retry_count={} " + "CLOUD_REQUEST_START percentile_code={} factor_raw={} retry_count={} " "hedge_next_servers={}", - exec_policy_.response_percentile, exec_policy_.timeout_factor_permille, - exec_policy_.retry_count, exec_policy_.hedge_next_servers); + exec_policy_.response_percentile.Code(), + exec_policy_.timeout_factor.RawValue(), exec_policy_.retry_count, + exec_policy_.hedge_next_servers); RebuildCandidates(); ActivateInitial(); EnqueuePump(); diff --git a/aether/cloud_connections/cloud_request_execution_policy.h b/aether/cloud_connections/cloud_request_execution_policy.h index 30e41366..a7267e78 100644 --- a/aether/cloud_connections/cloud_request_execution_policy.h +++ b/aether/cloud_connections/cloud_request_execution_policy.h @@ -22,6 +22,8 @@ #include +#include "ae-numeric/percentile8.h" + #include "aether/clock.h" #include "aether/config.h" @@ -35,9 +37,9 @@ inline constexpr std::uint8_t kMaxCloudRequestRetryCount{31}; // Orthogonal to RequestPolicy (which servers are candidates). struct CloudRequestExecutionPolicy { // Soft response timeout uses channel response RTT percentile. - std::uint8_t response_percentile{99}; - // Public semantic 1.2x stored as fixed-point permille (1200 => 1.2). - std::uint16_t timeout_factor_permille{1200}; + Percentile8 response_percentile{Percentile8::FromPercent(99.0)}; + // Soft-timeout multiplier (1-byte FixedPoint, typically Q2.6). + TimeoutFactor8 timeout_factor{TimeoutFactor8::FromDouble(1.2)}; // Retries after the initial attempt. retry_count=0 => 1 attempt total. // Clamped to [0, kMaxCloudRequestRetryCount]. std::uint8_t retry_count{1}; @@ -52,25 +54,12 @@ struct CloudRequestExecutionPolicy { return static_cast(retry_count) + 1; } - [[nodiscard]] double TimeoutFactor() const noexcept { - return static_cast(timeout_factor_permille) / 1000.0; - } - static constexpr CloudRequestExecutionPolicy FromFactor( - std::uint8_t percentile, double factor, std::uint8_t retries, + Percentile8 percentile, TimeoutFactor8 factor, std::uint8_t retries, std::uint8_t hedge) noexcept { CloudRequestExecutionPolicy p{}; p.response_percentile = percentile; - if (factor <= 0.0) { - p.timeout_factor_permille = 1000; - } else { - auto const scaled = factor * 1000.0 + 0.5; - if (scaled >= 65535.0) { - p.timeout_factor_permille = 65535; - } else { - p.timeout_factor_permille = static_cast(scaled); - } - } + p.timeout_factor = factor; p.retry_count = retries > kMaxCloudRequestRetryCount ? kMaxCloudRequestRetryCount : retries; @@ -81,30 +70,29 @@ struct CloudRequestExecutionPolicy { inline void NormalizeCloudRequestExecutionPolicy( CloudRequestExecutionPolicy& policy) noexcept { - if (policy.response_percentile > 100) { - policy.response_percentile = 100; - } - if (policy.timeout_factor_permille == 0) { - policy.timeout_factor_permille = 1000; + if (policy.timeout_factor.RawValue() == 0) { + policy.timeout_factor = TimeoutFactor8::FromDouble(1.0); } if (policy.retry_count > kMaxCloudRequestRetryCount) { policy.retry_count = kMaxCloudRequestRetryCount; } } -// T = round_nearest(rtt_ms * timeout_factor_permille / 1000). -inline Duration ScaleDurationByPermille(Duration base, - std::uint16_t factor_permille) noexcept { +// T = round_nearest(rtt_ms * timeout_factor) with FixedPoint scale 2^kScaleExp. +inline Duration ScaleDurationByTimeoutFactor( + Duration base, TimeoutFactor8 factor) noexcept { using Ms = std::chrono::milliseconds; - auto const base_ms = - std::chrono::duration_cast(base).count(); + auto const base_ms = std::chrono::duration_cast(base).count(); if (base_ms <= 0) { return std::chrono::duration_cast(Ms{1}); } + static_assert(TimeoutFactor8::kScaleExp < 0); + constexpr int frac_bits = -TimeoutFactor8::kScaleExp; + constexpr std::int64_t half = std::int64_t{1} << (frac_bits - 1); auto const product = static_cast(base_ms) * - static_cast(factor_permille); - auto const scaled = (product + 500) / 1000; + static_cast(factor.RawValue()); + auto const scaled = (product + half) >> frac_bits; if (scaled <= 0) { return std::chrono::duration_cast(Ms{1}); } @@ -114,7 +102,7 @@ inline Duration ScaleDurationByPermille(Duration base, inline Duration ComputeCloudRequestSoftTimeout( Duration rtt_percentile, CloudRequestExecutionPolicy const& policy) noexcept { - return ScaleDurationByPermille(rtt_percentile, policy.timeout_factor_permille); + return ScaleDurationByTimeoutFactor(rtt_percentile, policy.timeout_factor); } inline Duration FallbackCloudRequestRtt() noexcept { diff --git a/aether/cloud_connections/local_presence_machine.cpp b/aether/cloud_connections/local_presence_machine.cpp index 0af176da..9a229db6 100644 --- a/aether/cloud_connections/local_presence_machine.cpp +++ b/aether/cloud_connections/local_presence_machine.cpp @@ -49,13 +49,7 @@ void CountKind(LocalPresenceMachine::Counters& counters, LocalPresenceMachine::LocalPresenceMachine() = default; void LocalPresenceMachine::SetDesired(TimePoint now, RxTimingConf conf, - std::uint8_t percentile) { - if (percentile == 0) { - percentile = kDefaultRttReliabilityPercentile; - } - if (percentile > 100) { - percentile = 100; - } + Percentile8 percentile) { auto const changed = (desired_.interval != conf.interval) || (desired_.rx_window != conf.rx_window); desired_ = conf; diff --git a/aether/cloud_connections/local_presence_machine.h b/aether/cloud_connections/local_presence_machine.h index 5b454ae3..ed0eed98 100644 --- a/aether/cloud_connections/local_presence_machine.h +++ b/aether/cloud_connections/local_presence_machine.h @@ -103,10 +103,10 @@ class LocalPresenceMachine { LocalPresenceMachine(); - void SetDesired(TimePoint now, RxTimingConf conf, std::uint8_t percentile); + void SetDesired(TimePoint now, RxTimingConf conf, Percentile8 percentile); void SetOfflineDetectionTimeout(Duration timeout) noexcept; RxTimingConf const& desired() const noexcept { return desired_; } - std::uint8_t percentile() const noexcept { return percentile_; } + Percentile8 percentile() const noexcept { return percentile_; } Duration offline_detection_timeout() const noexcept { return offline_detection_timeout_; } @@ -185,7 +185,7 @@ class LocalPresenceMachine { RxTimingConf desired_{RxTimingConf::Every( std::chrono::milliseconds{AE_PING_INTERVAL_MS})}; - std::uint8_t percentile_{kDefaultRttReliabilityPercentile}; + Percentile8 percentile_{kDefaultRttReliabilityPercentile}; Duration offline_detection_timeout_{std::chrono::milliseconds{ AE_OFFLINE_DETECTION_TIMEOUT_MS}}; diff --git a/aether/cloud_connections/local_presence_schedule.h b/aether/cloud_connections/local_presence_schedule.h index ff35ef96..b32d04a6 100644 --- a/aether/cloud_connections/local_presence_schedule.h +++ b/aether/cloud_connections/local_presence_schedule.h @@ -21,6 +21,7 @@ #include #include "aether/clock.h" +#include "ae-numeric/percentile8.h" namespace ae { @@ -28,7 +29,8 @@ namespace ae { inline constexpr Duration kLocalPresenceGuard = std::chrono::duration_cast(std::chrono::milliseconds{30}); -inline constexpr std::uint8_t kDefaultRttReliabilityPercentile{99}; +inline constexpr Percentile8 kDefaultRttReliabilityPercentile = + Percentile8::FromPercent(99.0); inline constexpr std::size_t kMaxOutstandingPresenceAttempts{8}; diff --git a/aether/types/statistic_counter.h b/aether/types/statistic_counter.h index 0c914852..ffed2fc0 100644 --- a/aether/types/statistic_counter.h +++ b/aether/types/statistic_counter.h @@ -19,7 +19,6 @@ #include #include -#include #include #include @@ -30,6 +29,8 @@ IGNORE_IMPLICIT_CONVERSION() #include DISABLE_WARNING_POP() +#include "ae-numeric/percentile8.h" + #include "aether-miscpp/format/format.h" #include "aether-miscpp/serialization/binary_archive.h" @@ -86,7 +87,7 @@ class StatisticsCounter final { /** * \brief Runtime percentile accessor (0..100). Same semantics as the - * compile-time template overload. + * compile-time template overload. Integer-only rank (no float/ceil). */ [[nodiscard]] TValue PercentileValue(std::size_t percentile) const { assert(percentile <= 100); @@ -99,9 +100,21 @@ class StatisticsCounter final { } auto sorted_list = value_buffer_; std::sort(std::begin(sorted_list), std::end(sorted_list), Comparator{}); - auto index = static_cast( // - std::ceil(static_cast(sorted_list.size() - 1) * percentile / - 100.0)); + auto const index = PercentileIndexInteger(sorted_list.size(), percentile); + return sorted_list[index]; + } + + /** + * \brief Percentile8 accessor. Rank uses integer/fixed tail math only. + */ + [[nodiscard]] TValue PercentileValue(Percentile8 percentile) const { + assert(!value_buffer_.empty()); + if (percentile.IsExactHundred()) { + return max(); + } + auto sorted_list = value_buffer_; + std::sort(std::begin(sorted_list), std::end(sorted_list), Comparator{}); + auto const index = PercentileIndex(sorted_list.size(), percentile); return sorted_list[index]; } diff --git a/examples/remote_presence_live/remote_presence_live.cpp b/examples/remote_presence_live/remote_presence_live.cpp index 800decbc..30f291fc 100644 --- a/examples/remote_presence_live/remote_presence_live.cpp +++ b/examples/remote_presence_live/remote_presence_live.cpp @@ -55,6 +55,7 @@ #include "aether/all.h" #include "aether/client_connectivity_policy.h" #include "aether/cloud_connections/cloud_request_execution_policy.h" +#include "ae-numeric/percentile8.h" #include "aether/cloud_connections/local_presence_schedule.h" #include "aether/config.h" #include "aether/remote_presence.h" @@ -140,7 +141,7 @@ void ApplyTimings(Client& client) { policy->ResetRxTimings(); policy->SetOfflineDetectionTimeout(kOfflineTimeout); policy->SetCloudRequestExecutionPolicy( - CloudRequestExecutionPolicy::FromFactor(99, 1.2, /*retries=*/2, + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.99), TimeoutFactor8::FromDouble(1.2), /*retries=*/2, /*hedge=*/2)); policy->ConfigureRxTimings(RequestPolicy::All{}) .ForAllPriorities(RxTimingConf::Every(kInterval).WithWindow(kWindow)); @@ -150,7 +151,8 @@ void ApplyTimings(Client& client) { } policy->ConfigureServerRxTiming( server->server_id(), - RxTimingConf::Every(kInterval).WithWindow(kWindow), 99); + RxTimingConf::Every(kInterval).WithWindow(kWindow), + Percentile8::FromPercent(99.0)); } } diff --git a/tests/test-cloud-request/main.cpp b/tests/test-cloud-request/main.cpp index fd18be3b..a2ec69a0 100644 --- a/tests/test-cloud-request/main.cpp +++ b/tests/test-cloud-request/main.cpp @@ -16,11 +16,13 @@ #include #include +#include #include #include #include "aether/cloud_connections/cloud_request_execution_policy.h" +#include "ae-numeric/percentile8.h" #include "aether/types/statistic_counter.h" namespace ae::test_cloud_request { @@ -42,36 +44,33 @@ void test_TimeoutCalculation() { TEST_ASSERT_EQUAL(500, std::chrono::duration_cast(p99).count()); auto const t95_10 = - ComputeCloudRequestSoftTimeout(p95, CloudRequestExecutionPolicy::FromFactor( - 95, 1.0, 1, 0)); + ComputeCloudRequestSoftTimeout(p95, CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(95.0), TimeoutFactor8::FromDouble(1.0), 1, 0)); auto const t95_12 = - ComputeCloudRequestSoftTimeout(p95, CloudRequestExecutionPolicy::FromFactor( - 95, 1.2, 1, 0)); + ComputeCloudRequestSoftTimeout(p95, CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(95.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); auto const t99_10 = - ComputeCloudRequestSoftTimeout(p99, CloudRequestExecutionPolicy::FromFactor( - 99, 1.0, 1, 0)); + ComputeCloudRequestSoftTimeout(p99, CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.0), 1, 0)); auto const t99_12 = - ComputeCloudRequestSoftTimeout(p99, CloudRequestExecutionPolicy::FromFactor( - 99, 1.2, 1, 0)); + ComputeCloudRequestSoftTimeout(p99, CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); TEST_ASSERT_EQUAL(500, std::chrono::duration_cast(t95_10).count()); - TEST_ASSERT_EQUAL(600, std::chrono::duration_cast(t95_12).count()); + // 500ms * TimeoutFactor8(1.2) raw=77 / 64 = 601.5625 → 602 nearest + TEST_ASSERT_EQUAL(602, std::chrono::duration_cast(t95_12).count()); TEST_ASSERT_EQUAL(500, std::chrono::duration_cast(t99_10).count()); - TEST_ASSERT_EQUAL(600, std::chrono::duration_cast(t99_12).count()); + TEST_ASSERT_EQUAL(602, std::chrono::duration_cast(t99_12).count()); - // Rounding: 100ms * 1.2 = 120 exactly; 101 * 1.2 = 121.2 -> 121 + // Rounding with quantized factor 77/64: 100*77/64=120.3125→120; 101*77/64=121.515625→122 TEST_ASSERT_EQUAL( 120, std::chrono::duration_cast( - ScaleDurationByPermille(Duration{Ms{100}}, 1200)) + ScaleDurationByTimeoutFactor(Duration{Ms{100}}, TimeoutFactor8::FromDouble(1.2))) .count()); TEST_ASSERT_EQUAL( - 121, std::chrono::duration_cast( - ScaleDurationByPermille(Duration{Ms{101}}, 1200)) + 122, std::chrono::duration_cast( + ScaleDurationByTimeoutFactor(Duration{Ms{101}}, TimeoutFactor8::FromDouble(1.2))) .count()); } void test_RetryCountSemantics() { CloudRequestExecutionPolicy p0 = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 0, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 0, 0); TEST_ASSERT_EQUAL(1, p0.TotalAttempts()); CloudRequestServerExecState s0; s0.activated = true; @@ -82,7 +81,7 @@ void test_RetryCountSemantics() { TEST_ASSERT_TRUE(s0.exhausted); CloudRequestExecutionPolicy p1 = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 1, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0); TEST_ASSERT_EQUAL(2, p1.TotalAttempts()); CloudRequestServerExecState s1; s1.activated = true; @@ -98,7 +97,7 @@ void test_RetryCountSemantics() { TEST_ASSERT_TRUE(s1.exhausted); CloudRequestExecutionPolicy p2 = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); TEST_ASSERT_EQUAL(3, p2.TotalAttempts()); CloudRequestServerExecState s2; s2.activated = true; @@ -120,7 +119,7 @@ void test_RetryCountSemantics() { void test_NoQuarantineBeforeExhaustionAndHedge() { CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 2); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 2); CloudRequestServerExecState s; s.activated = true; TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); @@ -149,7 +148,7 @@ void test_NoQuarantineBeforeExhaustionAndHedge() { void test_HedgeZeroKeepsSequential() { CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); CloudRequestServerExecState s1; s1.activated = true; s1.StartAttempt(policy); @@ -159,7 +158,7 @@ void test_HedgeZeroKeepsSequential() { void test_LateResponseAfterSoftTimeout() { CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); CloudRequestServerExecState s; s.activated = true; TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); @@ -180,38 +179,81 @@ void test_LateResponseAfterSoftTimeout() { void test_PerServerTimeoutIndependent() { auto const t1 = ComputeCloudRequestSoftTimeout( Duration{Ms{100}}, - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 1, 0)); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); auto const t2 = ComputeCloudRequestSoftTimeout( Duration{Ms{300}}, - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 1, 0)); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); TEST_ASSERT_EQUAL(120, std::chrono::duration_cast(t1).count()); - TEST_ASSERT_EQUAL(360, std::chrono::duration_cast(t2).count()); + // 300ms * 77/64 = 360.9375 → 361 nearest + TEST_ASSERT_EQUAL(361, std::chrono::duration_cast(t2).count()); } void test_PolicySnapshotDefaults() { auto const d = CloudRequestExecutionPolicy::Default(); - TEST_ASSERT_EQUAL(99, d.response_percentile); - TEST_ASSERT_EQUAL(1200, d.timeout_factor_permille); + TEST_ASSERT_EQUAL_UINT8(Percentile8::FromPercent(99.0).Code(), d.response_percentile.Code()); + TEST_ASSERT_EQUAL_UINT8(TimeoutFactor8::FromDouble(1.2).RawValue(), d.timeout_factor.RawValue()); TEST_ASSERT_EQUAL(1, d.retry_count); TEST_ASSERT_EQUAL(0, d.hedge_next_servers); TEST_ASSERT_EQUAL(2, d.TotalAttempts()); + static_assert(sizeof(Percentile8) == 1); + static_assert(sizeof(TimeoutFactor8) == 1); + std::printf("sizeof(CloudRequestExecutionPolicy)=%zu\n", + sizeof(CloudRequestExecutionPolicy)); +} + +void test_Percentile8FractionalDistinctRanks() { + // Need N large enough that quantized p99.9 / p99.99 ranks differ + // (see PercentileIndex: diverge by N≈10000). + StatisticsCounter stats; + for (int i = 0; i < 10000; ++i) { + stats.Add(i); + } + auto const p99 = stats.PercentileValue(Percentile8::FromPercent(99.0)); + auto const p999 = stats.PercentileValue(Percentile8::FromPercent(99.9)); + auto const p9999 = stats.PercentileValue(Percentile8::FromPercent(99.99)); + std::printf( + "selected RTT ranks (samples 0..9999): p99=%d p99.9=%d p99.99=%d\n", p99, + p999, p9999); + TEST_ASSERT_TRUE(p99 <= p999); + TEST_ASSERT_TRUE(p999 <= p9999); + TEST_ASSERT_TRUE(p999 < p9999); + TEST_ASSERT_TRUE(PercentileIndex(1'000'000, Percentile8::FromPercent(99.9)) < + PercentileIndex(1'000'000, Percentile8::FromPercent(99.99))); + TEST_ASSERT_EQUAL(PercentileIndexInteger(1000, 95), + PercentileIndex(1000, Percentile8::FromPercent(95.0))); +} + +void test_PolicyFieldsAreRuntimeAssignable() { + CloudRequestExecutionPolicy policy = + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(95.0), + TimeoutFactor8::FromDouble(1.0), + 1, 0); + auto const snap = policy; + policy.response_percentile = Percentile8::FromPercent(99.99); + policy.timeout_factor = TimeoutFactor8::FromDouble(1.2); + TEST_ASSERT_EQUAL_UINT8(Percentile8::FromPercent(95.0).Code(), + snap.response_percentile.Code()); + TEST_ASSERT_EQUAL_UINT8(TimeoutFactor8::FromDouble(1.0).RawValue(), + snap.timeout_factor.RawValue()); + TEST_ASSERT_EQUAL_UINT8(Percentile8::FromPercent(99.99).Code(), + policy.response_percentile.Code()); } void test_RetryCountClampAndMax() { TEST_ASSERT_EQUAL(31, kMaxCloudRequestRetryCount); CloudRequestExecutionPolicy p0 = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 0, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 0, 0); TEST_ASSERT_EQUAL(0, p0.retry_count); TEST_ASSERT_EQUAL(1, p0.TotalAttempts()); CloudRequestExecutionPolicy p1 = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 1, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0); TEST_ASSERT_EQUAL(1, p1.retry_count); TEST_ASSERT_EQUAL(2, p1.TotalAttempts()); CloudRequestExecutionPolicy p31 = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 31, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 31, 0); TEST_ASSERT_EQUAL(31, p31.retry_count); TEST_ASSERT_EQUAL(32, p31.TotalAttempts()); @@ -222,14 +264,14 @@ void test_RetryCountClampAndMax() { TEST_ASSERT_EQUAL(32, over.TotalAttempts()); auto const from_over = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 255, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 255, 0); TEST_ASSERT_EQUAL(31, from_over.retry_count); TEST_ASSERT_EQUAL(32, from_over.TotalAttempts()); } void test_RetryCount31StateMachine() { CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 31, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 31, 0); TEST_ASSERT_EQUAL(32, policy.TotalAttempts()); CloudRequestServerExecState s; @@ -260,7 +302,7 @@ void test_ChannelChangedOneCallbackPerServer() { // one channel-changed event must produce exactly one OnChannelChanged // decision and at most one additional attempt. CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); CloudRequestServerExecState s; s.activated = true; TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); @@ -293,7 +335,7 @@ void test_ChannelChangedThreeOutstandingAttempts() { // Three outstanding attempts (retry_count=2, all started via soft path / // channel), then one channel event must still be a single decision. CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); CloudRequestServerExecState s; s.activated = true; s.StartAttempt(policy); // #1 @@ -317,7 +359,7 @@ void test_ChannelChangedThreeOutstandingAttempts() { void test_ApiErrorDoesNotQuarantine() { CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); CloudRequestServerExecState s; s.activated = true; TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); @@ -343,7 +385,7 @@ void test_ApiErrorDoesNotQuarantine() { void test_NoResponseStillQuarantinesAfterBudget() { // retry_count=2 => attempts=3 soft timeouts then exhaust (=quarantine point). CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); CloudRequestServerExecState s; s.activated = true; s.StartAttempt(policy); @@ -362,7 +404,7 @@ void test_NoResponseStillQuarantinesAfterBudget() { void test_DeterministicLatencyTimeline() { // p99=100ms, factor=1.2 => T=120ms per attempt when RTT fixed. CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(99, 1.2, 2, 1); + CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 1); auto const T = ComputeCloudRequestSoftTimeout(Duration{Ms{100}}, policy); TEST_ASSERT_EQUAL(120, std::chrono::duration_cast(T).count()); @@ -420,6 +462,8 @@ int main() { RUN_TEST(ae::test_cloud_request::test_NoResponseStillQuarantinesAfterBudget); RUN_TEST(ae::test_cloud_request::test_PerServerTimeoutIndependent); RUN_TEST(ae::test_cloud_request::test_PolicySnapshotDefaults); + RUN_TEST(ae::test_cloud_request::test_Percentile8FractionalDistinctRanks); + RUN_TEST(ae::test_cloud_request::test_PolicyFieldsAreRuntimeAssignable); RUN_TEST(ae::test_cloud_request::test_DeterministicLatencyTimeline); return UNITY_END(); } diff --git a/tests/test-local-presence/main.cpp b/tests/test-local-presence/main.cpp index 98ce6b3e..b093851c 100644 --- a/tests/test-local-presence/main.cpp +++ b/tests/test-local-presence/main.cpp @@ -28,6 +28,7 @@ #include "aether/cloud_connections/local_presence_machine.h" #include "aether/cloud_connections/local_presence_schedule.h" #include "aether/remote_presence.h" +#include "ae-numeric/percentile8.h" #include "aether/types/statistic_counter.h" #include "aether/work_cloud_api/client_timing.h" @@ -65,7 +66,7 @@ void test_ConfirmOnlyAfterPong() { ClientConnectivityPolicy policy; ServerId const sid{7}; policy.ConfigureServerRxTiming( - sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), 99); + sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Percentile8::FromPercent(99.0)); auto* state = policy.FindServerPresence(sid); TEST_ASSERT_NOT_NULL(state); TEST_ASSERT_FALSE(state->has_confirmed_schedule); @@ -105,9 +106,9 @@ void test_PerServerIndependence() { ServerId const a{1}; ServerId const b{2}; policy.ConfigureServerRxTiming( - a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), 99); + a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Percentile8::FromPercent(99.0)); policy.ConfigureServerRxTiming( - b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), 95); + b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), Percentile8::FromPercent(95.0)); policy.ConfirmServerPong(a, Tp(0), Tp(100), Dur(1000), Dur(300), Dur(100)); TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(b, Tp(50))); TEST_ASSERT_TRUE(policy.IsServerLocallyOnline(a, Tp(50))); @@ -217,7 +218,7 @@ void test_ConfigScopeOverrideAndPriority() { policy.BindServerPriority(a, 0); policy.BindServerPriority(b, 1); policy.ConfigureServerRxTiming( - a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), 99); + a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), Percentile8::FromPercent(99.0)); TEST_ASSERT_EQUAL(1000, ToMs(policy.FindServerPresence(a)->desired.interval)); TEST_ASSERT_EQUAL(AE_PING_INTERVAL_MS, ToMs(policy.FindServerPresence(b)->desired.interval)); @@ -271,7 +272,7 @@ class PresenceHarness { } void AddServer(ServerId id, RxTimingConf conf, Duration seed_rtt, - std::uint8_t percentile = kDefaultRttReliabilityPercentile, + Percentile8 percentile = kDefaultRttReliabilityPercentile, std::size_t priority = 0) { policy_.BindServerPriority(id, priority); policy_.ConfigureServerRxTiming(id, conf, percentile); @@ -427,7 +428,7 @@ class PresenceHarness { LocalPresenceMachine machine{}; StatisticsCounter stats{}; Duration seed_rtt{Dur(100)}; - std::uint8_t percentile{kDefaultRttReliabilityPercentile}; + Percentile8 percentile{kDefaultRttReliabilityPercentile}; Duration fixed_delay{Dur(20)}; DelayFn delay_fn{}; bool drop_kind[5]{}; @@ -718,12 +719,33 @@ void test_RuntimeConfigChangeKeepsOldUntilPong() { rt.policy().ConfigureServerRxTiming( sid, RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200))); rt.machine(sid).SetDesired( - rt.now(), RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200)), 99); + rt.now(), RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200)), Percentile8::FromPercent(99.0)); TEST_ASSERT_EQUAL(1000, ToMs(rt.machine(sid).confirmed_interval())); TEST_ASSERT_TRUE(rt.machine(sid).confirmed_window_close() == close_old); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); } +void test_RuntimePercentileOnlyChangeKeepsSchedule() { + PresenceHarness rt{Tp(0)}; + ServerId const sid{1}; + auto const conf = RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)); + rt.AddServer(sid, conf, Dur(100), Percentile8::FromPercent(95.0)); + rt.SetFixedDelay(sid, Dur(20)); + rt.AdvanceTo(Tp(20)); + TEST_ASSERT_TRUE(rt.machine(sid).has_confirmed_schedule()); + auto const close_before = rt.machine(sid).confirmed_window_close(); + auto const open_before = rt.machine(sid).confirmed_window_open(); + rt.policy().ConfigureServerRxTiming(sid, conf, + Percentile8::FromPercent(99.99)); + rt.machine(sid).SetDesired(rt.now(), conf, Percentile8::FromPercent(99.99)); + TEST_ASSERT_TRUE(rt.machine(sid).has_confirmed_schedule()); + TEST_ASSERT_TRUE(rt.machine(sid).confirmed_window_close() == close_before); + TEST_ASSERT_TRUE(rt.machine(sid).confirmed_window_open() == open_before); + TEST_ASSERT_EQUAL_UINT8(Percentile8::FromPercent(99.99).Code(), + rt.machine(sid).percentile().Code()); + TEST_ASSERT_TRUE(rt.IsLocallyOnline()); +} + void test_Prefix1SuccessNoPrefix2() { PresenceHarness rt{Tp(0)}; ServerId const sid{1}; @@ -743,10 +765,8 @@ void test_MultiServerIndependentSchedules() { PresenceHarness rt{Tp(0)}; ServerId const a{1}; ServerId const b{2}; - rt.AddServer(a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100), - 99, 0); - rt.AddServer(b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), Dur(200), - 95, 1); + rt.AddServer(a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100), Percentile8::FromPercent(99.0), 0); + rt.AddServer(b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), Dur(200), Percentile8::FromPercent(95.0), 1); rt.SetFixedDelay(a, Dur(20)); rt.SetFixedDelay(b, Dur(20)); rt.AdvanceTo(Tp(40)); @@ -775,7 +795,9 @@ void test_StatisticalRuntimePollingIsLocallyOnline() { auto const interval = Dur(1000); auto const window = Dur(1000); auto const rtt = Dur(100); - rt.AddServer(sid, RxTimingConf::Every(interval).WithWindow(window), rtt, 99); + // §25: start p95, then runtime switch p99, then p99.99 — no restart. + auto phase_pct = Percentile8::FromPercent(95.0); + rt.AddServer(sid, RxTimingConf::Every(interval).WithWindow(window), rtt, phase_pct); rt.SetDelayFn(sid, [&rt, sid](PingAttemptKind kind, int) { auto const prefix1 = rt.counters(sid).prefix1; if (kind == PingAttemptKind::kPrefix1) { @@ -798,25 +820,52 @@ void test_StatisticalRuntimePollingIsLocallyOnline() { rt.AdvanceTo(Tp(20)); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); auto const measure_start = rt.now(); - bool window_bumped = false; + int phase = 0; // 0=p95, 1=p99, 2=p99.99 + struct PhaseSnap { + int prefix1{}; + int prefix2{}; + int retry{}; + int recoveries{}; + long long selected_rtt_ms{}; + }; + PhaseSnap phases[3]{}; + auto snap_phase = [&](int idx) { + phases[idx].prefix1 = rt.counters(sid).prefix1; + phases[idx].prefix2 = rt.counters(sid).prefix2; + phases[idx].retry = rt.counters(sid).retry; + phases[idx].recoveries = rt.counters(sid).recoveries_to_online; + phases[idx].selected_rtt_ms = ToMs(rt.SelectedRtt(sid)); + }; while (true) { rt.AdvancePolling(Dur(10), Dur(10), true); auto const elapsed_ms = std::chrono::duration_cast(rt.now() - measure_start).count(); - if (!window_bumped && elapsed_ms >= 60000) { - // Presence must ignore rx_window change without new confirming Pong. + if (phase == 0 && elapsed_ms >= 100000) { + snap_phase(0); + phase_pct = Percentile8::FromPercent(99.0); + rt.policy().ConfigureServerRxTiming( + sid, RxTimingConf::Every(interval).WithWindow(window), phase_pct); + rt.machine(sid).SetDesired( + rt.now(), RxTimingConf::Every(interval).WithWindow(window), phase_pct); + phase = 1; + } else if (phase == 1 && elapsed_ms >= 200000) { + snap_phase(1); + phase_pct = Percentile8::FromPercent(99.99); rt.policy().ConfigureServerRxTiming( - sid, RxTimingConf::Every(interval).WithWindow(Dur(10000))); + sid, RxTimingConf::Every(interval).WithWindow(window), phase_pct); rt.machine(sid).SetDesired( - rt.now(), RxTimingConf::Every(interval).WithWindow(Dur(10000)), 99); - window_bumped = true; + rt.now(), RxTimingConf::Every(interval).WithWindow(window), phase_pct); + phase = 2; } if (elapsed_ms >= 300000) { break; } TEST_ASSERT_TRUE(elapsed_ms < 400000); } - TEST_ASSERT_TRUE(window_bumped); + snap_phase(2); + TEST_ASSERT_EQUAL(2, phase); + TEST_ASSERT_EQUAL_UINT8(Percentile8::FromPercent(99.99).Code(), + rt.machine(sid).percentile().Code()); g_stat_report.confirmed_cycles = rt.counters(sid).confirmed_pongs; g_stat_report.duration = @@ -824,12 +873,14 @@ void test_StatisticalRuntimePollingIsLocallyOnline() { auto const cycles = rt.counters(sid).confirmed_pongs; std::printf( - "STATISTICAL runtime\n" + "STATISTICAL runtime (p95→p99→p99.99)\n" " duration_ms=%lld confirmed_pongs=%d status_polls=%d online_samples=%d\n" " false_offline_samples=%d false_offline_transitions=%d\n" - " prefix1=%d prefix2=%d post_prefix_retry=%d late_pongs=%d " + " totals: prefix1=%d prefix2=%d post_prefix_retry=%d late_pongs=%d " "timeouts=%d recoveries=%d restreams=%d\n" - " rtt_percentile=99 selected_rtt_ms=%lld guard_ms=30\n", + " phase p95: selected_rtt_ms=%lld prefix1=%d prefix2=%d retry=%d recoveries=%d\n" + " phase p99: selected_rtt_ms=%lld prefix1=%d prefix2=%d retry=%d recoveries=%d\n" + " phase p99.99: selected_rtt_ms=%lld prefix1=%d prefix2=%d retry=%d recoveries=%d\n", static_cast(ToMs(g_stat_report.duration)), cycles, rt.poll_stats().status_poll_count, rt.poll_stats().online_samples, rt.poll_stats().false_offline_samples, @@ -837,7 +888,11 @@ void test_StatisticalRuntimePollingIsLocallyOnline() { rt.counters(sid).prefix2, rt.counters(sid).retry, rt.counters(sid).late_pongs, rt.counters(sid).scheduler_timeouts, rt.counters(sid).recoveries_to_online, rt.counters(sid).restreams, - static_cast(ToMs(rt.SelectedRtt(sid)))); + phases[0].selected_rtt_ms, phases[0].prefix1, phases[0].prefix2, + phases[0].retry, phases[0].recoveries, phases[1].selected_rtt_ms, + phases[1].prefix1, phases[1].prefix2, phases[1].retry, + phases[1].recoveries, phases[2].selected_rtt_ms, phases[2].prefix1, + phases[2].prefix2, phases[2].retry, phases[2].recoveries); TEST_ASSERT_EQUAL(0, rt.poll_stats().false_offline_samples); TEST_ASSERT_EQUAL(0, rt.poll_stats().false_offline_transitions); @@ -946,16 +1001,14 @@ void test_IntervalZeroWithoutPongKeepsConfirmed() { rt.AdvanceTo(Tp(20)); TEST_ASSERT_TRUE(rt.machine(sid).has_confirmed_schedule()); rt.machine(sid).SetDesired(rt.now(), - RxTimingConf::Every(Dur(0)).WithWindow(Dur(1000)), - 99); + RxTimingConf::Every(Dur(0)).WithWindow(Dur(1000)), Percentile8::FromPercent(99.0)); TEST_ASSERT_TRUE(rt.machine(sid).has_confirmed_schedule()); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); } void test_IntervalZeroWithPongClearsFuturePresence() { LocalPresenceMachine machine; - machine.SetDesired(Tp(0), RxTimingConf::Every(Dur(0)).WithWindow(Dur(1000)), - 99); + machine.SetDesired(Tp(0), RxTimingConf::Every(Dur(0)).WithWindow(Dur(1000)), Percentile8::FromPercent(99.0)); machine.ArmInitial(Tp(0)); auto tick = machine.TickNow(Tp(0), Dur(100)); TEST_ASSERT_TRUE(tick.want_send); @@ -1145,6 +1198,7 @@ int main() { RUN_TEST(ae::test_local_presence::test_PerServerIndependence); RUN_TEST(ae::test_local_presence::test_OfflineOnlyAfterOfflineDetectionTimeout); RUN_TEST(ae::test_local_presence::test_RuntimeIntervalChangeKeepsOldConfirmed); + RUN_TEST(ae::test_local_presence::test_RuntimePercentileOnlyChangeKeepsSchedule); RUN_TEST(ae::test_local_presence::test_RuntimePercentile); RUN_TEST(ae::test_local_presence::test_ReliabilityP95VsP99PrefixTimes); RUN_TEST(ae::test_local_presence::test_AggregateIgnoresDeselected); From 0b0e3b54b9ffa730c41597c8b18f6a75255bded3 Mon Sep 17 00:00:00 2001 From: aethernet-io Date: Thu, 3 Sep 2026 11:05:18 -0700 Subject: [PATCH 11/11] Migrate Percentile8 to 16-bit FixedPoint Percentile. Pin ae-numeric 3ab9e73; store percentile tail as FixedPoint and drop Exponential wire codes from policy/Presence paths. Co-authored-by: Cursor --- CMakeLists.txt | 2 +- aether/client_connectivity_policy.cpp | 2 +- aether/client_connectivity_policy.h | 4 +- aether/cloud_connections/cloud_request.cpp | 8 +- .../cloud_request_execution_policy.h | 6 +- .../local_presence_machine.cpp | 2 +- .../local_presence_machine.h | 6 +- .../local_presence_schedule.h | 6 +- aether/types/statistic_counter.h | 6 +- .../remote_presence_live.cpp | 6 +- tests/test-cloud-request/main.cpp | 82 ++++++++++--------- tests/test-local-presence/main.cpp | 44 +++++----- 12 files changed, 89 insertions(+), 85 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b9c1a32..401bae0a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,7 +135,7 @@ CPMAddPackage( CPMAddPackage( NAME ae-numeric GIT_REPOSITORY "https://github.com/aethernetio/aethernet-numeric.git" - GIT_TAG "1002f91fb2682f641ec4431c6c57e6c66e620744" + GIT_TAG "3ab9e7310a2f8e6240931261c07c3a0c39771ec2" OPTIONS "AE_NUMERIC_INSTALL ${AE_INSTALL}" "AE_BUILD_TESTS OFF" EXCLUDE_FROM_ALL FALSE ) diff --git a/aether/client_connectivity_policy.cpp b/aether/client_connectivity_policy.cpp index ade21560..9280b08a 100644 --- a/aether/client_connectivity_policy.cpp +++ b/aether/client_connectivity_policy.cpp @@ -92,7 +92,7 @@ auto ClientConnectivityPolicy::ConfigureRxTimings( void ClientConnectivityPolicy::ConfigureServerRxTiming( ServerId server_id, RxTimingConf conf, - Percentile8 rtt_reliability_percentile) { + Percentile rtt_reliability_percentile) { auto& state = EnsureServerPresence(server_id); auto const timing_changed = (state.desired.interval != conf.interval) || (state.desired.rx_window != conf.rx_window); diff --git a/aether/client_connectivity_policy.h b/aether/client_connectivity_policy.h index a933fb22..f0a128aa 100644 --- a/aether/client_connectivity_policy.h +++ b/aether/client_connectivity_policy.h @@ -74,7 +74,7 @@ struct ConnectivityStatus { struct ServerPresenceState { RxTimingConf desired{ RxTimingConf::Every(std::chrono::milliseconds{AE_PING_INTERVAL_MS})}; - Percentile8 rtt_reliability_percentile{kDefaultRttReliabilityPercentile}; + Percentile rtt_reliability_percentile{kDefaultRttReliabilityPercentile}; bool has_confirmed_schedule{false}; Duration confirmed_interval{}; @@ -149,7 +149,7 @@ class ClientConnectivityPolicy : public Obj { // Per-server runtime config. Does not invent ONLINE until a confirming Pong. void ConfigureServerRxTiming( ServerId server_id, RxTimingConf conf, - Percentile8 rtt_reliability_percentile = + Percentile rtt_reliability_percentile = kDefaultRttReliabilityPercentile); void SetServerSelectedForAggregate(ServerId server_id, bool selected); diff --git a/aether/cloud_connections/cloud_request.cpp b/aether/cloud_connections/cloud_request.cpp index 98548a8e..9a38c25a 100644 --- a/aether/cloud_connections/cloud_request.cpp +++ b/aether/cloud_connections/cloud_request.cpp @@ -57,9 +57,9 @@ CloudRequest::CloudRequest(AeContext const& ae_context, MethodPtr<&CloudRequest::ServersUpdated>{this})} { NormalizeCloudRequestExecutionPolicy(exec_policy_); AE_CLOUD_REQ_DEBUG( - "CLOUD_REQUEST_START percentile_code={} factor_raw={} retry_count={} " + "CLOUD_REQUEST_START percentile_tail_raw={} factor_raw={} retry_count={} " "hedge_next_servers={}", - exec_policy_.response_percentile.Code(), + exec_policy_.response_percentile.TailPercent().RawValue(), exec_policy_.timeout_factor.RawValue(), exec_policy_.retry_count, exec_policy_.hedge_next_servers); RebuildCandidates(); @@ -81,9 +81,9 @@ CloudRequest::CloudRequest(AeContext const& ae_context, MethodPtr<&CloudRequest::ServersUpdated>{this})} { NormalizeCloudRequestExecutionPolicy(exec_policy_); AE_CLOUD_REQ_DEBUG( - "CLOUD_REQUEST_START percentile_code={} factor_raw={} retry_count={} " + "CLOUD_REQUEST_START percentile_tail_raw={} factor_raw={} retry_count={} " "hedge_next_servers={}", - exec_policy_.response_percentile.Code(), + exec_policy_.response_percentile.TailPercent().RawValue(), exec_policy_.timeout_factor.RawValue(), exec_policy_.retry_count, exec_policy_.hedge_next_servers); RebuildCandidates(); diff --git a/aether/cloud_connections/cloud_request_execution_policy.h b/aether/cloud_connections/cloud_request_execution_policy.h index a7267e78..7cd5314a 100644 --- a/aether/cloud_connections/cloud_request_execution_policy.h +++ b/aether/cloud_connections/cloud_request_execution_policy.h @@ -22,7 +22,7 @@ #include -#include "ae-numeric/percentile8.h" +#include "ae-numeric/percentile.h" #include "aether/clock.h" #include "aether/config.h" @@ -37,7 +37,7 @@ inline constexpr std::uint8_t kMaxCloudRequestRetryCount{31}; // Orthogonal to RequestPolicy (which servers are candidates). struct CloudRequestExecutionPolicy { // Soft response timeout uses channel response RTT percentile. - Percentile8 response_percentile{Percentile8::FromPercent(99.0)}; + Percentile response_percentile{Percentile::FromPercent(99.0)}; // Soft-timeout multiplier (1-byte FixedPoint, typically Q2.6). TimeoutFactor8 timeout_factor{TimeoutFactor8::FromDouble(1.2)}; // Retries after the initial attempt. retry_count=0 => 1 attempt total. @@ -55,7 +55,7 @@ struct CloudRequestExecutionPolicy { } static constexpr CloudRequestExecutionPolicy FromFactor( - Percentile8 percentile, TimeoutFactor8 factor, std::uint8_t retries, + Percentile percentile, TimeoutFactor8 factor, std::uint8_t retries, std::uint8_t hedge) noexcept { CloudRequestExecutionPolicy p{}; p.response_percentile = percentile; diff --git a/aether/cloud_connections/local_presence_machine.cpp b/aether/cloud_connections/local_presence_machine.cpp index 9a229db6..286069dd 100644 --- a/aether/cloud_connections/local_presence_machine.cpp +++ b/aether/cloud_connections/local_presence_machine.cpp @@ -49,7 +49,7 @@ void CountKind(LocalPresenceMachine::Counters& counters, LocalPresenceMachine::LocalPresenceMachine() = default; void LocalPresenceMachine::SetDesired(TimePoint now, RxTimingConf conf, - Percentile8 percentile) { + Percentile percentile) { auto const changed = (desired_.interval != conf.interval) || (desired_.rx_window != conf.rx_window); desired_ = conf; diff --git a/aether/cloud_connections/local_presence_machine.h b/aether/cloud_connections/local_presence_machine.h index ed0eed98..8b818606 100644 --- a/aether/cloud_connections/local_presence_machine.h +++ b/aether/cloud_connections/local_presence_machine.h @@ -103,10 +103,10 @@ class LocalPresenceMachine { LocalPresenceMachine(); - void SetDesired(TimePoint now, RxTimingConf conf, Percentile8 percentile); + void SetDesired(TimePoint now, RxTimingConf conf, Percentile percentile); void SetOfflineDetectionTimeout(Duration timeout) noexcept; RxTimingConf const& desired() const noexcept { return desired_; } - Percentile8 percentile() const noexcept { return percentile_; } + Percentile percentile() const noexcept { return percentile_; } Duration offline_detection_timeout() const noexcept { return offline_detection_timeout_; } @@ -185,7 +185,7 @@ class LocalPresenceMachine { RxTimingConf desired_{RxTimingConf::Every( std::chrono::milliseconds{AE_PING_INTERVAL_MS})}; - Percentile8 percentile_{kDefaultRttReliabilityPercentile}; + Percentile percentile_{kDefaultRttReliabilityPercentile}; Duration offline_detection_timeout_{std::chrono::milliseconds{ AE_OFFLINE_DETECTION_TIMEOUT_MS}}; diff --git a/aether/cloud_connections/local_presence_schedule.h b/aether/cloud_connections/local_presence_schedule.h index b32d04a6..c41d1704 100644 --- a/aether/cloud_connections/local_presence_schedule.h +++ b/aether/cloud_connections/local_presence_schedule.h @@ -21,7 +21,7 @@ #include #include "aether/clock.h" -#include "ae-numeric/percentile8.h" +#include "ae-numeric/percentile.h" namespace ae { @@ -29,8 +29,8 @@ namespace ae { inline constexpr Duration kLocalPresenceGuard = std::chrono::duration_cast(std::chrono::milliseconds{30}); -inline constexpr Percentile8 kDefaultRttReliabilityPercentile = - Percentile8::FromPercent(99.0); +inline constexpr Percentile kDefaultRttReliabilityPercentile = + Percentile::FromPercent(99.0); inline constexpr std::size_t kMaxOutstandingPresenceAttempts{8}; diff --git a/aether/types/statistic_counter.h b/aether/types/statistic_counter.h index ffed2fc0..5ce80949 100644 --- a/aether/types/statistic_counter.h +++ b/aether/types/statistic_counter.h @@ -29,7 +29,7 @@ IGNORE_IMPLICIT_CONVERSION() #include DISABLE_WARNING_POP() -#include "ae-numeric/percentile8.h" +#include "ae-numeric/percentile.h" #include "aether-miscpp/format/format.h" #include "aether-miscpp/serialization/binary_archive.h" @@ -105,9 +105,9 @@ class StatisticsCounter final { } /** - * \brief Percentile8 accessor. Rank uses integer/fixed tail math only. + * \brief Percentile accessor. Rank uses integer/fixed tail math only. */ - [[nodiscard]] TValue PercentileValue(Percentile8 percentile) const { + [[nodiscard]] TValue PercentileValue(Percentile percentile) const { assert(!value_buffer_.empty()); if (percentile.IsExactHundred()) { return max(); diff --git a/examples/remote_presence_live/remote_presence_live.cpp b/examples/remote_presence_live/remote_presence_live.cpp index 30f291fc..f67a7f73 100644 --- a/examples/remote_presence_live/remote_presence_live.cpp +++ b/examples/remote_presence_live/remote_presence_live.cpp @@ -55,7 +55,7 @@ #include "aether/all.h" #include "aether/client_connectivity_policy.h" #include "aether/cloud_connections/cloud_request_execution_policy.h" -#include "ae-numeric/percentile8.h" +#include "ae-numeric/percentile.h" #include "aether/cloud_connections/local_presence_schedule.h" #include "aether/config.h" #include "aether/remote_presence.h" @@ -141,7 +141,7 @@ void ApplyTimings(Client& client) { policy->ResetRxTimings(); policy->SetOfflineDetectionTimeout(kOfflineTimeout); policy->SetCloudRequestExecutionPolicy( - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.99), TimeoutFactor8::FromDouble(1.2), /*retries=*/2, + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.99), TimeoutFactor8::FromDouble(1.2), /*retries=*/2, /*hedge=*/2)); policy->ConfigureRxTimings(RequestPolicy::All{}) .ForAllPriorities(RxTimingConf::Every(kInterval).WithWindow(kWindow)); @@ -152,7 +152,7 @@ void ApplyTimings(Client& client) { policy->ConfigureServerRxTiming( server->server_id(), RxTimingConf::Every(kInterval).WithWindow(kWindow), - Percentile8::FromPercent(99.0)); + Percentile::FromPercent(99.0)); } } diff --git a/tests/test-cloud-request/main.cpp b/tests/test-cloud-request/main.cpp index a2ec69a0..eae7b741 100644 --- a/tests/test-cloud-request/main.cpp +++ b/tests/test-cloud-request/main.cpp @@ -22,7 +22,7 @@ #include #include "aether/cloud_connections/cloud_request_execution_policy.h" -#include "ae-numeric/percentile8.h" +#include "ae-numeric/percentile.h" #include "aether/types/statistic_counter.h" namespace ae::test_cloud_request { @@ -44,13 +44,13 @@ void test_TimeoutCalculation() { TEST_ASSERT_EQUAL(500, std::chrono::duration_cast(p99).count()); auto const t95_10 = - ComputeCloudRequestSoftTimeout(p95, CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(95.0), TimeoutFactor8::FromDouble(1.0), 1, 0)); + ComputeCloudRequestSoftTimeout(p95, CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(95.0), TimeoutFactor8::FromDouble(1.0), 1, 0)); auto const t95_12 = - ComputeCloudRequestSoftTimeout(p95, CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(95.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); + ComputeCloudRequestSoftTimeout(p95, CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(95.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); auto const t99_10 = - ComputeCloudRequestSoftTimeout(p99, CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.0), 1, 0)); + ComputeCloudRequestSoftTimeout(p99, CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.0), 1, 0)); auto const t99_12 = - ComputeCloudRequestSoftTimeout(p99, CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); + ComputeCloudRequestSoftTimeout(p99, CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); TEST_ASSERT_EQUAL(500, std::chrono::duration_cast(t95_10).count()); // 500ms * TimeoutFactor8(1.2) raw=77 / 64 = 601.5625 → 602 nearest TEST_ASSERT_EQUAL(602, std::chrono::duration_cast(t95_12).count()); @@ -70,7 +70,7 @@ void test_TimeoutCalculation() { void test_RetryCountSemantics() { CloudRequestExecutionPolicy p0 = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 0, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 0, 0); TEST_ASSERT_EQUAL(1, p0.TotalAttempts()); CloudRequestServerExecState s0; s0.activated = true; @@ -81,7 +81,7 @@ void test_RetryCountSemantics() { TEST_ASSERT_TRUE(s0.exhausted); CloudRequestExecutionPolicy p1 = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0); TEST_ASSERT_EQUAL(2, p1.TotalAttempts()); CloudRequestServerExecState s1; s1.activated = true; @@ -97,7 +97,7 @@ void test_RetryCountSemantics() { TEST_ASSERT_TRUE(s1.exhausted); CloudRequestExecutionPolicy p2 = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); TEST_ASSERT_EQUAL(3, p2.TotalAttempts()); CloudRequestServerExecState s2; s2.activated = true; @@ -119,7 +119,7 @@ void test_RetryCountSemantics() { void test_NoQuarantineBeforeExhaustionAndHedge() { CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 2); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 2); CloudRequestServerExecState s; s.activated = true; TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); @@ -148,7 +148,7 @@ void test_NoQuarantineBeforeExhaustionAndHedge() { void test_HedgeZeroKeepsSequential() { CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); CloudRequestServerExecState s1; s1.activated = true; s1.StartAttempt(policy); @@ -158,7 +158,7 @@ void test_HedgeZeroKeepsSequential() { void test_LateResponseAfterSoftTimeout() { CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); CloudRequestServerExecState s; s.activated = true; TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); @@ -179,10 +179,10 @@ void test_LateResponseAfterSoftTimeout() { void test_PerServerTimeoutIndependent() { auto const t1 = ComputeCloudRequestSoftTimeout( Duration{Ms{100}}, - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); auto const t2 = ComputeCloudRequestSoftTimeout( Duration{Ms{300}}, - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0)); TEST_ASSERT_EQUAL(120, std::chrono::duration_cast(t1).count()); // 300ms * 77/64 = 360.9375 → 361 nearest TEST_ASSERT_EQUAL(361, std::chrono::duration_cast(t2).count()); @@ -190,70 +190,74 @@ void test_PerServerTimeoutIndependent() { void test_PolicySnapshotDefaults() { auto const d = CloudRequestExecutionPolicy::Default(); - TEST_ASSERT_EQUAL_UINT8(Percentile8::FromPercent(99.0).Code(), d.response_percentile.Code()); + TEST_ASSERT_EQUAL_UINT16( + Percentile::FromPercent(99.0).TailPercent().RawValue(), + d.response_percentile.TailPercent().RawValue()); TEST_ASSERT_EQUAL_UINT8(TimeoutFactor8::FromDouble(1.2).RawValue(), d.timeout_factor.RawValue()); TEST_ASSERT_EQUAL(1, d.retry_count); TEST_ASSERT_EQUAL(0, d.hedge_next_servers); TEST_ASSERT_EQUAL(2, d.TotalAttempts()); - static_assert(sizeof(Percentile8) == 1); + static_assert(sizeof(Percentile) == 2); static_assert(sizeof(TimeoutFactor8) == 1); std::printf("sizeof(CloudRequestExecutionPolicy)=%zu\n", sizeof(CloudRequestExecutionPolicy)); } -void test_Percentile8FractionalDistinctRanks() { +void test_PercentileFractionalDistinctRanks() { // Need N large enough that quantized p99.9 / p99.99 ranks differ // (see PercentileIndex: diverge by N≈10000). StatisticsCounter stats; for (int i = 0; i < 10000; ++i) { stats.Add(i); } - auto const p99 = stats.PercentileValue(Percentile8::FromPercent(99.0)); - auto const p999 = stats.PercentileValue(Percentile8::FromPercent(99.9)); - auto const p9999 = stats.PercentileValue(Percentile8::FromPercent(99.99)); + auto const p99 = stats.PercentileValue(Percentile::FromPercent(99.0)); + auto const p999 = stats.PercentileValue(Percentile::FromPercent(99.9)); + auto const p9999 = stats.PercentileValue(Percentile::FromPercent(99.99)); std::printf( "selected RTT ranks (samples 0..9999): p99=%d p99.9=%d p99.99=%d\n", p99, p999, p9999); TEST_ASSERT_TRUE(p99 <= p999); TEST_ASSERT_TRUE(p999 <= p9999); TEST_ASSERT_TRUE(p999 < p9999); - TEST_ASSERT_TRUE(PercentileIndex(1'000'000, Percentile8::FromPercent(99.9)) < - PercentileIndex(1'000'000, Percentile8::FromPercent(99.99))); + TEST_ASSERT_TRUE(PercentileIndex(1'000'000, Percentile::FromPercent(99.9)) < + PercentileIndex(1'000'000, Percentile::FromPercent(99.99))); TEST_ASSERT_EQUAL(PercentileIndexInteger(1000, 95), - PercentileIndex(1000, Percentile8::FromPercent(95.0))); + PercentileIndex(1000, Percentile::FromPercent(95.0))); } void test_PolicyFieldsAreRuntimeAssignable() { CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(95.0), + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(95.0), TimeoutFactor8::FromDouble(1.0), 1, 0); auto const snap = policy; - policy.response_percentile = Percentile8::FromPercent(99.99); + policy.response_percentile = Percentile::FromPercent(99.99); policy.timeout_factor = TimeoutFactor8::FromDouble(1.2); - TEST_ASSERT_EQUAL_UINT8(Percentile8::FromPercent(95.0).Code(), - snap.response_percentile.Code()); + TEST_ASSERT_EQUAL_UINT16( + Percentile::FromPercent(95.0).TailPercent().RawValue(), + snap.response_percentile.TailPercent().RawValue()); TEST_ASSERT_EQUAL_UINT8(TimeoutFactor8::FromDouble(1.0).RawValue(), snap.timeout_factor.RawValue()); - TEST_ASSERT_EQUAL_UINT8(Percentile8::FromPercent(99.99).Code(), - policy.response_percentile.Code()); + TEST_ASSERT_EQUAL_UINT16( + Percentile::FromPercent(99.99).TailPercent().RawValue(), + policy.response_percentile.TailPercent().RawValue()); } void test_RetryCountClampAndMax() { TEST_ASSERT_EQUAL(31, kMaxCloudRequestRetryCount); CloudRequestExecutionPolicy p0 = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 0, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 0, 0); TEST_ASSERT_EQUAL(0, p0.retry_count); TEST_ASSERT_EQUAL(1, p0.TotalAttempts()); CloudRequestExecutionPolicy p1 = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 1, 0); TEST_ASSERT_EQUAL(1, p1.retry_count); TEST_ASSERT_EQUAL(2, p1.TotalAttempts()); CloudRequestExecutionPolicy p31 = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 31, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 31, 0); TEST_ASSERT_EQUAL(31, p31.retry_count); TEST_ASSERT_EQUAL(32, p31.TotalAttempts()); @@ -264,14 +268,14 @@ void test_RetryCountClampAndMax() { TEST_ASSERT_EQUAL(32, over.TotalAttempts()); auto const from_over = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 255, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 255, 0); TEST_ASSERT_EQUAL(31, from_over.retry_count); TEST_ASSERT_EQUAL(32, from_over.TotalAttempts()); } void test_RetryCount31StateMachine() { CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 31, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 31, 0); TEST_ASSERT_EQUAL(32, policy.TotalAttempts()); CloudRequestServerExecState s; @@ -302,7 +306,7 @@ void test_ChannelChangedOneCallbackPerServer() { // one channel-changed event must produce exactly one OnChannelChanged // decision and at most one additional attempt. CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); CloudRequestServerExecState s; s.activated = true; TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); @@ -335,7 +339,7 @@ void test_ChannelChangedThreeOutstandingAttempts() { // Three outstanding attempts (retry_count=2, all started via soft path / // channel), then one channel event must still be a single decision. CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); CloudRequestServerExecState s; s.activated = true; s.StartAttempt(policy); // #1 @@ -359,7 +363,7 @@ void test_ChannelChangedThreeOutstandingAttempts() { void test_ApiErrorDoesNotQuarantine() { CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); CloudRequestServerExecState s; s.activated = true; TEST_ASSERT_EQUAL(1, s.StartAttempt(policy)); @@ -385,7 +389,7 @@ void test_ApiErrorDoesNotQuarantine() { void test_NoResponseStillQuarantinesAfterBudget() { // retry_count=2 => attempts=3 soft timeouts then exhaust (=quarantine point). CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 0); CloudRequestServerExecState s; s.activated = true; s.StartAttempt(policy); @@ -404,7 +408,7 @@ void test_NoResponseStillQuarantinesAfterBudget() { void test_DeterministicLatencyTimeline() { // p99=100ms, factor=1.2 => T=120ms per attempt when RTT fixed. CloudRequestExecutionPolicy policy = - CloudRequestExecutionPolicy::FromFactor(Percentile8::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 1); + CloudRequestExecutionPolicy::FromFactor(Percentile::FromPercent(99.0), TimeoutFactor8::FromDouble(1.2), 2, 1); auto const T = ComputeCloudRequestSoftTimeout(Duration{Ms{100}}, policy); TEST_ASSERT_EQUAL(120, std::chrono::duration_cast(T).count()); @@ -462,7 +466,7 @@ int main() { RUN_TEST(ae::test_cloud_request::test_NoResponseStillQuarantinesAfterBudget); RUN_TEST(ae::test_cloud_request::test_PerServerTimeoutIndependent); RUN_TEST(ae::test_cloud_request::test_PolicySnapshotDefaults); - RUN_TEST(ae::test_cloud_request::test_Percentile8FractionalDistinctRanks); + RUN_TEST(ae::test_cloud_request::test_PercentileFractionalDistinctRanks); RUN_TEST(ae::test_cloud_request::test_PolicyFieldsAreRuntimeAssignable); RUN_TEST(ae::test_cloud_request::test_DeterministicLatencyTimeline); return UNITY_END(); diff --git a/tests/test-local-presence/main.cpp b/tests/test-local-presence/main.cpp index b093851c..f7ab071a 100644 --- a/tests/test-local-presence/main.cpp +++ b/tests/test-local-presence/main.cpp @@ -28,7 +28,7 @@ #include "aether/cloud_connections/local_presence_machine.h" #include "aether/cloud_connections/local_presence_schedule.h" #include "aether/remote_presence.h" -#include "ae-numeric/percentile8.h" +#include "ae-numeric/percentile.h" #include "aether/types/statistic_counter.h" #include "aether/work_cloud_api/client_timing.h" @@ -66,7 +66,7 @@ void test_ConfirmOnlyAfterPong() { ClientConnectivityPolicy policy; ServerId const sid{7}; policy.ConfigureServerRxTiming( - sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Percentile8::FromPercent(99.0)); + sid, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Percentile::FromPercent(99.0)); auto* state = policy.FindServerPresence(sid); TEST_ASSERT_NOT_NULL(state); TEST_ASSERT_FALSE(state->has_confirmed_schedule); @@ -106,9 +106,9 @@ void test_PerServerIndependence() { ServerId const a{1}; ServerId const b{2}; policy.ConfigureServerRxTiming( - a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Percentile8::FromPercent(99.0)); + a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Percentile::FromPercent(99.0)); policy.ConfigureServerRxTiming( - b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), Percentile8::FromPercent(95.0)); + b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), Percentile::FromPercent(95.0)); policy.ConfirmServerPong(a, Tp(0), Tp(100), Dur(1000), Dur(300), Dur(100)); TEST_ASSERT_FALSE(policy.IsServerLocallyOnline(b, Tp(50))); TEST_ASSERT_TRUE(policy.IsServerLocallyOnline(a, Tp(50))); @@ -218,7 +218,7 @@ void test_ConfigScopeOverrideAndPriority() { policy.BindServerPriority(a, 0); policy.BindServerPriority(b, 1); policy.ConfigureServerRxTiming( - a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), Percentile8::FromPercent(99.0)); + a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)), Percentile::FromPercent(99.0)); TEST_ASSERT_EQUAL(1000, ToMs(policy.FindServerPresence(a)->desired.interval)); TEST_ASSERT_EQUAL(AE_PING_INTERVAL_MS, ToMs(policy.FindServerPresence(b)->desired.interval)); @@ -272,7 +272,7 @@ class PresenceHarness { } void AddServer(ServerId id, RxTimingConf conf, Duration seed_rtt, - Percentile8 percentile = kDefaultRttReliabilityPercentile, + Percentile percentile = kDefaultRttReliabilityPercentile, std::size_t priority = 0) { policy_.BindServerPriority(id, priority); policy_.ConfigureServerRxTiming(id, conf, percentile); @@ -428,7 +428,7 @@ class PresenceHarness { LocalPresenceMachine machine{}; StatisticsCounter stats{}; Duration seed_rtt{Dur(100)}; - Percentile8 percentile{kDefaultRttReliabilityPercentile}; + Percentile percentile{kDefaultRttReliabilityPercentile}; Duration fixed_delay{Dur(20)}; DelayFn delay_fn{}; bool drop_kind[5]{}; @@ -719,7 +719,7 @@ void test_RuntimeConfigChangeKeepsOldUntilPong() { rt.policy().ConfigureServerRxTiming( sid, RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200))); rt.machine(sid).SetDesired( - rt.now(), RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200)), Percentile8::FromPercent(99.0)); + rt.now(), RxTimingConf::Every(Dur(10000)).WithWindow(Dur(200)), Percentile::FromPercent(99.0)); TEST_ASSERT_EQUAL(1000, ToMs(rt.machine(sid).confirmed_interval())); TEST_ASSERT_TRUE(rt.machine(sid).confirmed_window_close() == close_old); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); @@ -729,20 +729,20 @@ void test_RuntimePercentileOnlyChangeKeepsSchedule() { PresenceHarness rt{Tp(0)}; ServerId const sid{1}; auto const conf = RxTimingConf::Every(Dur(1000)).WithWindow(Dur(1000)); - rt.AddServer(sid, conf, Dur(100), Percentile8::FromPercent(95.0)); + rt.AddServer(sid, conf, Dur(100), Percentile::FromPercent(95.0)); rt.SetFixedDelay(sid, Dur(20)); rt.AdvanceTo(Tp(20)); TEST_ASSERT_TRUE(rt.machine(sid).has_confirmed_schedule()); auto const close_before = rt.machine(sid).confirmed_window_close(); auto const open_before = rt.machine(sid).confirmed_window_open(); rt.policy().ConfigureServerRxTiming(sid, conf, - Percentile8::FromPercent(99.99)); - rt.machine(sid).SetDesired(rt.now(), conf, Percentile8::FromPercent(99.99)); + Percentile::FromPercent(99.99)); + rt.machine(sid).SetDesired(rt.now(), conf, Percentile::FromPercent(99.99)); TEST_ASSERT_TRUE(rt.machine(sid).has_confirmed_schedule()); TEST_ASSERT_TRUE(rt.machine(sid).confirmed_window_close() == close_before); TEST_ASSERT_TRUE(rt.machine(sid).confirmed_window_open() == open_before); - TEST_ASSERT_EQUAL_UINT8(Percentile8::FromPercent(99.99).Code(), - rt.machine(sid).percentile().Code()); + TEST_ASSERT_TRUE(rt.machine(sid).percentile() == + Percentile::FromPercent(99.99)); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); } @@ -765,8 +765,8 @@ void test_MultiServerIndependentSchedules() { PresenceHarness rt{Tp(0)}; ServerId const a{1}; ServerId const b{2}; - rt.AddServer(a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100), Percentile8::FromPercent(99.0), 0); - rt.AddServer(b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), Dur(200), Percentile8::FromPercent(95.0), 1); + rt.AddServer(a, RxTimingConf::Every(Dur(1000)).WithWindow(Dur(300)), Dur(100), Percentile::FromPercent(99.0), 0); + rt.AddServer(b, RxTimingConf::Every(Dur(3000)).WithWindow(Dur(700)), Dur(200), Percentile::FromPercent(95.0), 1); rt.SetFixedDelay(a, Dur(20)); rt.SetFixedDelay(b, Dur(20)); rt.AdvanceTo(Tp(40)); @@ -796,7 +796,7 @@ void test_StatisticalRuntimePollingIsLocallyOnline() { auto const window = Dur(1000); auto const rtt = Dur(100); // §25: start p95, then runtime switch p99, then p99.99 — no restart. - auto phase_pct = Percentile8::FromPercent(95.0); + auto phase_pct = Percentile::FromPercent(95.0); rt.AddServer(sid, RxTimingConf::Every(interval).WithWindow(window), rtt, phase_pct); rt.SetDelayFn(sid, [&rt, sid](PingAttemptKind kind, int) { auto const prefix1 = rt.counters(sid).prefix1; @@ -842,7 +842,7 @@ void test_StatisticalRuntimePollingIsLocallyOnline() { std::chrono::duration_cast(rt.now() - measure_start).count(); if (phase == 0 && elapsed_ms >= 100000) { snap_phase(0); - phase_pct = Percentile8::FromPercent(99.0); + phase_pct = Percentile::FromPercent(99.0); rt.policy().ConfigureServerRxTiming( sid, RxTimingConf::Every(interval).WithWindow(window), phase_pct); rt.machine(sid).SetDesired( @@ -850,7 +850,7 @@ void test_StatisticalRuntimePollingIsLocallyOnline() { phase = 1; } else if (phase == 1 && elapsed_ms >= 200000) { snap_phase(1); - phase_pct = Percentile8::FromPercent(99.99); + phase_pct = Percentile::FromPercent(99.99); rt.policy().ConfigureServerRxTiming( sid, RxTimingConf::Every(interval).WithWindow(window), phase_pct); rt.machine(sid).SetDesired( @@ -864,8 +864,8 @@ void test_StatisticalRuntimePollingIsLocallyOnline() { } snap_phase(2); TEST_ASSERT_EQUAL(2, phase); - TEST_ASSERT_EQUAL_UINT8(Percentile8::FromPercent(99.99).Code(), - rt.machine(sid).percentile().Code()); + TEST_ASSERT_TRUE(rt.machine(sid).percentile() == + Percentile::FromPercent(99.99)); g_stat_report.confirmed_cycles = rt.counters(sid).confirmed_pongs; g_stat_report.duration = @@ -1001,14 +1001,14 @@ void test_IntervalZeroWithoutPongKeepsConfirmed() { rt.AdvanceTo(Tp(20)); TEST_ASSERT_TRUE(rt.machine(sid).has_confirmed_schedule()); rt.machine(sid).SetDesired(rt.now(), - RxTimingConf::Every(Dur(0)).WithWindow(Dur(1000)), Percentile8::FromPercent(99.0)); + RxTimingConf::Every(Dur(0)).WithWindow(Dur(1000)), Percentile::FromPercent(99.0)); TEST_ASSERT_TRUE(rt.machine(sid).has_confirmed_schedule()); TEST_ASSERT_TRUE(rt.IsLocallyOnline()); } void test_IntervalZeroWithPongClearsFuturePresence() { LocalPresenceMachine machine; - machine.SetDesired(Tp(0), RxTimingConf::Every(Dur(0)).WithWindow(Dur(1000)), Percentile8::FromPercent(99.0)); + machine.SetDesired(Tp(0), RxTimingConf::Every(Dur(0)).WithWindow(Dur(1000)), Percentile::FromPercent(99.0)); machine.ArmInitial(Tp(0)); auto tick = machine.TickNow(Tp(0), Dur(100)); TEST_ASSERT_TRUE(tick.want_send);