diff --git a/rs/messaging/src/canister_http_spent.rs b/rs/messaging/src/canister_http_spent.rs index 56a9af7951fe..fce7c5299711 100644 --- a/rs/messaging/src/canister_http_spent.rs +++ b/rs/messaging/src/canister_http_spent.rs @@ -1,4 +1,4 @@ -use ic_logger::{ReplicaLogger, error, info}; +use ic_logger::{ReplicaLogger, error}; use ic_metrics::MetricsRegistry; use ic_replicated_state::ReplicatedState; use ic_types::messages::CallbackId; @@ -73,8 +73,8 @@ struct CanisterAccounting { } /// Applies the HTTP outcall spend reports carried by `spent` to the calling -/// canisters: it credits each caller the refund derived from its per-replica -/// allowance (`allowance − spent`) and reports the spent cycles as consumed. +/// canisters: it refunds each caller the unspent part of its per-replica allowance +/// (`allowance − spent`) and reports the spent cycles as consumed. /// /// Reports are only applied to contexts that have already been responded to, /// i.e. those in `delivered_canister_http_request_contexts`. @@ -177,7 +177,7 @@ pub(crate) fn deliver_canister_http_spent( } } - apply_accounting(state, accounting, log); + apply_accounting(state, accounting); } /// Times out delivered `CanisterHttpRequestContext`s and refunds the calling @@ -212,7 +212,7 @@ pub(crate) fn refund_timed_out_canister_http_contexts( accounting.entry(context.request.sender).or_default().refund += applied; } - apply_accounting(state, accounting, log); + apply_accounting(state, accounting); } /// Applies a reported spend of `amount`, covered by `allowance`, against the @@ -288,52 +288,35 @@ fn apply_capped( applied } -/// Credits the accumulated per-canister `refund` to the corresponding canisters' -/// balances and reports the accumulated `consumed` cycles in their cost metrics -/// as well as in the subnet's. +/// Pools the accumulated per-canister `refund`, for Message Routing to deliver to +/// the corresponding canisters; and reports the accumulated `consumed` cycles in +/// their cost metrics as well as in the subnet's. /// -/// If the calling canister no longer exists, its refund cannot be credited back -/// and is lost. It was taken out of the canister's balance when the request was -/// made, so it is reported as consumed by deleted canisters; otherwise these cycles -/// would silently disappear from [`SubnetMetrics::consumed_cycles_total()`]. +/// Refunds are pooled rather than credited to the caller's balance directly, because +/// the caller need not be hosted here anymore: an HTTP outcall's refundable cycles +/// are only settled once its spend reports have been applied (or it has timed out), +/// by which time a subnet split may have migrated the caller to another subnet. +/// Message Routing delivers a pooled refund wherever the caller lives (via the +/// loopback stream, if that is still this subnet) and accounts for the cycles as +/// lost if the caller no longer exists at all. +/// +/// The `consumed` cycles are reported on the caller only if it is still local. The +/// subnet-wide metrics are updated either way: the spend was consumed by the subnet +/// whether or not the caller is still around. fn apply_accounting( state: &mut ReplicatedState, accounting: BTreeMap, - log: &ReplicaLogger, ) { let mut subnet_consumed = NominalCycles::zero(); - let mut lost_by_deleted_canisters = NominalCycles::zero(); for (sender, accounting) in accounting { - if accounting.refund.is_zero() && accounting.consumed.is_zero() { - continue; - } - // The spend was consumed by the subnet whether or not the calling canister - // is still around. subnet_consumed += accounting.consumed; - match state.canister_state_make_mut(&sender) { - Some(canister) => { - canister.system_state.add_cycles(accounting.refund); - if !accounting.consumed.is_zero() { - canister - .system_state - .observe_consumed_cycles_for_https_outcall(accounting.consumed); - } - } - None => { - info!( - log, - "Canister {} for an HTTP outcall no longer exists; its refund of {} cycles is \ - lost and reported as consumed by deleted canisters (consumed {} cycles).", - sender, - accounting.refund, - accounting.consumed - ); - lost_by_deleted_canisters += CompoundCycles::::new( - accounting.refund, - CanisterCyclesCostSchedule::Normal, - ) - .nominal(); - } + state.add_refund(sender, accounting.refund); + if !accounting.consumed.is_zero() + && let Some(canister) = state.canister_state_make_mut(&sender) + { + canister + .system_state + .observe_consumed_cycles_for_https_outcall(accounting.consumed); } } @@ -343,13 +326,6 @@ fn apply_accounting( subnet_metrics .observe_consumed_cycles_with_use_case(CyclesUseCase::HTTPOutcalls, subnet_consumed); } - if !lost_by_deleted_canisters.is_zero() { - subnet_metrics.observe_consumed_cycles_with_use_case( - CyclesUseCase::DeletedCanisters, - lost_by_deleted_canisters, - ); - subnet_metrics.observe_consumed_cycles_by_deleted_canisters(lost_by_deleted_canisters); - } } #[cfg(test)] @@ -532,6 +508,25 @@ mod tests { } } + /// The cycles pooled to be refunded to `caller`; zero if none. + fn refunded(state: &ReplicatedState, caller: CanisterId) -> Cycles { + state + .refunds() + .iter() + .find(|refund| refund.recipient() == caller) + .map_or_else(Cycles::zero, |refund| refund.amount()) + } + + /// The whole refund pool, by recipient. As opposed to [`refunded()`], this also + /// pins down which canisters have a pooled refund at all. + fn pooled_refunds(state: &ReplicatedState) -> BTreeMap { + state + .refunds() + .iter() + .map(|refund| (refund.recipient(), refund.amount())) + .collect() + } + fn balance(state: &ReplicatedState, caller: CanisterId) -> Cycles { state .canister_state(&caller) @@ -634,10 +629,10 @@ mod tests { ); } - /// An initial report on a normal subnet credits the collective refund + /// An initial report on a normal subnet pools the collective refund /// (`allowance * nodes − spent`) and reports the spent cycles as consumed. #[test] - fn initial_report_credits_refund_and_reports_consumed() { + fn initial_report_pools_refund_and_reports_consumed() { let allowance = Cycles::new(1_000); let spent = Cycles::new(9_500); let refundable = allowance * SUBNET_SIZE; // 13_000 @@ -655,10 +650,7 @@ mod tests { deliver_canister_http_spent(&mut state, &report, &no_op_logger(), &metrics); // refund = 13 * 1_000 − 9_500 = 3_500. - assert_eq!( - balance(&state, caller), - INITIAL_BALANCE + Cycles::new(3_500) - ); + assert_eq!(refunded(&state, caller), Cycles::new(3_500)); assert_eq!(consumed(&state, caller), spent.get()); // The spend is also reported at the subnet level, like the base fee. assert_eq!( @@ -671,11 +663,11 @@ mod tests { assert_errors(&[], &metrics_registry); } - /// An asynchronous report on a normal subnet credits `allowance − spent` for + /// An asynchronous report on a normal subnet pools `allowance − spent` for /// each reporting node and reports the per-node spend as consumed. Only the /// reporting nodes are accounted; the rest are refunded on timeout. #[test] - fn asynchronous_report_credits_refund_and_reports_consumed() { + fn asynchronous_report_pools_refund_and_reports_consumed() { let allowance = Cycles::new(1_000); let refundable = allowance * SUBNET_SIZE; // 13_000 let (mut state, caller) = setup(Some((Replication::FullyReplicated, refundable))); @@ -696,10 +688,7 @@ mod tests { // refund = (1_000 − 250) + (1_000 − 400) + (1_000 − 900) = 1_450; // consumed = 250 + 400 + 900 = 1_550. - assert_eq!( - balance(&state, caller), - INITIAL_BALANCE + Cycles::new(1_450) - ); + assert_eq!(refunded(&state, caller), Cycles::new(1_450)); assert_eq!(consumed(&state, caller), 1_550); assert_eq!(subnet_consumed(&state), SUBNET_CONSUMED_BEFORE + 1_550); let status = get_refund_status(&state, refundable); @@ -732,7 +721,7 @@ mod tests { }; deliver_canister_http_spent(&mut state, &report, &no_op_logger(), &metrics); - assert_eq!(balance(&state, caller), INITIAL_BALANCE); + assert_eq!(refunded(&state, caller), Cycles::zero()); assert_eq!(consumed(&state, caller), spent.get()); assert_eq!( get_refund_status(&state, allowance * SUBNET_SIZE).refunded_cycles, @@ -767,7 +756,7 @@ mod tests { deliver_canister_http_spent(&mut state, &report, &no_op_logger(), &metrics); // Only node 1's unused allowance is refunded; the other two refund nothing. - assert_eq!(balance(&state, caller), INITIAL_BALANCE + Cycles::new(600)); + assert_eq!(refunded(&state, caller), Cycles::new(600)); assert_eq!(consumed(&state, caller), 4_500); assert_errors(&[(ERROR_SPENT_EXCEEDS_ALLOWANCE, 2)], &metrics_registry); } @@ -792,7 +781,7 @@ mod tests { }; deliver_canister_http_spent(&mut state, &report, &no_op_logger(), &metrics); - assert_eq!(balance(&state, caller), INITIAL_BALANCE); + assert_eq!(refunded(&state, caller), Cycles::zero()); assert_eq!(consumed(&state, caller), spent.get()); assert_eq!( subnet_consumed(&state), @@ -824,7 +813,7 @@ mod tests { }; deliver_canister_http_spent(&mut state, &report, &no_op_logger(), &metrics); - assert_eq!(balance(&state, caller), INITIAL_BALANCE); + assert_eq!(refunded(&state, caller), Cycles::zero()); assert_eq!(consumed(&state, caller), 1_000); assert_eq!(subnet_consumed(&state), SUBNET_CONSUMED_BEFORE + 1_000); let status = get_refund_status(&state, Cycles::zero()); @@ -844,7 +833,7 @@ mod tests { let timeout = UNIX_EPOCH + Duration::from_secs(3 * 60); // > 2min timeout. refund_timed_out_canister_http_contexts(&mut state, timeout, &no_op_logger(), &metrics); - assert_eq!(balance(&state, caller), INITIAL_BALANCE); + assert_eq!(refunded(&state, caller), Cycles::zero()); assert_eq!(consumed(&state, caller), 0); assert_eq!(subnet_consumed(&state), SUBNET_CONSUMED_BEFORE); assert!( @@ -857,7 +846,7 @@ mod tests { assert_errors(&[], &metrics_registry); } - /// Asynchronous reports credit `allowance − spent` per node and report the + /// Asynchronous reports refund `allowance − spent` per node and report the /// per-node spend; a node that has already been accounted is ignored, making /// repeated reports idempotent. #[test] @@ -887,7 +876,7 @@ mod tests { }; deliver_canister_http_spent(&mut state, &first, &log, &metrics); // refund = (1_000 − 400) + (1_000 − 750) = 850; consumed = 1_150. - assert_eq!(balance(&state, caller), INITIAL_BALANCE + Cycles::new(850)); + assert_eq!(refunded(&state, caller), Cycles::new(850)); assert_eq!(consumed(&state, caller), 1_150); assert_eq!( get_refund_status(&state, allowance * 3_usize).refunding_nodes, @@ -908,10 +897,7 @@ mod tests { }; deliver_canister_http_spent(&mut state, &second, &log, &metrics); // Only node 3 is newly accounted: refund += 1_000 − 700 = 300; consumed += 700. - assert_eq!( - balance(&state, caller), - INITIAL_BALANCE + Cycles::new(1_150) - ); + assert_eq!(refunded(&state, caller), Cycles::new(1_150)); assert_eq!(consumed(&state, caller), 1_850); assert_eq!( get_refund_status(&state, allowance * 3_usize).refunding_nodes, @@ -941,7 +927,7 @@ mod tests { }], }; deliver_canister_http_spent(&mut state, &async_report, &log, &metrics); - let balance_after_async = balance(&state, caller); + let refunded_after_async = refunded(&state, caller); let consumed_after_async = consumed(&state, caller); // An initial report now arrives for the same callback; it must be dropped. @@ -955,7 +941,7 @@ mod tests { }; deliver_canister_http_spent(&mut state, &initial_report, &log, &metrics); - assert_eq!(balance(&state, caller), balance_after_async); + assert_eq!(refunded(&state, caller), refunded_after_async); assert_eq!(consumed(&state, caller), consumed_after_async); // Only the node accounted by the asynchronous report is recorded. assert_eq!( @@ -986,7 +972,7 @@ mod tests { asynchronous: vec![], }; deliver_canister_http_spent(&mut state, &initial_report, &log, &metrics); - let balance_after_initial = balance(&state, caller); + let refunded_after_initial = refunded(&state, caller); let consumed_after_initial = consumed(&state, caller); // A late report from node 1, which the initial report already accounted. @@ -999,7 +985,7 @@ mod tests { }; deliver_canister_http_spent(&mut state, &async_report, &log, &metrics); - assert_eq!(balance(&state, caller), balance_after_initial); + assert_eq!(refunded(&state, caller), refunded_after_initial); assert_eq!(consumed(&state, caller), consumed_after_initial); assert_errors(&[(ERROR_DUPLICATE_NODE_REPORT, 1)], &metrics_registry); } @@ -1050,16 +1036,13 @@ mod tests { // refund = (13 * 1_000 − 9_500) + (1_000 − 400) + (1_000 − 600) = 4_500; // consumed = 9_500 + 400 + 600 = 10_500. - assert_eq!( - balance(&state, caller), - INITIAL_BALANCE + Cycles::new(4_500) - ); + assert_eq!(refunded(&state, caller), Cycles::new(4_500)); assert_eq!(consumed(&state, caller), 10_500); assert_eq!(subnet_consumed(&state), SUBNET_CONSUMED_BEFORE + 10_500); assert_errors(&[], &metrics_registry); } - /// The credited refund is capped so that `refunded_cycles` never exceeds + /// The refund is capped so that `refunded_cycles` never exceeds /// `refundable_cycles`, even if the reported allowances would sum to more. #[test] fn refund_is_capped_at_refundable() { @@ -1084,7 +1067,7 @@ mod tests { }; deliver_canister_http_spent(&mut state, &report, &no_op_logger(), &metrics); - assert_eq!(balance(&state, caller), INITIAL_BALANCE + refundable); + assert_eq!(refunded(&state, caller), refundable); // Capping the refund does not affect the reported spend. assert_eq!(consumed(&state, caller), spent.get()); assert_eq!( @@ -1098,7 +1081,7 @@ mod tests { assert_errors(&[(ERROR_REFUND_CAPPED, 1)], &metrics_registry); } - /// A report for an unknown callback is dropped without crediting anything. + /// A report for an unknown callback is dropped without refunding anything. /// This is not an error: contexts of requests that were priced with /// [`PricingVersion::Legacy`] are not retained after they were responded to, /// but their responses still carry a spend report. @@ -1120,18 +1103,20 @@ mod tests { }; deliver_canister_http_spent(&mut state, &report, &no_op_logger(), &metrics); - assert_eq!(balance(&state, caller), INITIAL_BALANCE); + assert_eq!(refunded(&state, caller), Cycles::zero()); assert_eq!(consumed(&state, caller), 0); assert_eq!(subnet_consumed(&state), SUBNET_CONSUMED_BEFORE); assert_errors(&[], &metrics_registry); } - /// A refund for a canister that no longer exists cannot be credited back and - /// is reported as consumed by deleted canisters, so that the cycles do not - /// disappear from the subnet's totals. The spend is reported as consumed as - /// usual. + /// A refund is pooled even for a canister that is not hosted here (whether it + /// was deleted, or migrated away by a subnet split), because it is Message + /// Routing that resolves where the recipient lives. It is also Message Routing + /// that accounts for the cycles as lost, on induction, if the recipient turns + /// out not to exist anywhere. The spend is reported as consumed as usual, but + /// only at the subnet level: there is no local canister to report it on. #[test] - fn report_for_missing_canister_is_reported_as_lost() { + fn report_for_missing_canister_is_pooled_for_message_routing() { let allowance = Cycles::new(1_000); let (mut state, caller) = setup(Some(( Replication::FullyReplicated, @@ -1139,7 +1124,7 @@ mod tests { ))); let (metrics_registry, metrics) = metrics(); - // The caller is deleted before its spend report is delivered. + // The caller is gone by the time its spend report is delivered. state.remove_canister(&caller); let report = CanisterHttpSpent { @@ -1153,14 +1138,16 @@ mod tests { deliver_canister_http_spent(&mut state, &report, &no_op_logger(), &metrics); assert!(state.canister_state(&caller).is_none()); + // refund = 13 * 1_000 − 9_500 = 3_500, pooled rather than written off here. + assert_eq!(refunded(&state, caller), Cycles::new(3_500)); assert_eq!(subnet_consumed(&state), SUBNET_CONSUMED_BEFORE + 9_500); - // refund = 13 * 1_000 − 9_500 = 3_500, all of it lost. - assert_eq!(subnet_lost_by_deleted_canisters(&state), 3_500); + assert_eq!(subnet_lost_by_deleted_canisters(&state), 0); assert_errors(&[], &metrics_registry); } - /// On a free subnet nothing was charged, so a deleted canister loses nothing; - /// the spend is still reported as consumed. + /// On a free subnet nothing was charged, so there is nothing to refund to a + /// canister that is no longer hosted here; the spend is still reported as + /// consumed. #[test] fn free_subnet_report_for_missing_canister_loses_nothing() { let (mut state, caller) = @@ -1179,11 +1166,98 @@ mod tests { }; deliver_canister_http_spent(&mut state, &report, &no_op_logger(), &metrics); + assert_eq!(refunded(&state, caller), Cycles::zero()); assert_eq!(subnet_consumed(&state), SUBNET_CONSUMED_BEFORE + 9_500); assert_eq!(subnet_lost_by_deleted_canisters(&state), 0); assert_errors(&[], &metrics_registry); } + /// Refunds are pooled for Message Routing to deliver, not credited to the + /// caller's balance here. This is what lets them reach a caller that a subnet + /// split has migrated to another subnet: by the time an outcall's refundable + /// cycles are settled (once its spend reports have been applied, or it has + /// timed out), the caller need not be hosted here anymore. + #[test] + fn refunds_are_pooled_rather_than_credited_directly() { + let allowance = Cycles::new(1_000); + let refundable = allowance * SUBNET_SIZE; + let (mut state, caller) = setup(Some((Replication::FullyReplicated, refundable))); + let (metrics_registry, metrics) = metrics(); + + let report = CanisterHttpSpent { + initial: vec![CanisterHttpInitialSpent { + callback: CALLBACK, + amount: Cycles::new(9_500), + nodes: all_nodes(), + }], + asynchronous: vec![], + }; + deliver_canister_http_spent(&mut state, &report, &no_op_logger(), &metrics); + + // The balance is untouched... + assert_eq!(balance(&state, caller), INITIAL_BALANCE); + // ...the refund is in the pool, addressed to the caller... + assert_eq!( + pooled_refunds(&state), + BTreeMap::from([(caller, Cycles::new(3_500))]) + ); + // ...and the consumed cycles are still reported on the caller itself. + assert_eq!(consumed(&state, caller), 9_500); + assert_errors(&[], &metrics_registry); + } + + /// Refunds are pooled per recipient, so reports for different callers result in + /// one pool entry each, rather than in a single merged refund. + #[test] + fn refunds_are_pooled_per_recipient() { + const OTHER_CALLBACK: CallbackId = CallbackId::new(43); + // A second caller, which need not be hosted here for its refund to be + // pooled (see `report_for_missing_canister_is_pooled_for_message_routing`). + let other_caller = canister_test_id(2); + + let allowance = Cycles::new(1_000); + let refundable = allowance * SUBNET_SIZE; + let (mut state, caller) = setup(Some((Replication::FullyReplicated, refundable))); + let other_replication = Replication::FullyReplicated; + let other_refund_status = refund_status(refundable, &other_replication); + insert_context( + &mut state, + OTHER_CALLBACK, + other_caller, + other_replication, + other_refund_status, + CanisterCyclesCostSchedule::Normal, + ); + let (metrics_registry, metrics) = metrics(); + + let report = CanisterHttpSpent { + initial: vec![ + CanisterHttpInitialSpent { + callback: CALLBACK, + amount: Cycles::new(9_500), + nodes: all_nodes(), + }, + CanisterHttpInitialSpent { + callback: OTHER_CALLBACK, + amount: Cycles::new(12_000), + nodes: all_nodes(), + }, + ], + asynchronous: vec![], + }; + deliver_canister_http_spent(&mut state, &report, &no_op_logger(), &metrics); + + // One pool entry per caller: 13_000 − 9_500 and 13_000 − 12_000. + assert_eq!( + pooled_refunds(&state), + BTreeMap::from([ + (caller, Cycles::new(3_500)), + (other_caller, Cycles::new(1_000)), + ]) + ); + assert_errors(&[], &metrics_registry); + } + /// On timeout, the replicas that never responded are refunded their full /// per-replica allowance (here: all 13 of a fully-replicated request), and no /// consumed cycles are reported for them. @@ -1199,7 +1273,10 @@ mod tests { refund_timed_out_canister_http_contexts(&mut state, timeout, &no_op_logger(), &metrics); // No node responded, so all 13 allowances are returned. - assert_eq!(balance(&state, caller), INITIAL_BALANCE + refundable); + assert_eq!(refunded(&state, caller), refundable); + // Pooled, rather than credited to the caller's balance, just like the + // refunds applied from spend reports. + assert_eq!(balance(&state, caller), INITIAL_BALANCE); assert_eq!(consumed(&state, caller), 0); assert_eq!(subnet_consumed(&state), SUBNET_CONSUMED_BEFORE); // The context has been removed. @@ -1237,17 +1314,14 @@ mod tests { }], }; deliver_canister_http_spent(&mut state, &report, &log, &metrics); - assert_eq!( - balance(&state, caller), - INITIAL_BALANCE + allowance * 3_usize - ); + assert_eq!(refunded(&state, caller), allowance * 3_usize); let timeout = UNIX_EPOCH + Duration::from_secs(3 * 60); refund_timed_out_canister_http_contexts(&mut state, timeout, &log, &metrics); // The remaining 10 replicas' allowances are refunded on timeout, for a // total of the full refundable amount. - assert_eq!(balance(&state, caller), INITIAL_BALANCE + refundable); + assert_eq!(refunded(&state, caller), refundable); assert_errors(&[], &metrics_registry); } @@ -1272,7 +1346,7 @@ mod tests { refund_timed_out_canister_http_contexts(&mut state, timeout, &no_op_logger(), &metrics); // The 3 committee members' allowances, not the 13 nodes of the subnet. - assert_eq!(balance(&state, caller), INITIAL_BALANCE + refundable); + assert_eq!(refunded(&state, caller), refundable); assert_errors(&[], &metrics_registry); } @@ -1290,7 +1364,7 @@ mod tests { let timeout = UNIX_EPOCH + Duration::from_secs(3 * 60); refund_timed_out_canister_http_contexts(&mut state, timeout, &no_op_logger(), &metrics); - assert_eq!(balance(&state, caller), INITIAL_BALANCE + allowance); + assert_eq!(refunded(&state, caller), allowance); assert_errors(&[], &metrics_registry); } @@ -1310,7 +1384,7 @@ mod tests { + (DELIVERED_CANISTER_HTTP_REQUEST_CONTEXT_TIMEOUT - Duration::from_nanos(1)); refund_timed_out_canister_http_contexts(&mut state, before_timeout, &log, &metrics); - assert_eq!(balance(&state, caller), INITIAL_BALANCE); + assert_eq!(refunded(&state, caller), Cycles::zero()); assert_eq!( get_refund_status(&state, refundable).refunded_cycles, Cycles::zero() @@ -1320,7 +1394,7 @@ mod tests { let at_timeout = UNIX_EPOCH + DELIVERED_CANISTER_HTTP_REQUEST_CONTEXT_TIMEOUT; refund_timed_out_canister_http_contexts(&mut state, at_timeout, &log, &metrics); - assert_eq!(balance(&state, caller), INITIAL_BALANCE + refundable); + assert_eq!(refunded(&state, caller), refundable); assert!( state .metadata @@ -1348,7 +1422,7 @@ mod tests { refund_timed_out_canister_http_contexts(&mut state, timeout, &no_op_logger(), &metrics); // Uncapped, the 13 non-responders would have been refunded 13_000. - assert_eq!(balance(&state, caller), INITIAL_BALANCE + refundable); + assert_eq!(refunded(&state, caller), refundable); assert_errors(&[(ERROR_REFUND_CAPPED, 1)], &metrics_registry); } } diff --git a/rs/replicated_state/src/replicated_state.rs b/rs/replicated_state/src/replicated_state.rs index 56976d4a5fc0..f89037f7c339 100644 --- a/rs/replicated_state/src/replicated_state.rs +++ b/rs/replicated_state/src/replicated_state.rs @@ -37,7 +37,7 @@ use ic_types::{ time::CoarseTime, }; use ic_types_cycles::{ - CanisterCyclesCostSchedule, CompoundCycles, CyclesAccountManagerSubnetConfig, + CanisterCyclesCostSchedule, CompoundCycles, Cycles, CyclesAccountManagerSubnetConfig, CyclesUseCaseKind, DroppedMessages, }; use ic_validate_eq::ValidateEq; @@ -49,7 +49,7 @@ use std::sync::Arc; use strum_macros::{EnumCount, EnumIter}; #[cfg(debug_assertions)] -use ic_types_cycles::{Cycles, CyclesUseCase, NominalCycles}; +use ic_types_cycles::{CyclesUseCase, NominalCycles}; /// Maximum message length of a synthetic reject response produced by message /// routing. @@ -1105,6 +1105,17 @@ impl ReplicatedState { Ok(()) } + /// Pools `amount` cycles to be refunded to `receiver`, wherever it is hosted. + /// + /// Message Routing routes the pooled refunds (via the loopback stream, if the + /// recipient is local) and credits them on induction, accounting for them as + /// lost if the recipient no longer exists. + /// + /// No-op if `amount` is zero. + pub fn add_refund(&mut self, receiver: CanisterId, amount: Cycles) { + self.refunds.add(receiver, amount); + } + /// Credits the cycles in `refund` to the recipient canister's balance. /// /// Returns `true` if the recipient canister exists and was credited, `false` @@ -1806,7 +1817,6 @@ impl ReplicatedStateMessageRouting for ReplicatedState { pub mod testing { use super::*; - use ic_types_cycles::Cycles; /// Exposes `ReplicatedState` internals for use in other crates' unit tests. pub trait ReplicatedStateTesting { @@ -1830,9 +1840,6 @@ pub mod testing { /// Testing only: Returns the number of messages across all canister and /// subnet output queues. fn output_message_count(&self) -> usize; - - /// Testing only: Adds the given refund to the subnet-wide refund pool. - fn add_refund(&mut self, receiver: CanisterId, amount: Cycles); } impl ReplicatedStateTesting for ReplicatedState { @@ -1865,10 +1872,6 @@ pub mod testing { .sum::() + self.subnet_queues.output_queues_message_count() } - - fn add_refund(&mut self, receiver: CanisterId, amount: Cycles) { - self.refunds.add(receiver, amount); - } } /// Early warning system / stumbling block forcing the authors of changes adding diff --git a/rs/state_manager/src/split/tests.rs b/rs/state_manager/src/split/tests.rs index eeaaff099549..bda1d686fa83 100644 --- a/rs/state_manager/src/split/tests.rs +++ b/rs/state_manager/src/split/tests.rs @@ -18,7 +18,7 @@ use ic_registry_subnet_type::SubnetType; use ic_replicated_state::{ CheckpointLoadingMetrics, ReplicatedState, SystemMetadata, canister_state::canister_snapshots::CanisterSnapshot, - page_map::TestPageAllocatorFileDescriptorImpl, testing::ReplicatedStateTesting, + page_map::TestPageAllocatorFileDescriptorImpl, }; use ic_state_layout::{ CANISTER_FILE, CANISTER_STATES_DIR, CHECKPOINTS_DIR, INGRESS_HISTORY_FILE, ProtoFileWith, diff --git a/rs/types/types/src/messages/inter_canister.rs b/rs/types/types/src/messages/inter_canister.rs index 135635864aa2..829e4707af06 100644 --- a/rs/types/types/src/messages/inter_canister.rs +++ b/rs/types/types/src/messages/inter_canister.rs @@ -558,7 +558,9 @@ impl Hash for Response { } /// XNet message type (like `Request` and `Response`) for guaranteed delivery of -/// refunds for best-effort calls. +/// cycles that are refunded outside of a response: e.g. the payment of a dropped +/// best-effort call; or the unspent part of an HTTP outcall's payment, which is +/// only settled after the response was already delivered. /// /// Represents an _anonymous refund_. ///