diff --git a/Cargo.lock b/Cargo.lock index 83b345c86..a8a105ba4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -833,6 +833,8 @@ dependencies = [ "assert2", "assert_matches", "async-trait", + "base64 0.22.1", + "bincode", "candid", "canlog", "cksol-types", @@ -856,10 +858,14 @@ dependencies = [ "sol_rpc_types", "solana-address 2.3.0", "solana-client", + "solana-hash 4.2.0", "solana-keypair", + "solana-message", "solana-native-token", "solana-signature", + "solana-system-interface 3.1.0", "solana-system-transaction", + "solana-transaction", "tokio", ] @@ -898,6 +904,7 @@ version = "0.1.0" dependencies = [ "askama", "assert_matches", + "async-trait", "base64 0.22.1", "bincode", "canbench-rs", diff --git a/docs/design.md b/docs/design.md index 0d2ae8d7e..e6b239d6b 100644 --- a/docs/design.md +++ b/docs/design.md @@ -363,9 +363,9 @@ The ckSOL minter then fetches the transaction with `getTransaction` and *settles Once settled, the balance of the main account is increased by the amount received. The transaction fee is borne by the batch: for each deposit in the transaction, the ckSOL minter enqueues a *pending mint* of the sweepable amount minus the deposit's share of the fee, `ceil(fee / k)`, to the user's account. Rounding the share up guarantees that the total of the pending mints is never larger than the amount received on the main account, so the ckSOL supply is always covered. The amount to mint of each deposit is recorded in the event together with the amount received, so that replaying the event log restores the pending mints as they were enqueued rather than recomputing them. -Pending mints are processed on a timer: each one is a single `icrc1_transfer` call whose memo contains the sweep signature. The `created_at_time` of the transfer is fixed when the pending mint is enqueued and stored with it, so that every retry sends exactly the same arguments and the ckSOL ledger deduplicates it: a mint whose ledger call fails stays in the queue and is retried on the next run, and a `Duplicate` reply from the ledger is recorded as a successful mint with the block index it carries. Since the ledger only deduplicates transfers within its 24-hour window, a pending mint older than that is quarantined instead of retried. The sweep transaction itself is never resubmitted once it is finalized. +Pending mints are processed on a timer: each one is a single `icrc1_transfer` call whose memo contains the sweep signature and the deposit id. The `created_at_time` of the transfer is fixed when the pending mint is enqueued and stored with it, so that every retry sends exactly the same arguments and the ckSOL ledger deduplicates it: a mint whose ledger call fails stays in the queue and is retried on the next run, and a `Duplicate` reply from the ledger is recorded as a successful mint with the block index it carries. Since the ledger only deduplicates transfers within its 24-hour window, a pending mint older than that is quarantined instead of retried. The sweep transaction itself is never resubmitted once it is finalized. -A user can follow the progress with `deposit_status`, which takes a deposit id and reports its status: `Queued` with the sweepable amount, `Swept` and `Finalized` with the sweep signature, `Minted` with the ledger block index, and `Dropped` and `Quarantined` with the sweep signature when there is one. Since every sweep has its own id, a `Minted` status stays observable after the account has been swept again; the mint itself is also visible on the ckSOL ledger, whose memo links it to the sweep signature. +A user can follow the progress with `deposit_status`, which takes a deposit id and reports its status: `Queued` with the sweepable amount, `Swept` and `Finalized` with the sweep signature, `Minted` with the ledger block index and the minted amount, and `Dropped` and `Quarantined` with the sweep signature when there is one. Since every sweep has its own id, a `Minted` status stays observable after the account has been swept again; the mint itself is also visible on the ckSOL ledger, whose memo links it to the sweep signature and the deposit id. Two failure cases exist before the transaction is finalized, and neither is retried by the ckSOL minter: diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index 6d5741589..779df212d 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -10,6 +10,8 @@ license.workspace = true [dependencies] assert_matches = { workspace = true } async-trait = { workspace = true } +base64 = { workspace = true } +bincode = { workspace = true } candid = { workspace = true } canlog = { workspace = true } cksol-types = { path = "../libs/types" } @@ -31,10 +33,14 @@ sol_rpc_client = { workspace = true } sol_rpc_types = { workspace = true } solana-address = { workspace = true } solana-client = { workspace = true } +solana-hash = { workspace = true } solana-keypair = { workspace = true } +solana-message = { workspace = true } solana-native-token = { workspace = true } solana-signature = { workspace = true } +solana-system-interface = { workspace = true, features = ["bincode"] } solana-system-transaction = { workspace = true } +solana-transaction = { workspace = true, features = ["bincode"] } tokio = { workspace = true } [dev-dependencies] diff --git a/integration_tests/src/fixtures.rs b/integration_tests/src/fixtures.rs index be5f97928..82ea559ce 100644 --- a/integration_tests/src/fixtures.rs +++ b/integration_tests/src/fixtures.rs @@ -1,5 +1,6 @@ -use crate::Setup; +use crate::{Setup, validator::FEE_PER_SIGNATURE}; use async_trait::async_trait; +use base64::{Engine, engine::general_purpose::STANDARD}; use cksol_types::{GetDepositAddressArgs, ProcessDepositArgs, Signature}; use ic_pocket_canister_runtime::{ ExecuteHttpOutcallMocks, JsonRpcRequestMatcher, JsonRpcResponse, MockHttpOutcalls, @@ -222,6 +223,16 @@ impl MockBuilder { .check_signature_statuses(get_signature_statuses_finalized_response()) } + /// Mock for `getTransaction` for a sweep of [`DEFAULT_CALLER_DEPOSIT_ADDRESS`] under the + /// given signature, reporting metadata that credits the minter's main account with the + /// sweepable amount minus the transaction fee of one signature. + pub fn get_sweep_transaction(self, signature: &Signature, sweepable_amount: Lamport) -> Self { + self.expect( + get_transaction_request(&signature.to_string()), + sweep_transaction_response(sweepable_amount), + ) + } + fn check_signature_statuses(self, response: JsonRpcResponse) -> Self { self.expect(get_signature_statuses_request(), response) } @@ -310,6 +321,61 @@ pub fn get_deposit_transaction_response() -> JsonRpcResponse { })) } +/// JSON-RPC `getTransaction` response for a sweep transaction moving the sweepable +/// amount of [`DEFAULT_CALLER_DEPOSIT_ADDRESS`], minus the fee it pays as the only +/// signer, to [`MINTER_ADDRESS`]. +fn sweep_transaction_response(sweepable_amount: Lamport) -> JsonRpcResponse { + const MAIN_BALANCE_BEFORE_SWEEP: Lamport = 5_000_000_000; + let deposit_address: Address = DEFAULT_CALLER_DEPOSIT_ADDRESS.parse().unwrap(); + let transfer_amount = sweepable_amount - FEE_PER_SIGNATURE; + let message = solana_message::Message::new_with_blockhash( + &[solana_system_interface::instruction::transfer( + &deposit_address, + &MINTER_ADDRESS, + transfer_amount, + )], + Some(&deposit_address), + &solana_hash::Hash::default(), + ); + let transaction = solana_transaction::versioned::VersionedTransaction::from( + solana_transaction::Transaction::new_unsigned(message), + ); + let encoded_transaction = STANDARD.encode( + bincode::serialize(&transaction).expect("serializing the transaction should succeed"), + ); + JsonRpcResponse::from(json!({ + "jsonrpc": "2.0", + "result": { + "blockTime": 1700000000_i64, + "meta": { + "computeUnitsConsumed": 150, + "err": null, + "fee": FEE_PER_SIGNATURE, + "innerInstructions": [], + "loadedAddresses": { "readonly": [], "writable": [] }, + "logMessages": [], + "postBalances": [ + RENT_EXEMPTION_THRESHOLD, + MAIN_BALANCE_BEFORE_SWEEP + transfer_amount, + 1 + ], + "postTokenBalances": [], + "preBalances": [ + RENT_EXEMPTION_THRESHOLD + sweepable_amount, + MAIN_BALANCE_BEFORE_SWEEP, + 1 + ], + "preTokenBalances": [], + "rewards": [], + "status": { "Ok": null } + }, + "slot": 350_000_000_u64, + "transaction": [encoded_transaction, "base64"] + }, + "id": 1 + })) +} + fn get_balance_request() -> JsonRpcRequestMatcher { JsonRpcRequestMatcher::with_method("getBalance") } diff --git a/integration_tests/src/ledger_init_args.rs b/integration_tests/src/ledger_init_args.rs index 3fc297321..67c79800f 100644 --- a/integration_tests/src/ledger_init_args.rs +++ b/integration_tests/src/ledger_init_args.rs @@ -1,11 +1,16 @@ //! Since the init args for the ICRC1 ledger are not published in a public crate, //! redefine them here to initialize the canister correctly. use candid::{CandidType, Deserialize, Nat, Principal}; -use cksol_types::MAX_SERIALIZED_MEMO_BYTES; use icrc_ledger_types::{icrc::generic_value::Value, icrc1::account::Account}; use serde::Serialize; pub const LEDGER_TRANSFER_FEE: u64 = 50; +/// The `max_memo_length` the ckSOL ledger is deployed with, matching the value most +/// ICRC-1 ledgers use. +/// +/// One byte below the largest memo the minter can produce, so the mint of a deposit +/// id beyond `u32::MAX` would not fit. Raising it takes a ledger upgrade. +const LEDGER_MAX_MEMO_BYTES: u16 = 80; const NNS_ROOT_PRINCIPAL: Principal = Principal::from_slice(&[0_u8, 0, 0, 0, 0, 0, 0, 3, 1, 1]); const FEE_COLLECTOR_SUBACCOUNT: [u8; 32] = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x0f, @@ -42,7 +47,7 @@ pub fn ledger_init_args( cycles_for_archive_creation: Some(100_000_000_000_000), max_transactions_per_response: None, }, - max_memo_length: Some(MAX_SERIALIZED_MEMO_BYTES), + max_memo_length: Some(LEDGER_MAX_MEMO_BYTES), feature_flags: None, index_principal: None, }) diff --git a/integration_tests/src/lib.rs b/integration_tests/src/lib.rs index 70cc2e7ea..824083b65 100644 --- a/integration_tests/src/lib.rs +++ b/integration_tests/src/lib.rs @@ -91,6 +91,9 @@ impl SetupBuilder { pub struct Setup { env: Option, + /// Whether the instance produces blocks on its own, which decides how + /// [`Setup::advance_time_and_settle`] lets a timer round run. + auto_progress: bool, minter_canister_id: CanisterId, ledger_canister_id: CanisterId, sol_rpc_canister_id: CanisterId, @@ -204,7 +207,8 @@ impl Setup { None }; - let env = if let PocketIcMode::LiveMode = make_live { + let auto_progress = matches!(make_live, PocketIcMode::LiveMode); + let env = if auto_progress { let mut env = env; let _ = env.make_live(None).await; env @@ -218,6 +222,7 @@ impl Setup { Self { env: Some(env), + auto_progress, minter_canister_id, ledger_canister_id, sol_rpc_canister_id, @@ -317,13 +322,38 @@ impl Setup { self.env.as_ref().unwrap().advance_time(duration).await } - /// Advances time and then lets the timers that became due complete their - /// HTTP outcalls. An outcall still in flight when time is advanced again - /// times out, and a timer needs several sequential outcalls per round. + /// Advances time and then lets the timers that became due complete their round. + /// + /// An instance that produces blocks on its own only needs wall-clock time for the + /// HTTP outcalls of the round to come back, and an outcall still in flight when + /// time is advanced again times out. An instance that does not produce blocks runs + /// nothing until it is ticked, and a round takes several sequential messages. pub async fn advance_time_and_settle(&self, duration: Duration) { const OUTCALL_SETTLE_DELAY: Duration = Duration::from_secs(2); + const TICKS_PER_ROUND: usize = 10; + self.advance_time(duration).await; - tokio::time::sleep(OUTCALL_SETTLE_DELAY).await; + if self.auto_progress { + tokio::time::sleep(OUTCALL_SETTLE_DELAY).await; + } else { + for _ in 0..TICKS_PER_ROUND { + self.tick().await; + } + } + } + + /// Advances time until the deposit is minted and returns the minted amount. + pub async fn wait_for_deposit_minted(&self, deposit_id: DepositSolId) -> Lamport { + const MAX_ITERATIONS: usize = 30; + for _ in 0..MAX_ITERATIONS { + if let DepositSolStatus::Minted { minted_amount, .. } = + self.minter().deposit_status(deposit_id).await + { + return minted_amount; + } + self.advance_time_and_settle(Duration::from_mins(1)).await; + } + panic!("Deposit {deposit_id} was not minted within timeout"); } pub async fn execute_http_mocks(&self, mut mocks: impl ExecuteHttpOutcallMocks) { diff --git a/integration_tests/tests/solana_test_validator.rs b/integration_tests/tests/solana_test_validator.rs index 3e4e6ae9c..c82a49bda 100644 --- a/integration_tests/tests/solana_test_validator.rs +++ b/integration_tests/tests/solana_test_validator.rs @@ -345,5 +345,17 @@ async fn should_sweep_a_full_batch_of_deposits_in_one_transaction() { ); } + for ((&deposit_id, &account), &deposit_amount) in + deposit_ids.iter().zip(&accounts).zip(&deposit_amounts) + { + let expected_minted_amount = deposit_amount - RENT_EXEMPTION_THRESHOLD - FEE_PER_SIGNATURE; + let minted_amount = setup.wait_for_deposit_minted(deposit_id).await; + assert_eq!(minted_amount, expected_minted_amount); + assert_eq!( + setup.ledger().balance_of(account).await, + expected_minted_amount + ); + } + setup.drop().await; } diff --git a/integration_tests/tests/tests.rs b/integration_tests/tests/tests.rs index fc38a30c5..ed9592e39 100644 --- a/integration_tests/tests/tests.rs +++ b/integration_tests/tests/tests.rs @@ -9,6 +9,7 @@ use cksol_int_tests::{ default_process_deposit_args, deposit_transaction_signature, get_deposit_transaction_response, }, + validator::FEE_PER_SIGNATURE, }; use cksol_types::{ DepositId, DepositSolArgs, DepositSolError, DepositSolStatus, DepositStatus, @@ -1191,7 +1192,8 @@ mod deposit_sol_tests { } #[tokio::test] - async fn should_sweep_queued_deposit_after_timer() { + async fn should_sweep_finalize_and_mint_queued_deposit_after_timers() { + const SWEEPABLE_AMOUNT: Lamport = BALANCE_ABOVE_MINIMUM - RENT_EXEMPTION_THRESHOLD; let setup = SetupBuilder::new().with_proxy_canister().build().await; let deposit_id = setup .minter() @@ -1228,6 +1230,31 @@ mod deposit_sol_tests { ))); }); + setup.advance_time(FINALIZE_TRANSACTIONS_DELAY).await; + setup + .execute_http_mocks( + MockBuilder::with_start_id(16) + .finalize_transaction(SUBMISSION_BLOCK_HEIGHT) + .get_sweep_transaction(&sweep_signature, SWEEPABLE_AMOUNT) + .build(), + ) + .await; + + setup.wait_for_deposit_minted(deposit_id).await; + + let minted_amount = SWEEPABLE_AMOUNT - FEE_PER_SIGNATURE; + assert_eq!( + setup.minter().deposit_status(deposit_id).await, + DepositSolStatus::Minted { + block_index: 0, + minted_amount, + } + ); + assert_eq!( + setup.ledger().balance_of(DEFAULT_CALLER_ACCOUNT).await, + minted_amount + ); + setup.drop().await; } @@ -1364,7 +1391,9 @@ mod deposit_sol_tests { let setup = setup .check_metrics() .await - .assert_contains_metric_matching(r"quarantined_swept_deposits 1 \d+") + .assert_contains_metric_matching( + r#"quarantined_swept_deposits\{cause="sweep_unreadable"\} 1 \d+"#, + ) .into(); let result = setup.minter().deposit_sol(DEFAULT_CALLER_ACCOUNT).await; diff --git a/libs/types-internal/src/event.rs b/libs/types-internal/src/event.rs index 3e3b7e1ac..368fb8a28 100644 --- a/libs/types-internal/src/event.rs +++ b/libs/types-internal/src/event.rs @@ -150,6 +150,31 @@ pub enum EventType { /// The chain code used to derive subkeys (32 bytes). chain_code: Vec, }, + /// The minter minted ckSOL on the ledger for a swept deposit whose sweep + /// was credited. + MintedSweptDeposit { + /// The identifier of the minted deposit. + deposit_id: u64, + /// The mint transaction index on the ckSOL ledger. + mint_block_index: u64, + }, + /// The pending mint of a swept deposit cannot be retried: either it became + /// older than the 24-hour deduplication window of the ckSOL ledger, or the + /// ledger definitively rejected it. Retrying the transfer with the same + /// arguments fails forever, and fresh arguments could double mint. + /// + /// The deposit is quarantined to avoid any double minting and will not be + /// further processed without manual intervention. + /// + /// If the minter was down past the deduplication window, the underlying + /// transfer may nevertheless have landed on the ledger. Manual resolution + /// must therefore first search the ledger for a mint whose memo carries the + /// sweep signature before crediting by hand, otherwise a double mint + /// results. + QuarantinedPendingMint { + /// The identifier of the deposit whose pending mint was quarantined. + deposit_id: u64, + }, } /// The mint enqueued for one deposit of a `CreditedSweep` event. diff --git a/libs/types/src/lib.rs b/libs/types/src/lib.rs index 9f2e5bf0d..f6662b25f 100644 --- a/libs/types/src/lib.rs +++ b/libs/types/src/lib.rs @@ -123,8 +123,6 @@ impl From for DepositSolArgs { pub type DepositSolId = u64; /// The status of a deposit queued by the `deposit_sol` ckSOL minter endpoint. -/// -/// The `Minted` variant will follow as the sweep flow is implemented. #[derive(Clone, Eq, PartialEq, Debug, CandidType, Deserialize, Serialize)] pub enum DepositSolStatus { /// No deposit with this identifier was queued. @@ -146,6 +144,14 @@ pub enum DepositSolStatus { /// The signature of the sweep transaction. signature: Signature, }, + /// The minter minted ckSOL for the deposit on the ledger. + Minted { + /// The mint transaction index on the ckSOL ledger. + block_index: LedgerMintIndex, + /// The minted amount: the swept amount minus the deposit's share of the + /// transaction fee of the sweep. + minted_amount: Lamport, + }, /// The sweep transaction failed, or expired without ever being seen on chain, so no /// ckSOL is owed. Calling `deposit_sol` again queues a new sweep of whatever balance /// the deposit address still holds. @@ -153,12 +159,15 @@ pub enum DepositSolStatus { /// The signature of the sweep transaction. signature: Signature, }, - /// The sweep transaction was finalized, but its outcome did not match the plan the - /// minter submitted it with, so the amount to credit cannot be determined safely and - /// no ckSOL was minted. This is not expected to happen and the minter does not - /// process the deposit any further: releasing or crediting it requires a minter - /// upgrade. Meanwhile the account stays in flight, so `deposit_sol` keeps rejecting - /// it and a new deposit has to use a different subaccount. + /// The minter stopped processing the deposit: either the finalized sweep's outcome + /// did not match the plan the minter submitted it with, so the amount to credit + /// cannot be determined safely and no ckSOL was minted, or the ckSOL mint of the + /// credited deposit could not be completed, in which case the mint may nevertheless + /// have landed on the ledger. This is not expected to happen and resolving it + /// requires a minter upgrade: before minting by hand, search the ledger for a mint + /// whose memo carries the sweep signature and the deposit id, otherwise a double + /// mint results. Meanwhile the account stays in flight, so `deposit_sol` keeps + /// rejecting it and a new deposit has to use a different subaccount. Quarantined { /// The signature of the sweep transaction. signature: Signature, diff --git a/libs/types/src/memo/mod.rs b/libs/types/src/memo/mod.rs index b3cc3606a..a1b84e1bc 100644 --- a/libs/types/src/memo/mod.rs +++ b/libs/types/src/memo/mod.rs @@ -1,3 +1,4 @@ +use crate::DepositSolId; use derive_more::From; use minicbor::{Decode, Encode, Encoder}; use solana_address::Address; @@ -8,6 +9,26 @@ mod tests; /// Maximum size in bytes of a [`Memo`] when serialized into an [ICRC-1 memo]. /// +/// The largest memo is a [`MintMemo::Sweep`], which encodes to 81 bytes: +/// +/// | Bytes | Content | +/// | --- | --- | +/// | 2 | the [`Memo`] enum, as an array holding the index of [`Memo::Mint`] | +/// | 1 | the array holding the single field of [`Memo::Mint`] | +/// | 2 | the [`MintMemo`] enum, as an array holding the index of [`MintMemo::Sweep`] | +/// | 1 | the array holding the two fields of [`MintMemo::Sweep`] | +/// | 2 | the byte string header of the signature, whose length needs a byte of its own | +/// | 64 | the sweep signature | +/// | 9 | the deposit id, as an unsigned integer | +/// +/// Only the deposit id varies, since CBOR encodes an integer in as few bytes as +/// possible: it takes nine bytes above [`u32::MAX`] and at most five below, which +/// keeps the memo within 80 bytes for every deposit id reachable in practice. +/// +/// A ledger whose `max_memo_length` is below this value rejects the mints whose memo +/// exceeds its limit, and one below the size of the smallest memo, such as the ICRC-1 +/// default of 32 bytes, rejects all of them. +/// /// # Example /// /// ```rust @@ -26,7 +47,7 @@ mod tests; /// ``` /// /// [ICRC-1 memo]: icrc_ledger_types::icrc1::transfer::Memo -pub const MAX_SERIALIZED_MEMO_BYTES: u16 = 80; +pub const MAX_SERIALIZED_MEMO_BYTES: u16 = 81; /// A ckSOL minter ledger memo. #[derive(Clone, Eq, PartialEq, Debug, Decode, Encode, From)] @@ -49,6 +70,16 @@ pub enum MintMemo { #[cbor(n(0), with = "minicbor::bytes")] signature: [u8; 64], }, + /// The minter converted a deposit swept to its main account to ckSOL. + #[n(1)] + Sweep { + /// The transaction signature of the sweep that moved the deposit. + #[cbor(n(0), with = "minicbor::bytes")] + signature: [u8; 64], + /// The identifier of the swept deposit in the minter. + #[n(1)] + deposit_id: DepositSolId, + }, } /// The minter burned some ckSOL tokens. @@ -72,6 +103,20 @@ impl MintMemo { signature: <[u8; SIGNATURE_BYTES]>::from(signature.into()), } } + + /// Create a [`MintMemo::Sweep`] memo instance from the [`Signature`] of the sweep + /// and the identifier of the swept deposit. + /// + /// [`Signature`]: solana_signature::Signature + pub fn sweep( + signature: impl Into, + deposit_id: DepositSolId, + ) -> Self { + Self::Sweep { + signature: <[u8; SIGNATURE_BYTES]>::from(signature.into()), + deposit_id, + } + } } impl BurnMemo { diff --git a/libs/types/src/memo/tests.rs b/libs/types/src/memo/tests.rs index 57d0643ca..a6c2b8aa5 100644 --- a/libs/types/src/memo/tests.rs +++ b/libs/types/src/memo/tests.rs @@ -1,9 +1,41 @@ use crate::{ - BurnMemo, + BurnMemo, DepositSolId, memo::{MAX_SERIALIZED_MEMO_BYTES, Memo as CkSolMinterMemo, MintMemo}, }; use icrc_ledger_types::icrc1::transfer::Memo as Icrc1Memo; -use proptest::prelude::*; +use proptest::{ + array::uniform, + prelude::{Strategy, any}, + prop_assert, prop_oneof, proptest, +}; + +/// The `max_memo_length` that ICRC-1 ledgers are typically deployed with. +const TYPICAL_LEDGER_MAX_MEMO_BYTES: usize = 80; + +/// The largest deposit id whose mint memo still fits into a typical ledger memo. +/// +/// CBOR spends five bytes on an unsigned integer up to [`u32::MAX`] and nine on any +/// larger one, so the memo outgrows [`TYPICAL_LEDGER_MAX_MEMO_BYTES`] exactly there. +const LARGEST_DEPOSIT_ID_WITHIN_TYPICAL_LEDGER_MEMO: DepositSolId = u32::MAX as DepositSolId; + +#[test] +fn should_reach_the_maximum_size_with_the_largest_sweep_memo() { + assert_eq!( + encoded_len(sweep_memo(DepositSolId::MAX)), + MAX_SERIALIZED_MEMO_BYTES as usize + ); +} + +#[test] +fn should_outgrow_a_typical_ledger_memo_one_deposit_id_past_the_largest_u32() { + let largest = LARGEST_DEPOSIT_ID_WITHIN_TYPICAL_LEDGER_MEMO; + + let within = encoded_len(sweep_memo(largest)); + let beyond = encoded_len(sweep_memo(largest + 1)); + + assert!(within <= TYPICAL_LEDGER_MAX_MEMO_BYTES); + assert!(beyond > TYPICAL_LEDGER_MAX_MEMO_BYTES); +} proptest! { #[test] @@ -12,6 +44,37 @@ proptest! { prop_assert!(encoded.0.len() <= MAX_SERIALIZED_MEMO_BYTES as usize); } + + #[test] + fn should_fit_into_a_typical_ledger_memo_up_to_the_largest_u32_deposit_id( + memo in arb_sweep_mint_memo(0..=LARGEST_DEPOSIT_ID_WITHIN_TYPICAL_LEDGER_MEMO), + ) { + let encoded = Icrc1Memo::from(CkSolMinterMemo::Mint(memo)); + + prop_assert!(encoded.0.len() <= TYPICAL_LEDGER_MAX_MEMO_BYTES); + } + + #[test] + fn should_outgrow_a_typical_ledger_memo_beyond_the_largest_u32_deposit_id( + memo in arb_sweep_mint_memo( + (LARGEST_DEPOSIT_ID_WITHIN_TYPICAL_LEDGER_MEMO + 1)..=DepositSolId::MAX, + ), + ) { + let encoded = Icrc1Memo::from(CkSolMinterMemo::Mint(memo)); + + prop_assert!(encoded.0.len() > TYPICAL_LEDGER_MAX_MEMO_BYTES); + } +} + +fn sweep_memo(deposit_id: DepositSolId) -> CkSolMinterMemo { + CkSolMinterMemo::Mint(MintMemo::Sweep { + signature: [0xAB; 64], + deposit_id, + }) +} + +fn encoded_len(memo: CkSolMinterMemo) -> usize { + Icrc1Memo::from(memo).0.len() } fn arb_memo() -> impl Strategy { @@ -22,13 +85,25 @@ fn arb_memo() -> impl Strategy { } fn arb_mint_memo() -> impl Strategy { - arb_signature().prop_map(|signature| MintMemo::Convert { + prop_oneof![ + arb_signature().prop_map(|signature| MintMemo::Convert { + signature: signature.into(), + }), + arb_sweep_mint_memo(any::()), + ] +} + +fn arb_sweep_mint_memo( + deposit_ids: impl Strategy, +) -> impl Strategy { + (arb_signature(), deposit_ids).prop_map(|(signature, deposit_id)| MintMemo::Sweep { signature: signature.into(), + deposit_id, }) } fn arb_signature() -> impl Strategy { - prop::array::uniform::<_, 64>(any::()).prop_map(solana_signature::Signature::from) + uniform::<_, 64>(any::()).prop_map(solana_signature::Signature::from) } fn arb_burn_memo() -> impl Strategy { diff --git a/minter/Cargo.toml b/minter/Cargo.toml index b6aabed83..a33b19ef2 100644 --- a/minter/Cargo.toml +++ b/minter/Cargo.toml @@ -56,6 +56,7 @@ solana-transaction-status-client-types = { workspace = true } thiserror = { workspace = true } [dev-dependencies] +async-trait = { workspace = true } base64 = { workspace = true } candid_parser = { workspace = true } mockall = { workspace = true } diff --git a/minter/cksol_minter.did b/minter/cksol_minter.did index 18c49fada..f2ecce9d9 100644 --- a/minter/cksol_minter.did +++ b/minter/cksol_minter.did @@ -141,8 +141,6 @@ type DepositSolArgs = record { type DepositSolId = nat64; // The status of a deposit queued by the `deposit_sol` endpoint. -// -// The `Minted` variant will follow as the sweep flow is implemented. type DepositSolStatus = variant { // No deposit with this identifier was queued. NotFound; @@ -163,6 +161,14 @@ type DepositSolStatus = variant { // The signature of the sweep transaction. signature: Signature; }; + // The minter minted ckSOL for the deposit on the ledger. + Minted : record { + // The mint transaction index on the ckSOL ledger. + block_index: nat64; + // The minted amount: the swept amount minus the deposit's share of the + // transaction fee of the sweep. + minted_amount: Lamport; + }; // The sweep transaction failed, or expired without ever being seen on chain, // so no ckSOL is owed. Calling `deposit_sol` again queues a new sweep of // whatever balance the deposit address still holds. @@ -170,13 +176,16 @@ type DepositSolStatus = variant { // The signature of the sweep transaction. signature: Signature; }; - // The sweep transaction was finalized, but its outcome did not match the - // plan the minter submitted it with, so the amount to credit cannot be - // determined safely and no ckSOL was minted. This is not expected to happen - // and the minter does not process the deposit any further: releasing or - // crediting it requires a minter upgrade. Meanwhile the account stays in - // flight, so `deposit_sol` keeps rejecting it and a new deposit has to - // use a different subaccount. + // The minter stopped processing the deposit: either the finalized sweep's + // outcome did not match the plan the minter submitted it with, so the + // amount to credit cannot be determined safely and no ckSOL was minted, or + // the ckSOL mint of the credited deposit could not be completed, in which + // case the mint may nevertheless have landed on the ledger. This is not + // expected to happen and resolving it requires a minter upgrade: before + // minting by hand, search the ledger for a mint whose memo carries the + // sweep signature and the deposit id, otherwise a double mint results. + // Meanwhile the account stays in flight, so `deposit_sol` keeps rejecting + // it and a new deposit has to use a different subaccount. Quarantined : record { // The signature of the sweep transaction. signature: Signature; @@ -553,6 +562,31 @@ type EventType = variant { // The chain code used to derive subkeys (32 bytes). chain_code: blob; }; + // The minter minted ckSOL on the ledger for a swept deposit whose sweep + // was credited. + MintedSweptDeposit : record { + // The identifier of the minted deposit. + deposit_id: DepositSolId; + // The mint transaction index on the ckSOL ledger. + mint_block_index: nat64; + }; + // The pending mint of a swept deposit cannot be retried: either it became + // older than the 24-hour deduplication window of the ckSOL ledger, or the + // ledger definitively rejected it. Retrying the transfer with the same + // arguments fails forever, and fresh arguments could double mint. + // + // The deposit is quarantined to avoid any double minting and will not be + // further processed without manual intervention. + // + // If the minter was down past the deduplication window, the underlying + // transfer may nevertheless have landed on the ledger. Manual resolution + // must therefore first search the ledger for a mint whose memo carries the + // sweep signature before crediting by hand, otherwise a double mint + // results. + QuarantinedPendingMint : record { + // The identifier of the deposit whose pending mint was quarantined. + deposit_id: DepositSolId; + }; }; // A single transaction can deposit to multiple accounts, so the signature alone diff --git a/minter/src/constants.rs b/minter/src/constants.rs index 38d95f959..31cc0b73b 100644 --- a/minter/src/constants.rs +++ b/minter/src/constants.rs @@ -15,6 +15,21 @@ pub const SWEEP_DEPOSITS_DELAY: Duration = Duration::from_mins(1); /// transfer creation stay unavailable until the key is recorded, so the retry is quick. pub const MINTER_PUBLIC_KEY_RETRY_DELAY: Duration = Duration::from_secs(5); +/// Interval of the timer minting the pending mints of credited sweeps. +pub const PROCESS_PENDING_MINTS_DELAY: Duration = Duration::from_mins(1); + +/// Maximum number of pending mints sent to the ckSOL ledger, one after the other, +/// in a single round of the minting timer. +pub const MAX_PENDING_MINTS_PER_ROUND: usize = 10; + +/// The window within which the ckSOL ledger deduplicates transfers by their +/// `created_at_time`, matching `TRANSACTION_WINDOW` of the ICRC-1 ledger: +/// +/// +/// Retrying a pending mint beyond this window is no longer deduplicated, so a +/// pending mint older than that is quarantined instead of retried. +pub const LEDGER_DEDUPLICATION_WINDOW: Duration = Duration::from_hours(24); + /// Matches the ICP HTTPS outcall response limit for variable-length RPC calls /// such as `getTransaction` and `getSignatureStatuses`: /// https://docs.internetcomputer.org/references/ic-interface-spec#ic-http_request diff --git a/minter/src/dashboard/mod.rs b/minter/src/dashboard/mod.rs index 06f6552a1..75b882089 100644 --- a/minter/src/dashboard/mod.rs +++ b/minter/src/dashboard/mod.rs @@ -1,6 +1,6 @@ use crate::{ address::minter_address, - state::{ConsolidationTransaction, State}, + state::{ConsolidationTransaction, QuarantineCause, State}, }; use askama::Template; use candid::Principal; @@ -40,6 +40,7 @@ pub(crate) const DEFAULT_PAGE_SIZE: usize = 100; pub struct DashboardPaginationParameters { pub quarantined_swept_deposits_start: usize, pub minted_deposits_start: usize, + pub minted_sweeps_start: usize, pub withdrawals_start: usize, pub consolidations_start: usize, } @@ -57,6 +58,7 @@ impl DashboardPaginationParameters { Ok(Self { quarantined_swept_deposits_start: parse(req, "quarantined_swept_deposits_start")?, minted_deposits_start: parse(req, "minted_deposits_start")?, + minted_sweeps_start: parse(req, "minted_sweeps_start")?, withdrawals_start: parse(req, "withdrawals_start")?, consolidations_start: parse(req, "consolidations_start")?, }) @@ -70,6 +72,7 @@ impl DashboardPaginationParameters { self.quarantined_swept_deposits_start, ), ("minted_deposits_start", self.minted_deposits_start), + ("minted_sweeps_start", self.minted_sweeps_start), ("withdrawals_start", self.withdrawals_start), ("consolidations_start", self.consolidations_start), ] @@ -97,7 +100,36 @@ impl DashboardPaginatedTable { page_offset_query_param: &str, other_query_params: String, ) -> Self { - let total_items = items.len(); + Self::from_rows( + items.iter(), + Clone::clone, + current_page_offset, + page_size, + num_cols, + table_reference, + page_offset_query_param, + other_query_params, + ) + } + + /// Paginates without materializing the whole table: only the rows of the current page + /// go through `to_row`, so rendering one page stays constant in the number of rows. + #[allow(clippy::too_many_arguments)] + pub fn from_rows( + rows: I, + to_row: F, + current_page_offset: usize, + page_size: usize, + num_cols: usize, + table_reference: &str, + page_offset_query_param: &str, + other_query_params: String, + ) -> Self + where + I: ExactSizeIterator, + F: Fn(I::Item) -> T, + { + let total_items = rows.len(); // Align offset to page boundary and clamp to the last valid page. let offset = if page_size == 0 || total_items == 0 { @@ -109,7 +141,7 @@ impl DashboardPaginatedTable { }; Self { - current_page: items.iter().skip(offset).take(page_size).cloned().collect(), + current_page: rows.skip(offset).take(page_size).map(to_row).collect(), pagination: DashboardTablePagination::new( total_items, offset, @@ -210,7 +242,19 @@ pub struct DashboardQuarantinedDeposit { pub deposit_id: String, pub account: String, pub signature: String, + pub cause: &'static str, pub planned_amount: String, + /// The mint the deposit is still owed, or `-` when the sweep was never credited. + pub amount_to_mint: String, +} + +/// A swept deposit whose ckSOL mint landed on the ledger. +#[derive(Clone)] +pub struct DashboardMintedSweep { + pub deposit_id: String, + pub account: String, + pub minted_amount: String, + pub mint_block_index: String, } #[derive(Clone)] @@ -239,6 +283,7 @@ pub struct DashboardTemplate { pub minimum_withdrawal_amount: String, pub balance: String, pub quarantined_swept_deposits_table: DashboardPaginatedTable, + pub minted_sweeps_table: DashboardPaginatedTable, pub deposits_table: DashboardPaginatedTable, pub consolidations_table: DashboardPaginatedTable, pub withdrawals_table: DashboardPaginatedTable, @@ -312,21 +357,47 @@ impl DashboardTemplate { .rev() .map(|(deposit_id, quarantined)| DashboardQuarantinedDeposit { deposit_id: deposit_id.to_string(), - account: quarantined.deposit.account.to_string(), - signature: quarantined.signature.to_string(), - planned_amount: lamports_to_sol(quarantined.deposit.sweepable_amount()), + account: quarantined.account().to_string(), + signature: quarantined.sweep_signature().to_string(), + cause: match quarantined.cause { + QuarantineCause::SweepUnreadable => "Sweep unreadable", + QuarantineCause::MintUnresolved { .. } => "Mint unresolved", + }, + planned_amount: lamports_to_sol(quarantined.planned_amount()), + amount_to_mint: match quarantined.cause { + QuarantineCause::SweepUnreadable => "-".to_string(), + QuarantineCause::MintUnresolved { amount_to_mint, .. } => { + lamports_to_sol(amount_to_mint) + } + }, }) .collect(); let quarantined_swept_deposits_table = DashboardPaginatedTable::from_items( &quarantined_swept_deposits, pagination.quarantined_swept_deposits_start, DEFAULT_PAGE_SIZE, - 4, + 6, "quarantined-swept-deposits", "quarantined_swept_deposits_start", pagination.other_params("quarantined_swept_deposits_start"), ); + let minted_sweeps_table = DashboardPaginatedTable::from_rows( + state.deposits().minted().iter().rev(), + |(deposit_id, minted)| DashboardMintedSweep { + deposit_id: deposit_id.to_string(), + account: minted.deposit.deposit.account.to_string(), + minted_amount: lamports_to_sol(minted.minted_amount), + mint_block_index: minted.mint_block_index.to_string(), + }, + pagination.minted_sweeps_start, + DEFAULT_PAGE_SIZE, + 4, + "minted-sweeps", + "minted_sweeps_start", + pagination.other_params("minted_sweeps_start"), + ); + let deposits_table = DashboardPaginatedTable::from_items( &deposits, pagination.minted_deposits_start, @@ -471,6 +542,7 @@ impl DashboardTemplate { minimum_withdrawal_amount: lamports_to_sol(state.minimum_withdrawal_amount()), balance: lamports_to_sol(state.balance()), quarantined_swept_deposits_table, + minted_sweeps_table, deposits_table, consolidations_table, withdrawals_table, diff --git a/minter/src/dashboard/tests.rs b/minter/src/dashboard/tests.rs index b1de0f492..4dac182c9 100644 --- a/minter/src/dashboard/tests.rs +++ b/minter/src/dashboard/tests.rs @@ -8,6 +8,7 @@ use crate::test_fixtures::{ quarantine_sweep, queue_deposit, submit_consolidation, submit_sweep, submit_withdrawal, succeed_transaction, }, + flow::deposit::DepositFlow, init_balance, init_schnorr_master_key, init_state, init_state_with_args, ledger_canister_id, signature, sol_rpc_canister_id, valid_init_args, }; @@ -201,7 +202,9 @@ fn should_display_quarantined_swept_deposits_with_the_sweep_signature() { "0", &account(1).to_string(), &sweep_signature.to_string(), + "Sweep unreadable", &lamports_to_sol(sweepable_amount), + "-", ], "quarantined swept deposits", ) @@ -211,6 +214,90 @@ fn should_display_quarantined_swept_deposits_with_the_sweep_signature() { ); } +#[test] +fn should_display_a_quarantined_pending_mint_with_the_amount_it_is_owed() { + init_state(); + init_schnorr_master_key(); + let sweepable_amount = 400_000_000; + let pending = DepositFlow::queue(account(1), sweepable_amount) + .sweep(signature(0xAA)) + .succeed() + .credit() + .single_pending_mint(); + let amount_to_mint = pending.amount_to_mint; + let quarantined = pending.quarantine(); + + DashboardAssert::assert_that(dashboard()).has_table_row_value( + "#quarantined-swept-deposits + table > tbody > tr:nth-child(1)", + &[ + &quarantined.deposit_id.to_string(), + &quarantined.account.to_string(), + &quarantined.sweep_signature.to_string(), + "Mint unresolved", + &lamports_to_sol(sweepable_amount), + &lamports_to_sol(amount_to_mint), + ], + "quarantined swept deposits", + ); +} + +#[test] +fn should_display_minted_swept_deposits_with_amount_and_block_index() { + init_state(); + init_schnorr_master_key(); + let minted = DepositFlow::queue(account(1), 400_000_000) + .sweep(signature(0xAA)) + .succeed() + .credit() + .single_pending_mint() + .mint(42); + + DashboardAssert::assert_that(dashboard()).has_table_row_value( + "#minted-sweeps + table > tbody > tr:nth-child(1)", + &[ + &minted.deposit_id.to_string(), + &minted.account.to_string(), + &lamports_to_sol(minted.minted_amount), + &minted.mint_block_index.to_string(), + ], + "minted swept deposits", + ); +} + +#[test] +fn should_paginate_minted_swept_deposits_across_multiple_pages() { + use crate::dashboard::DEFAULT_PAGE_SIZE; + + init_state(); + init_schnorr_master_key(); + + let total_minted_sweeps = DEFAULT_PAGE_SIZE + 1; + for i in 0..total_minted_sweeps { + DepositFlow::queue(account(i + 1), 400_000_000) + .sweep(signature(i)) + .succeed() + .credit() + .single_pending_mint() + .mint(i as u64); + } + + let page1 = dashboard(); + assert_eq!( + page1.minted_sweeps_table.current_page.len(), + DEFAULT_PAGE_SIZE + ); + assert!(page1.minted_sweeps_table.has_more_than_one_page()); + assert_eq!(page1.minted_sweeps_table.pagination.pages.len(), 2); + assert_eq!(page1.minted_sweeps_table.pagination.current_page_index, 1); + + let page2 = dashboard_with_pagination(DashboardPaginationParameters { + minted_sweeps_start: DEFAULT_PAGE_SIZE, + ..Default::default() + }); + assert_eq!(page2.minted_sweeps_table.current_page.len(), 1); + assert_eq!(page2.minted_sweeps_table.pagination.current_page_index, 2); +} + #[test] fn should_not_display_pagination_for_small_tables() { init_state(); diff --git a/minter/src/deposit/sweep/mint/mod.rs b/minter/src/deposit/sweep/mint/mod.rs new file mode 100644 index 000000000..8d03698e8 --- /dev/null +++ b/minter/src/deposit/sweep/mint/mod.rs @@ -0,0 +1,198 @@ +use crate::{ + constants::{LEDGER_DEDUPLICATION_WINDOW, MAX_PENDING_MINTS_PER_ROUND}, + guard::TimerGuard, + ledger::client::to_ledger_mint_index, + numeric::LedgerMintIndex, + runtime::CanisterRuntime, + state::{ + PendingMint, State, TaskType, audit::process_event, event::EventType, mutate_state, + read_state, + }, +}; +use canlog::log; +use cksol_types::{DepositSolId, Memo, MintMemo}; +use cksol_types_internal::log::Priority; +use icrc_ledger_types::icrc1::transfer::{NumTokens, TransferArg, TransferError}; +use scopeguard::ScopeGuard; +use std::time::Duration; + +#[cfg(test)] +mod tests; + +pub async fn process_pending_mints(runtime: R) { + let _guard = match TimerGuard::new(TaskType::Mint) { + Ok(guard) => guard, + Err(_) => return, + }; + + let MintRound { + pending_mints, + more_pending_mints_than_one_round, + } = read_state(MintRound::next); + + if pending_mints.is_empty() { + return; + } + + let selected_deposit_ids: Vec = pending_mints + .iter() + .map(|(deposit_id, _)| *deposit_id) + .collect(); + + for (deposit_id, pending) in pending_mints { + process_pending_mint(&runtime, deposit_id, pending).await; + } + + let round_made_progress = read_state(|state| { + selected_deposit_ids + .iter() + .any(|deposit_id| !state.deposits().pending_mints().contains_key(deposit_id)) + }); + if round_made_progress && more_pending_mints_than_one_round { + runtime.set_timer(Duration::ZERO, process_pending_mints); + } +} + +struct MintRound { + pending_mints: Vec<(DepositSolId, PendingMint)>, + more_pending_mints_than_one_round: bool, +} + +impl MintRound { + fn next(state: &State) -> Self { + Self { + pending_mints: state + .deposits() + .pending_mints() + .iter() + .map(|(deposit_id, pending)| (*deposit_id, *pending)) + .take(MAX_PENDING_MINTS_PER_ROUND) + .collect(), + more_pending_mints_than_one_round: state.deposits().pending_mints().len() + > MAX_PENDING_MINTS_PER_ROUND, + } + } +} + +async fn process_pending_mint( + runtime: &R, + deposit_id: DepositSolId, + pending: PendingMint, +) { + if is_beyond_deduplication_window(&pending, runtime.time()) { + log!( + Priority::Error, + "Quarantining deposit {deposit_id}: its pending mint is older than the deduplication window of the ledger" + ); + record_quarantined_pending_mint(runtime, deposit_id); + return; + } + + let quarantine_unless_defused = scopeguard::guard(deposit_id, |deposit_id| { + record_quarantined_pending_mint(runtime, deposit_id); + }); + + let client = read_state(|state| state.ledger_client(runtime.inter_canister_call_runtime())); + match client + .transfer(mint_transfer_arg(deposit_id, &pending)) + .await + { + Ok(Ok(mint_block_index)) => { + record_minted_swept_deposit(runtime, deposit_id, mint_block_index); + log!( + Priority::Info, + "Minted {} lamports for deposit {deposit_id} (ledger block index {})", + pending.amount_to_mint, + mint_block_index.get() + ); + ScopeGuard::into_inner(quarantine_unless_defused); + } + Ok(Err(TransferError::Duplicate { duplicate_of })) => { + let mint_block_index = to_ledger_mint_index(duplicate_of); + record_minted_swept_deposit(runtime, deposit_id, mint_block_index); + log!( + Priority::Info, + "Mint of deposit {deposit_id} was deduplicated by the ledger (ledger block index {})", + mint_block_index.get() + ); + ScopeGuard::into_inner(quarantine_unless_defused); + } + Ok(Err(TransferError::TooOld)) => { + log!( + Priority::Error, + "Quarantining deposit {deposit_id}: the ledger rejected its mint as outside the deduplication window" + ); + } + Ok(Err( + rejection @ (TransferError::BadFee { .. } + | TransferError::BadBurn { .. } + | TransferError::InsufficientFunds { .. }), + )) => { + log!( + Priority::Error, + "Quarantining deposit {deposit_id}: the ledger definitively rejected its mint from the minting account: {rejection:?}" + ); + } + Ok(Err( + transient @ (TransferError::TemporarilyUnavailable + | TransferError::GenericError { .. } + | TransferError::CreatedInFuture { .. }), + )) => { + ScopeGuard::into_inner(quarantine_unless_defused); + log!( + Priority::Info, + "Failed to mint deposit {deposit_id}, retrying on the next round: {transient:?}" + ); + } + Err(ic_error) => { + ScopeGuard::into_inner(quarantine_unless_defused); + log!( + Priority::Info, + "Failed to mint deposit {deposit_id}, retrying on the next round: {ic_error}" + ); + } + } +} + +fn is_beyond_deduplication_window(pending: &PendingMint, now: u64) -> bool { + let age = Duration::from_nanos(now.saturating_sub(pending.created_at_time)); + age > LEDGER_DEDUPLICATION_WINDOW +} + +fn mint_transfer_arg(deposit_id: DepositSolId, pending: &PendingMint) -> TransferArg { + TransferArg { + from_subaccount: None, + to: pending.account(), + fee: None, + created_at_time: Some(pending.created_at_time), + memo: Some(Memo::from(MintMemo::sweep(pending.sweep_signature(), deposit_id)).into()), + amount: NumTokens::from(pending.amount_to_mint), + } +} + +fn record_minted_swept_deposit( + runtime: &R, + deposit_id: DepositSolId, + mint_block_index: LedgerMintIndex, +) { + mutate_state(|state| { + process_event( + state, + EventType::MintedSweptDeposit { + deposit_id, + mint_block_index, + }, + runtime, + ) + }); +} + +fn record_quarantined_pending_mint(runtime: &R, deposit_id: DepositSolId) { + mutate_state(|state| { + process_event( + state, + EventType::QuarantinedPendingMint { deposit_id }, + runtime, + ) + }); +} diff --git a/minter/src/deposit/sweep/mint/tests.rs b/minter/src/deposit/sweep/mint/tests.rs new file mode 100644 index 000000000..5c6533da1 --- /dev/null +++ b/minter/src/deposit/sweep/mint/tests.rs @@ -0,0 +1,385 @@ +use super::process_pending_mints; +use crate::{ + constants::{GET_BALANCE_CYCLES, LEDGER_DEDUPLICATION_WINDOW, MAX_PENDING_MINTS_PER_ROUND}, + deposit::sweep::{deposit_sol, timer::MAX_DEPOSITS_PER_SWEEP}, + state::{TaskType, event::EventType, mutate_state, read_state, reset_state}, + storage::reset_events, + test_fixtures::{ + BLOCK_INDEX, EventsAssert, MINIMUM_DEPOSIT_AMOUNT, PROCESS_DEPOSIT_REQUIRED_CYCLES, + account, + flow::deposit::{DepositFlow, PendingMintFlow, SweepFlow}, + init_schnorr_master_key, init_state, + runtime::TestCanisterRuntime, + signature, + }, +}; +use candid::Nat; +use cksol_types::{DepositSolId, DepositSolStatus, Memo, MintMemo}; +use futures::FutureExt; +use ic_canister_runtime::IcError; +use icrc_ledger_types::icrc1::transfer::{BlockIndex, NumTokens, TransferArg, TransferError}; +use sol_rpc_types::{Lamport, MultiRpcResult}; +use std::panic::AssertUnwindSafe; + +type MintResult = Result; + +const SWEEP_SIGNATURE_INDEX: usize = 0xAA; +const SWEEPABLE_AMOUNT: Lamport = 25_000_000; +const CREDITED_AT_TIME: u64 = 1_234; + +#[tokio::test] +async fn should_return_early_if_no_pending_mints() { + init_state(); + let runtime = TestCanisterRuntime::new(); + + process_pending_mints(runtime.clone()).await; + + EventsAssert::assert_no_events_recorded(); + assert_eq!(runtime.set_timer_call_count(), 0); +} + +#[tokio::test] +async fn should_return_early_if_task_already_active() { + setup(); + credited_pending_mint(); + let events_before = EventsAssert::from_recorded(); + mutate_state(|s| { + s.active_tasks_mut().insert(TaskType::Mint); + }); + let runtime = TestCanisterRuntime::new(); + + process_pending_mints(runtime.clone()).await; + + assert_eq!(events_before, EventsAssert::from_recorded()); + assert!(runtime.sent_update_calls().is_empty()); +} + +#[tokio::test] +async fn should_mint_pending_deposit_and_release_the_account() { + setup(); + let pending = credited_pending_mint(); + let runtime = mint_runtime([Ok(BLOCK_INDEX.into())]); + + process_pending_mints(runtime.clone()).await; + + assert_eq!( + deposit_status(pending.deposit_id), + DepositSolStatus::Minted { + block_index: BLOCK_INDEX, + minted_amount: pending.amount_to_mint, + } + ); + EventsAssert::from_recorded().expect_contains_event_eq(EventType::MintedSweptDeposit { + deposit_id: pending.deposit_id, + mint_block_index: BLOCK_INDEX.into(), + }); + assert_eq!( + transfer_args_sent_by(&runtime), + vec![expected_transfer_arg(&pending)] + ); + assert_eq!(runtime.set_timer_call_count(), 0); + + let new_deposit_id = deposit_sol(&deposit_sol_runtime(), pending.account).await; + assert_eq!(new_deposit_id, Ok(pending.deposit_id + 1)); +} + +#[tokio::test] +async fn should_record_duplicate_reply_as_minted() { + setup(); + let pending = credited_pending_mint(); + let runtime = mint_runtime([Err(TransferError::Duplicate { + duplicate_of: BlockIndex::from(BLOCK_INDEX), + })]); + + process_pending_mints(runtime).await; + + assert_eq!( + deposit_status(pending.deposit_id), + DepositSolStatus::Minted { + block_index: BLOCK_INDEX, + minted_amount: pending.amount_to_mint, + } + ); + EventsAssert::from_recorded().expect_contains_event_eq(EventType::MintedSweptDeposit { + deposit_id: pending.deposit_id, + mint_block_index: BLOCK_INDEX.into(), + }); +} + +#[tokio::test] +async fn should_retry_after_transient_failure_with_exactly_the_same_arguments() { + type AddFailure = fn(TestCanisterRuntime) -> TestCanisterRuntime; + let transient_failures: [(&str, AddFailure); 4] = [ + ("the ledger is temporarily unavailable", |runtime| { + runtime.add_stub_response::(Err(TransferError::TemporarilyUnavailable)) + }), + ("the ledger returns a generic error", |runtime| { + runtime.add_stub_response::(Err(TransferError::GenericError { + error_code: Nat::from(42_u8), + message: "out of luck".to_string(), + })) + }), + ("the minter clock is ahead of the ledger", |runtime| { + runtime.add_stub_response::(Err(TransferError::CreatedInFuture { + ledger_time: 0, + })) + }), + ("the call to the ledger fails", |runtime| { + runtime.add_stub_error(IcError::CallPerformFailed) + }), + ]; + + for (name, add_failure) in transient_failures { + setup(); + let pending = credited_pending_mint(); + let events_before = EventsAssert::from_recorded(); + let failing_runtime = add_failure(TestCanisterRuntime::new().with_increasing_time()); + + process_pending_mints(failing_runtime.clone()).await; + + assert_eq!( + deposit_status(pending.deposit_id), + DepositSolStatus::Finalized { + signature: pending.sweep_signature.into() + }, + "{name}" + ); + assert_eq!(events_before, EventsAssert::from_recorded(), "{name}"); + + let retrying_runtime = mint_runtime([Ok(BLOCK_INDEX.into())]); + + process_pending_mints(retrying_runtime.clone()).await; + + assert_eq!( + deposit_status(pending.deposit_id), + DepositSolStatus::Minted { + block_index: BLOCK_INDEX, + minted_amount: pending.amount_to_mint, + }, + "{name}" + ); + let first_attempt = transfer_args_sent_by(&failing_runtime); + let retry = transfer_args_sent_by(&retrying_runtime); + assert_eq!( + first_attempt, + vec![expected_transfer_arg(&pending)], + "{name}" + ); + assert_eq!(first_attempt, retry, "{name}"); + } +} + +#[tokio::test] +async fn should_quarantine_pending_mint() { + struct QuarantineCase { + name: &'static str, + runtime: TestCanisterRuntime, + expected_ledger_calls: usize, + traps: bool, + } + let stale_now = CREDITED_AT_TIME + LEDGER_DEDUPLICATION_WINDOW.as_nanos() as u64 + 1; + let cases = [ + QuarantineCase { + name: "the pending mint is older than the deduplication window of the ledger", + runtime: TestCanisterRuntime::new().add_times([stale_now; 3]), + expected_ledger_calls: 0, + traps: false, + }, + QuarantineCase { + name: "the ledger rejects the mint as too old", + runtime: mint_runtime([Err(TransferError::TooOld)]), + expected_ledger_calls: 1, + traps: false, + }, + QuarantineCase { + name: "the ledger rejects the fee of the mint", + runtime: mint_runtime([Err(TransferError::BadFee { + expected_fee: Nat::from(10_u8), + })]), + expected_ledger_calls: 1, + traps: false, + }, + QuarantineCase { + name: "the ledger takes the mint for a burn below the minimum", + runtime: mint_runtime([Err(TransferError::BadBurn { + min_burn_amount: Nat::from(10_u8), + })]), + expected_ledger_calls: 1, + traps: false, + }, + QuarantineCase { + name: "the ledger reports insufficient funds on the minting account", + runtime: mint_runtime([Err(TransferError::InsufficientFunds { + balance: Nat::from(0_u8), + })]), + expected_ledger_calls: 1, + traps: false, + }, + QuarantineCase { + name: "the callback traps after the ledger minted because mint index is not u64", + runtime: mint_runtime([Ok(Nat::from(u128::MAX))]), + expected_ledger_calls: 1, + traps: true, + }, + ]; + + for QuarantineCase { + name, + runtime, + expected_ledger_calls, + traps, + } in cases + { + setup(); + let pending = credited_pending_mint(); + + let outcome = AssertUnwindSafe(process_pending_mints(runtime.clone())) + .catch_unwind() + .await; + + assert_eq!(outcome.is_err(), traps, "{name}"); + assert_eq!( + transfer_args_sent_by(&runtime).len(), + expected_ledger_calls, + "{name}" + ); + assert_eq!( + deposit_status(pending.deposit_id), + DepositSolStatus::Quarantined { + signature: pending.sweep_signature.into() + }, + "{name}" + ); + EventsAssert::from_recorded().expect_contains_event_eq(EventType::QuarantinedPendingMint { + deposit_id: pending.deposit_id, + }); + assert_eq!(runtime.set_timer_call_count(), 0, "{name}"); + } +} + +#[tokio::test] +async fn should_reschedule_until_all_pending_mints_are_processed() { + const NUM_DEPOSITS: usize = MAX_PENDING_MINTS_PER_ROUND + 1; + setup(); + credit_sweeps_of_deposits(NUM_DEPOSITS); + let runtime = mint_runtime( + (0..MAX_PENDING_MINTS_PER_ROUND as u64).map(|block_index| Ok(block_index.into())), + ); + + process_pending_mints(runtime.clone()).await; + + read_state(|s| { + assert_eq!(s.deposits().minted().len(), MAX_PENDING_MINTS_PER_ROUND); + assert_eq!(s.deposits().pending_mints().len(), 1); + }); + assert_eq!(runtime.set_timer_call_count(), 1); + + let runtime = mint_runtime([Ok((MAX_PENDING_MINTS_PER_ROUND as u64).into())]); + + process_pending_mints(runtime.clone()).await; + + read_state(|s| { + assert_eq!(s.deposits().minted().len(), NUM_DEPOSITS); + assert!(s.deposits().pending_mints().is_empty()); + }); + assert_eq!(runtime.set_timer_call_count(), 0); +} + +#[tokio::test] +async fn should_not_reschedule_after_a_round_of_transient_failures() { + const NUM_DEPOSITS: usize = MAX_PENDING_MINTS_PER_ROUND + 1; + setup(); + credit_sweeps_of_deposits(NUM_DEPOSITS); + let runtime = mint_runtime( + (0..MAX_PENDING_MINTS_PER_ROUND).map(|_| Err(TransferError::TemporarilyUnavailable)), + ); + + process_pending_mints(runtime.clone()).await; + + read_state(|s| { + assert_eq!(s.deposits().pending_mints().len(), NUM_DEPOSITS); + assert!(s.deposits().minted().is_empty()); + }); + assert_eq!( + transfer_args_sent_by(&runtime).len(), + MAX_PENDING_MINTS_PER_ROUND + ); + assert_eq!(runtime.set_timer_call_count(), 0); +} + +fn setup() { + reset_state(); + reset_events(); + init_state(); + init_schnorr_master_key(); +} + +fn credited_pending_mint() -> PendingMintFlow { + DepositFlow::queue(account(1), SWEEPABLE_AMOUNT) + .sweep(signature(SWEEP_SIGNATURE_INDEX)) + .succeed() + .credit_at(CREDITED_AT_TIME) + .single_pending_mint() +} + +fn credit_sweeps_of_deposits(num_deposits: usize) -> Vec { + let deposits: Vec<_> = (0..num_deposits) + .map(|index| DepositFlow::queue(account(index + 1), SWEEPABLE_AMOUNT)) + .collect(); + deposits + .chunks(MAX_DEPOSITS_PER_SWEEP) + .enumerate() + .flat_map(|(sweep_index, deposits)| { + SweepFlow::of(deposits.iter().copied()) + .submit(signature(SWEEP_SIGNATURE_INDEX + sweep_index)) + .succeed() + .credit() + .mints + }) + .collect() +} + +fn mint_runtime>(results: I) -> TestCanisterRuntime { + let mut runtime = TestCanisterRuntime::new().with_increasing_time(); + for result in results { + runtime = runtime.add_stub_response(result); + } + runtime +} + +fn deposit_sol_runtime() -> TestCanisterRuntime { + TestCanisterRuntime::new() + .with_increasing_time() + .expecting_charges() + .add_msg_cycles_available(PROCESS_DEPOSIT_REQUIRED_CYCLES) + .add_msg_cycles_refunded(GET_BALANCE_CYCLES / 2) + .add_stub_response(MultiRpcResult::::Consistent(Ok( + MINIMUM_DEPOSIT_AMOUNT, + ))) +} + +fn expected_transfer_arg(pending: &PendingMintFlow) -> TransferArg { + TransferArg { + from_subaccount: None, + to: pending.account, + fee: None, + created_at_time: Some(pending.created_at_time), + memo: Some(Memo::from(MintMemo::sweep(pending.sweep_signature, pending.deposit_id)).into()), + amount: NumTokens::from(pending.amount_to_mint), + } +} + +fn transfer_args_sent_by(runtime: &TestCanisterRuntime) -> Vec { + runtime + .sent_update_calls() + .iter() + .map(|call| { + assert_eq!(call.method, "icrc1_transfer"); + call.single_arg() + }) + .collect() +} + +fn deposit_status(deposit_id: DepositSolId) -> DepositSolStatus { + read_state(|s| s.deposits().status(deposit_id)) +} diff --git a/minter/src/deposit/sweep/mod.rs b/minter/src/deposit/sweep/mod.rs index 7c8108c5b..474df7a68 100644 --- a/minter/src/deposit/sweep/mod.rs +++ b/minter/src/deposit/sweep/mod.rs @@ -17,10 +17,12 @@ use icrc_ledger_types::icrc1::account::Account; mod tests; mod finalize; +mod mint; mod timer; -pub use crate::constants::SWEEP_DEPOSITS_DELAY; +pub use crate::constants::{PROCESS_PENDING_MINTS_DELAY, SWEEP_DEPOSITS_DELAY}; pub use finalize::credit_finalized_sweeps; +pub use mint::process_pending_mints; pub use timer::sweep_queued_deposits; pub async fn deposit_sol( @@ -53,7 +55,9 @@ pub async fn deposit_sol( DepositSolStatus::Quarantined { .. } => { Err(DepositSolError::Quarantined { deposit_id }) } - DepositSolStatus::Dropped { .. } | DepositSolStatus::NotFound => panic!( + DepositSolStatus::Dropped { .. } + | DepositSolStatus::Minted { .. } + | DepositSolStatus::NotFound => panic!( "BUG: in-flight deposit {deposit_id} of account {account:?} has status {status:?}" ), }; diff --git a/minter/src/ledger/client.rs b/minter/src/ledger/client.rs index ab9a29fea..52b547cf5 100644 --- a/minter/src/ledger/client.rs +++ b/minter/src/ledger/client.rs @@ -34,15 +34,7 @@ impl LedgerClient { 0, ) .await - .map(|result| { - result.map(|index| { - index - .0 - .to_u64() - .expect("ledger block index does not fit into u64") - .into() - }) - }) + .map(|result| result.map(to_ledger_mint_index)) } } @@ -56,3 +48,12 @@ impl LedgerClient { .await } } + +pub(crate) fn to_ledger_mint_index(block_index: BlockIndex) -> LedgerMintIndex { + LedgerMintIndex::from( + block_index + .0 + .to_u64() + .expect("ledger block index does not fit into u64"), + ) +} diff --git a/minter/src/main.rs b/minter/src/main.rs index c66edd0f2..b4f663fe3 100644 --- a/minter/src/main.rs +++ b/minter/src/main.rs @@ -3,7 +3,10 @@ use canlog::{Log, Sort}; use cksol_minter::{ address::fetch_and_record_minter_public_key, consolidate::{DEPOSIT_CONSOLIDATION_DELAY, consolidate_deposits}, - deposit::sweep::{SWEEP_DEPOSITS_DELAY, sweep_queued_deposits}, + deposit::sweep::{ + PROCESS_PENDING_MINTS_DELAY, SWEEP_DEPOSITS_DELAY, process_pending_mints, + sweep_queued_deposits, + }, monitor::{ FINALIZE_TRANSACTIONS_DELAY, RESUBMIT_TRANSACTIONS_DELAY, finalize_transactions, resubmit_transactions, @@ -259,6 +262,16 @@ fn get_events( public_key: public_key.serialize_raw().to_vec(), chain_code: chain_code.to_vec(), }, + EventType::MintedSweptDeposit { + deposit_id, + mint_block_index, + } => event::EventType::MintedSweptDeposit { + deposit_id, + mint_block_index: *mint_block_index.get(), + }, + EventType::QuarantinedPendingMint { deposit_id } => { + event::EventType::QuarantinedPendingMint { deposit_id } + } } } @@ -418,6 +431,9 @@ fn setup_timers() { ic_cdk_timers::set_timer_interval(SWEEP_DEPOSITS_DELAY, async || { sweep_queued_deposits(IcCanisterRuntime::new()).await; }); + ic_cdk_timers::set_timer_interval(PROCESS_PENDING_MINTS_DELAY, async || { + process_pending_mints(IcCanisterRuntime::new()).await; + }); ic_cdk_timers::set_timer_interval(FINALIZE_TRANSACTIONS_DELAY, async || { finalize_transactions(IcCanisterRuntime::new()).await; }); diff --git a/minter/src/metrics.rs b/minter/src/metrics.rs index 0cdaea900..1a5b53c9e 100644 --- a/minter/src/metrics.rs +++ b/minter/src/metrics.rs @@ -1,4 +1,4 @@ -use crate::state::State; +use crate::state::{QuarantineCause, State}; use crate::storage; use ic_metrics_encoder::MetricsEncoder; @@ -55,15 +55,35 @@ pub fn encode_metrics(w: &mut MetricsEncoder>, s: &State) -> std::io::Re s.deposits().pending_mints().len().metric_value(), "Number of swept deposits whose ckSOL mint is pending.", )?; + w.encode_gauge( + "minted_swept_deposits", + s.deposits().minted().len().metric_value(), + "Number of swept deposits whose ckSOL mint landed on the ledger.", + )?; w.encode_gauge( "dropped_deposits", s.deposits().dropped().len().metric_value(), "Number of deposits whose sweep transaction failed or expired.", )?; - w.encode_gauge( + let mut sweep_unreadable: usize = 0; + let mut mint_unresolved: usize = 0; + for quarantined in s.deposits().quarantined().values() { + match quarantined.cause { + QuarantineCause::SweepUnreadable => sweep_unreadable += 1, + QuarantineCause::MintUnresolved { .. } => mint_unresolved += 1, + } + } + w.gauge_vec( "quarantined_swept_deposits", - s.deposits().quarantined().len().metric_value(), - "Number of quarantined deposits whose finalized sweep did not match its plan and could not be credited.", + "Number of quarantined deposits by cause: a sweep whose finalized outcome did not match its plan credited nothing, while an unresolved mint was credited but could no longer be retried and may have landed on the ledger.", + )? + .value( + &[("cause", "sweep_unreadable")], + sweep_unreadable.metric_value(), + )? + .value( + &[("cause", "mint_unresolved")], + mint_unresolved.metric_value(), )?; w.encode_gauge( "deposits_to_consolidate", diff --git a/minter/src/state/audit.rs b/minter/src/state/audit.rs index 7d5918048..c85e24495 100644 --- a/minter/src/state/audit.rs +++ b/minter/src/state/audit.rs @@ -85,7 +85,7 @@ fn apply_state_transition(state: &mut State, payload: &EventType, timestamp: u64 amount_received, mints, } => { - state.process_credited_sweep(signature, *amount_received, mints); + state.process_credited_sweep(signature, *amount_received, mints, timestamp); } EventType::QuarantinedSweep { signature } => { state.process_quarantined_sweep(signature); @@ -99,6 +99,15 @@ fn apply_state_transition(state: &mut State, payload: &EventType, timestamp: u64 chain_code: *chain_code, }); } + EventType::MintedSweptDeposit { + deposit_id, + mint_block_index, + } => { + state.process_minted_swept_deposit(*deposit_id, mint_block_index); + } + EventType::QuarantinedPendingMint { deposit_id } => { + state.process_quarantined_pending_mint(*deposit_id); + } } } diff --git a/minter/src/state/deposits/mod.rs b/minter/src/state/deposits/mod.rs index 5a3c9a3ba..cb1637cd7 100644 --- a/minter/src/state/deposits/mod.rs +++ b/minter/src/state/deposits/mod.rs @@ -1,5 +1,6 @@ use crate::{ constants::RENT_EXEMPTION_THRESHOLD, + numeric::LedgerMintIndex, state::event::{CreditedDeposit, VersionedMessage}, }; use cksol_types::{DepositSolId, DepositSolStatus}; @@ -23,9 +24,9 @@ mod tests; /// A deposit is in exactly one stage at a time and only moves forward: /// /// ```text -/// queued --sendTransaction--> swept --getSignatureStatuses--> finalized --getTransaction--> pending_mints -/// | | -/// +--failed or expired--> dropped +--metadata mismatch--> quarantined +/// queued --sendTransaction--> swept --getSignatureStatuses--> finalized --getTransaction--> pending_mints --icrc1_transfer--> minted +/// | | | +/// +--failed or expired--> dropped +--metadata mismatch--> quarantined <--beyond the deduplication window--+ /// ``` /// /// * `queued`: the deposit address holds a sweepable amount and waits for a sweep transaction. @@ -42,8 +43,12 @@ mod tests; /// * `pending_mints`: the metadata returned by `getTransaction` was read and the transaction /// fee of the sweep was shared between its deposits. Each deposit now carries the amount to /// mint and only the mint on the ledger remains, so the deposits are tracked individually again. -/// * `quarantined`: the metadata contradicts the minter's model of the sweep, so nothing is -/// minted and the accounts stay rejected by `deposit_sol` until a minter upgrade. +/// * `minted`: the ckSOL mint landed on the ledger and the account is released. +/// * `quarantined`: the metadata contradicts the minter's model of the sweep, so nothing was +/// credited and nothing is minted, or a pending mint could no longer be retried within the +/// deduplication window of the ledger, in which case the sweep was credited and the mint is +/// still owed unless it already landed. The cause records which of the two it is, and the +/// accounts stay rejected by `deposit_sol` until a minter upgrade. /// /// Every account has at most one deposit in flight, so that `deposit_sol` can report the /// deposit it is already tracking instead of queueing the same balance twice. @@ -54,8 +59,9 @@ pub struct Deposits { swept: Sweeps, finalized: Sweeps, pending_mints: BTreeMap, + minted: BTreeMap, dropped: BTreeMap, - quarantined: BTreeMap, + quarantined: BTreeMap, in_flight_ids: BTreeMap, } @@ -80,11 +86,15 @@ impl Deposits { &self.pending_mints } + pub fn minted(&self) -> &BTreeMap { + &self.minted + } + pub fn dropped(&self) -> &BTreeMap { &self.dropped } - pub fn quarantined(&self) -> &BTreeMap { + pub fn quarantined(&self) -> &BTreeMap { &self.quarantined } @@ -110,7 +120,13 @@ impl Deposits { } if let Some(pending) = self.pending_mints.get(&deposit_id) { return DepositSolStatus::Finalized { - signature: pending.deposit.signature.into(), + signature: pending.sweep_signature().into(), + }; + } + if let Some(minted) = self.minted.get(&deposit_id) { + return DepositSolStatus::Minted { + block_index: *minted.mint_block_index.get(), + minted_amount: minted.minted_amount, }; } if let Some(dropped) = self.dropped.get(&deposit_id) { @@ -120,7 +136,7 @@ impl Deposits { } if let Some(quarantined) = self.quarantined.get(&deposit_id) { return DepositSolStatus::Quarantined { - signature: quarantined.signature.into(), + signature: quarantined.deposit.signature.into(), }; } DepositSolStatus::NotFound @@ -201,9 +217,12 @@ impl Deposits { .extend(sweep.deposits().iter().map(|(deposit_id, deposit)| { ( *deposit_id, - SweptDeposit { - deposit: *deposit, - signature: *signature, + QuarantinedDeposit { + deposit: SweptDeposit { + deposit: *deposit, + signature: *signature, + }, + cause: QuarantineCause::SweepUnreadable, }, ) })); @@ -227,7 +246,12 @@ impl Deposits { /// Moves every deposit of the given finalized sweep to the pending mints, each with /// the amount its mint carries. - pub(super) fn credit_sweep(&mut self, signature: &Signature, mints: &[CreditedDeposit]) { + pub(super) fn credit_sweep( + &mut self, + signature: &Signature, + mints: &[CreditedDeposit], + timestamp: u64, + ) { let sweep = self.finalized.remove(signature).unwrap_or_else(|| { panic!("Attempted to credit sweep {signature} that is not finalized") }); @@ -258,6 +282,7 @@ impl Deposits { signature: *signature, }, amount_to_mint: mint.amount_to_mint, + created_at_time: timestamp, }; assert!( self.pending_mints @@ -268,6 +293,37 @@ impl Deposits { ); } } + + pub(super) fn mint(&mut self, deposit_id: DepositSolId, mint_block_index: LedgerMintIndex) { + let pending = self.pending_mints.remove(&deposit_id).unwrap_or_else(|| { + panic!("Attempted to mint deposit {deposit_id} that has no pending mint") + }); + self.release_in_flight(deposit_id, &pending.account()); + self.minted.insert( + deposit_id, + MintedSweep { + deposit: pending.deposit, + minted_amount: pending.amount_to_mint, + mint_block_index, + }, + ); + } + + pub(super) fn quarantine_pending_mint(&mut self, deposit_id: DepositSolId) { + let pending = self.pending_mints.remove(&deposit_id).unwrap_or_else(|| { + panic!("Attempted to quarantine deposit {deposit_id} that has no pending mint") + }); + self.quarantined.insert( + deposit_id, + QuarantinedDeposit { + deposit: pending.deposit, + cause: QuarantineCause::MintUnresolved { + amount_to_mint: pending.amount_to_mint, + created_at_time: pending.created_at_time, + }, + }, + ); + } } /// A deposit address queued for a sweep to the minter's main account. @@ -325,4 +381,69 @@ pub struct PendingMint { pub deposit: SweptDeposit, /// The sweepable amount minus the deposit's share of the transaction fee of the sweep. pub amount_to_mint: Lamport, + /// The timestamp of the `CreditedSweep` event that enqueued this pending mint. + /// + /// Every retry of the mint sends this same `created_at_time` to the ckSOL + /// ledger, so the ledger deduplicates retries after an unknown outcome. + pub created_at_time: u64, +} + +impl PendingMint { + pub fn account(&self) -> Account { + self.deposit.deposit.account + } + + pub fn sweep_signature(&self) -> Signature { + self.deposit.signature + } +} + +/// A swept deposit the minter stopped processing, together with what resolving it takes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct QuarantinedDeposit { + pub deposit: SweptDeposit, + pub cause: QuarantineCause, +} + +impl QuarantinedDeposit { + pub fn account(&self) -> Account { + self.deposit.deposit.account + } + + pub fn sweep_signature(&self) -> Signature { + self.deposit.signature + } + + /// The amount the sweep planned to move, before the deposit's share of its fee. + pub fn planned_amount(&self) -> Lamport { + self.deposit.deposit.sweepable_amount() + } +} + +/// Why a swept deposit was quarantined, and therefore what a manual resolution has +/// to establish before crediting it by hand. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum QuarantineCause { + /// The finalized sweep contradicted the plan it was submitted with, so the amount + /// received could not be determined: nothing was credited and nothing was minted. + SweepUnreadable, + /// The sweep was credited and this mint is owed, but it fell out of the + /// deduplication window of the ledger before it was confirmed, so it may already + /// have landed. The ledger has to be searched for a mint carrying this deposit in + /// its memo before minting again. + MintUnresolved { + /// The amount the pending mint was going to mint. + amount_to_mint: Lamport, + /// The `created_at_time` the pending mint was sent with. + created_at_time: u64, + }, +} + +/// A swept deposit whose ckSOL mint landed on the ledger. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MintedSweep { + pub deposit: SweptDeposit, + /// The sweepable amount minus the deposit's share of the transaction fee of the sweep. + pub minted_amount: Lamport, + pub mint_block_index: LedgerMintIndex, } diff --git a/minter/src/state/deposits/tests.rs b/minter/src/state/deposits/tests.rs index a6ce5addf..d8079d5b1 100644 --- a/minter/src/state/deposits/tests.rs +++ b/minter/src/state/deposits/tests.rs @@ -1,6 +1,10 @@ -use super::{DepositBalance, Deposits, PendingMint, QueuedDeposit, SweptDeposit}; +use super::{ + DepositBalance, Deposits, MintedSweep, PendingMint, QuarantineCause, QuarantinedDeposit, + QueuedDeposit, SweptDeposit, +}; use crate::{ constants::{FEE_PER_SIGNATURE, RENT_EXEMPTION_THRESHOLD}, + numeric::LedgerMintIndex, state::event::CreditedDeposit, test_fixtures::{ MINTER_ADDRESS, planned_sweep, queued_deposit, queued_deposit_of, signature, sweep_message, @@ -8,9 +12,11 @@ use crate::{ }; use cksol_types::{DepositSolId, DepositSolStatus}; use sol_rpc_types::Lamport; +use solana_signature::Signature; use std::collections::BTreeMap; const SWEEP_SIGNATURE_INDEX: usize = 0xAA; +const CREDIT_TIMESTAMP: u64 = 1_234; mod deposit_balance { use super::{DepositBalance, Lamport, RENT_EXEMPTION_THRESHOLD}; @@ -335,26 +341,20 @@ mod finalize_swept { mod credit_sweep { use super::{ - BTreeMap, DepositSolStatus, Deposits, MINTER_ADDRESS, PendingMint, SWEEP_SIGNATURE_INDEX, - SweptDeposit, mint, queued_deposit, signature, sweep_message, + BTreeMap, CREDIT_TIMESTAMP, DepositSolStatus, Deposits, MINTER_ADDRESS, PendingMint, + SWEEP_SIGNATURE_INDEX, SweptDeposit, finalized_sweep, mint, queued_deposit, signature, + sweep_message, }; #[test] fn should_move_the_deposits_of_the_sweep_to_pending_mints_with_the_given_amounts() { - let mut deposits = Deposits::default(); - deposits.queue(0, queued_deposit(0)); - deposits.queue(1, queued_deposit(1)); - deposits.queue(2, queued_deposit(2)); - let sweep_signature = signature(SWEEP_SIGNATURE_INDEX); - deposits.sweep( - &[2, 0], - MINTER_ADDRESS, - &sweep_message([(2, queued_deposit(2)), (0, queued_deposit(0))]), + let (mut deposits, sweep_signature) = finalized_sweep(&[2, 0]); + + deposits.credit_sweep( &sweep_signature, + &[mint(2, 295), mint(0, 95)], + CREDIT_TIMESTAMP, ); - deposits.finalize_swept(&sweep_signature); - - deposits.credit_sweep(&sweep_signature, &[mint(2, 295), mint(0, 95)]); assert!(deposits.finalized().is_empty()); let pending_mint = |deposit_id, amount_to_mint| PendingMint { @@ -363,6 +363,7 @@ mod credit_sweep { signature: sweep_signature, }, amount_to_mint, + created_at_time: CREDIT_TIMESTAMP, }; assert_eq!( deposits.pending_mints(), @@ -395,89 +396,166 @@ mod credit_sweep { &sweep_signature, ); - deposits.credit_sweep(&sweep_signature, &[mint(0, 100)]); + deposits.credit_sweep(&sweep_signature, &[mint(0, 100)], CREDIT_TIMESTAMP); } #[test] #[should_panic(expected = "with 1 mints for 2 deposits")] fn should_panic_when_a_deposit_of_the_sweep_has_no_mint() { - let mut deposits = Deposits::default(); - deposits.queue(0, queued_deposit(0)); - deposits.queue(1, queued_deposit(1)); - let sweep_signature = signature(SWEEP_SIGNATURE_INDEX); - deposits.sweep( - &[0, 1], - MINTER_ADDRESS, - &sweep_message([(0, queued_deposit(0)), (1, queued_deposit(1))]), - &sweep_signature, - ); - deposits.finalize_swept(&sweep_signature); + let (mut deposits, sweep_signature) = finalized_sweep(&[0, 1]); - deposits.credit_sweep(&sweep_signature, &[mint(1, 200)]); + deposits.credit_sweep(&sweep_signature, &[mint(1, 200)], CREDIT_TIMESTAMP); } #[test] #[should_panic(expected = "is not part of sweep")] fn should_panic_when_a_mint_is_for_a_deposit_of_another_sweep() { - let mut deposits = Deposits::default(); - deposits.queue(0, queued_deposit(0)); - deposits.queue(1, queued_deposit(1)); - deposits.queue(2, queued_deposit(2)); - let sweep_signature = signature(SWEEP_SIGNATURE_INDEX); - deposits.sweep( - &[0, 2], - MINTER_ADDRESS, - &sweep_message([(0, queued_deposit(0)), (2, queued_deposit(2))]), + let (mut deposits, sweep_signature) = finalized_sweep(&[0, 2]); + + deposits.credit_sweep( &sweep_signature, + &[mint(0, 100), mint(1, 200)], + CREDIT_TIMESTAMP, ); - deposits.finalize_swept(&sweep_signature); - - deposits.credit_sweep(&sweep_signature, &[mint(0, 100), mint(1, 200)]); } #[test] #[should_panic(expected = "Attempted to credit deposit 0 twice")] fn should_panic_when_a_deposit_is_minted_twice() { - let mut deposits = Deposits::default(); - deposits.queue(0, queued_deposit(0)); - deposits.queue(1, queued_deposit(1)); - let sweep_signature = signature(SWEEP_SIGNATURE_INDEX); - deposits.sweep( - &[0, 1], - MINTER_ADDRESS, - &sweep_message([(0, queued_deposit(0)), (1, queued_deposit(1))]), + let (mut deposits, sweep_signature) = finalized_sweep(&[0, 1]); + + deposits.credit_sweep( &sweep_signature, + &[mint(0, 100), mint(0, 100)], + CREDIT_TIMESTAMP, ); - deposits.finalize_swept(&sweep_signature); - - deposits.credit_sweep(&sweep_signature, &[mint(0, 100), mint(0, 100)]); } #[test] #[should_panic(expected = "beyond its sweepable amount")] fn should_panic_when_a_mint_exceeds_the_sweepable_amount() { - let mut deposits = Deposits::default(); - deposits.queue(0, queued_deposit(0)); - let sweep_signature = signature(SWEEP_SIGNATURE_INDEX); - deposits.sweep( - &[0], - MINTER_ADDRESS, - &sweep_message([(0, queued_deposit(0))]), - &sweep_signature, - ); - deposits.finalize_swept(&sweep_signature); + let (mut deposits, sweep_signature) = finalized_sweep(&[0]); deposits.credit_sweep( &sweep_signature, &[mint(0, queued_deposit(0).sweepable_amount() + 1)], + CREDIT_TIMESTAMP, + ); + } +} + +mod mint { + use super::{ + BTreeMap, DepositSolStatus, Deposits, LedgerMintIndex, MintedSweep, SweptDeposit, + credited_sweep, mint, queued_deposit, + }; + + #[test] + fn should_move_the_pending_mint_to_minted_and_release_its_account() { + let (mut deposits, sweep_signature) = credited_sweep(&[mint(0, 100), mint(1, 200)]); + + deposits.mint(0, LedgerMintIndex::from(42)); + + assert_eq!( + deposits.pending_mints().keys().collect::>(), + vec![&1] + ); + assert_eq!( + deposits.minted(), + &BTreeMap::from([( + 0, + MintedSweep { + deposit: SweptDeposit { + deposit: queued_deposit(0), + signature: sweep_signature, + }, + minted_amount: 100, + mint_block_index: LedgerMintIndex::from(42), + } + )]) + ); + assert_eq!(deposits.in_flight_id(&queued_deposit(0).account), None); + assert_eq!(deposits.in_flight_id(&queued_deposit(1).account), Some(1)); + assert_eq!( + deposits.status(0), + DepositSolStatus::Minted { + block_index: 42, + minted_amount: 100, + } + ); + assert_eq!( + deposits.status(1), + DepositSolStatus::Finalized { + signature: sweep_signature.into() + } + ); + } + + #[test] + #[should_panic(expected = "Attempted to mint deposit 0 that has no pending mint")] + fn should_panic_without_pending_mint() { + Deposits::default().mint(0, LedgerMintIndex::from(42)); + } +} + +mod quarantine_pending_mint { + use super::{ + BTreeMap, CREDIT_TIMESTAMP, DepositSolStatus, Deposits, QuarantineCause, + QuarantinedDeposit, SweptDeposit, credited_sweep, mint, queued_deposit, + }; + + #[test] + fn should_move_the_pending_mint_to_quarantined_without_releasing_its_account() { + let (mut deposits, sweep_signature) = credited_sweep(&[mint(0, 100), mint(1, 200)]); + + deposits.quarantine_pending_mint(0); + + assert_eq!( + deposits.pending_mints().keys().collect::>(), + vec![&1] + ); + assert!(deposits.minted().is_empty()); + assert_eq!( + deposits.quarantined(), + &BTreeMap::from([( + 0, + QuarantinedDeposit { + deposit: SweptDeposit { + deposit: queued_deposit(0), + signature: sweep_signature, + }, + cause: QuarantineCause::MintUnresolved { + amount_to_mint: 100, + created_at_time: CREDIT_TIMESTAMP, + }, + }, + )]) + ); + for deposit_id in 0..2 { + assert_eq!( + deposits.in_flight_id(&queued_deposit(deposit_id).account), + Some(deposit_id) + ); + } + assert_eq!( + deposits.status(0), + DepositSolStatus::Quarantined { + signature: sweep_signature.into() + } ); } + + #[test] + #[should_panic(expected = "Attempted to quarantine deposit 0 that has no pending mint")] + fn should_panic_without_pending_mint() { + Deposits::default().quarantine_pending_mint(0); + } } mod quarantine_sweep { use super::{ - BTreeMap, DepositSolStatus, Deposits, MINTER_ADDRESS, SWEEP_SIGNATURE_INDEX, SweptDeposit, - queue_deposits, signature, sweep_message, + BTreeMap, DepositSolStatus, Deposits, MINTER_ADDRESS, QuarantineCause, QuarantinedDeposit, + SWEEP_SIGNATURE_INDEX, SweptDeposit, queue_deposits, signature, sweep_message, }; #[test] @@ -497,9 +575,12 @@ mod quarantine_sweep { assert!(deposits.finalized().is_empty()); assert!(deposits.pending_mints().is_empty()); - let quarantined = |deposit| SweptDeposit { - deposit, - signature: sweep_signature, + let quarantined = |deposit| QuarantinedDeposit { + deposit: SweptDeposit { + deposit, + signature: sweep_signature, + }, + cause: QuarantineCause::SweepUnreadable, }; assert_eq!( deposits.quarantined(), @@ -550,3 +631,41 @@ fn mint(deposit_id: DepositSolId, amount_to_mint: Lamport) -> CreditedDeposit { amount_to_mint, } } + +/// Sweeps the given deposits in that order under one signature and finalizes the +/// sweep. +/// +/// Deposit ids are queued in sequence, so every id up to the highest swept one is +/// queued, which leaves the ids in between queued but not swept. +fn finalized_sweep(deposit_ids: &[DepositSolId]) -> (Deposits, Signature) { + let highest_swept = *deposit_ids + .iter() + .max() + .expect("BUG: a sweep needs at least one deposit"); + let mut deposits = Deposits::default(); + for deposit_id in 0..=highest_swept { + deposits.queue(deposit_id, queued_deposit(deposit_id)); + } + let swept: Vec<_> = deposit_ids + .iter() + .map(|&deposit_id| (deposit_id, queued_deposit(deposit_id))) + .collect(); + let sweep_signature = signature(SWEEP_SIGNATURE_INDEX); + deposits.sweep( + deposit_ids, + MINTER_ADDRESS, + &sweep_message(swept), + &sweep_signature, + ); + deposits.finalize_swept(&sweep_signature); + (deposits, sweep_signature) +} + +/// Queues the deposit of every mint, sweeps and finalizes them in that order under +/// one signature and credits the sweep with the given mints at [`CREDIT_TIMESTAMP`]. +fn credited_sweep(mints: &[CreditedDeposit]) -> (Deposits, Signature) { + let deposit_ids: Vec<_> = mints.iter().map(|mint| mint.deposit_id).collect(); + let (mut deposits, sweep_signature) = finalized_sweep(&deposit_ids); + deposits.credit_sweep(&sweep_signature, mints, CREDIT_TIMESTAMP); + (deposits, sweep_signature) +} diff --git a/minter/src/state/event.rs b/minter/src/state/event.rs index 755fff2ba..9a6aeed77 100644 --- a/minter/src/state/event.rs +++ b/minter/src/state/event.rs @@ -193,6 +193,36 @@ pub enum EventType { #[cbor(n(1), with = "minicbor::bytes")] chain_code: [u8; 32], }, + /// The minter minted ckSOL on the ledger for a swept deposit whose sweep + /// was credited. + #[n(15)] + MintedSweptDeposit { + /// The identifier of the minted deposit. + #[n(0)] + deposit_id: DepositSolId, + /// The mint transaction index on the ckSOL ledger. + #[cbor(n(1), with = "cbor::id")] + mint_block_index: LedgerMintIndex, + }, + /// The pending mint of a swept deposit cannot be retried: either it became + /// older than the 24-hour deduplication window of the ckSOL ledger, or the + /// ledger definitively rejected it. Retrying the transfer with the same + /// arguments fails forever, and fresh arguments could double mint. + /// + /// The deposit is quarantined to avoid any double minting and will not be + /// further processed without manual intervention. + /// + /// If the minter was down past the deduplication window, the underlying + /// transfer may nevertheless have landed on the ledger. Manual resolution + /// must therefore first search the ledger for a mint whose memo carries the + /// sweep signature before crediting by hand, otherwise a double mint + /// results. + #[n(16)] + QuarantinedPendingMint { + /// The identifier of the deposit whose pending mint was quarantined. + #[n(0)] + deposit_id: DepositSolId, + }, } /// The mint enqueued for one deposit of a `CreditedSweep` event. diff --git a/minter/src/state/mod.rs b/minter/src/state/mod.rs index 6f61b1341..a4ce202c7 100644 --- a/minter/src/state/mod.rs +++ b/minter/src/state/mod.rs @@ -35,8 +35,9 @@ mod deposits; pub mod event; pub use deposits::{ - DepositBalance, Deposits, PendingMint, QueuedDeposit, SettledSweep, Sweep, SweepMismatch, - SweepRecoveryError, SweepSettlementError, Sweeps, SweptDeposit, Transfer, UnreadableOutcome, + DepositBalance, Deposits, MintedSweep, PendingMint, QuarantineCause, QuarantinedDeposit, + QueuedDeposit, SettledSweep, Sweep, SweepMismatch, SweepRecoveryError, SweepSettlementError, + Sweeps, SweptDeposit, Transfer, UnreadableOutcome, }; thread_local! { @@ -504,16 +505,29 @@ impl State { signature: &Signature, amount_received: Lamport, mints: &[CreditedDeposit], + timestamp: u64, ) { let amount_to_mint: Lamport = mints.iter().map(|mint| mint.amount_to_mint).sum(); assert!( amount_to_mint <= amount_received, "Attempted to credit sweep {signature} with mints of {amount_to_mint} lamports exceeding the {amount_received} lamports received" ); - self.deposits.credit_sweep(signature, mints); + self.deposits.credit_sweep(signature, mints, timestamp); self.balance += amount_received; } + fn process_minted_swept_deposit( + &mut self, + deposit_id: DepositSolId, + mint_block_index: &LedgerMintIndex, + ) { + self.deposits.mint(deposit_id, *mint_block_index); + } + + fn process_quarantined_pending_mint(&mut self, deposit_id: DepositSolId) { + self.deposits.quarantine_pending_mint(deposit_id); + } + fn process_quarantined_sweep(&mut self, signature: &Signature) { self.deposits.quarantine_sweep(signature); } diff --git a/minter/src/state/tests.rs b/minter/src/state/tests.rs index 67b834d83..2eda06d42 100644 --- a/minter/src/state/tests.rs +++ b/minter/src/state/tests.rs @@ -141,11 +141,12 @@ mod queued_deposits { mod swept_deposits { use super::*; use crate::{ - state::reset_state, - storage::reset_events, + state::{audit::replay_events, reset_state}, + storage::{reset_events, with_event_iter}, test_fixtures::{ DEFAULT_BLOCK_HEIGHT, events::{credit_sweep, quarantine_sweep, queue_deposits, submit_sweep}, + flow::deposit::{CreditedSweepFlow, DepositFlow, SweepFlow}, sweep_message, }, }; @@ -309,6 +310,68 @@ mod swept_deposits { }); } + #[test] + fn should_mint_pending_deposit_without_changing_the_balance() { + init_state(); + let credited = credit_sweep_of_two_deposits(0); + let [to_mint, _] = credited.pending_mints(); + + to_mint.mint(42); + + read_state(|s| assert_eq!(s.balance(), credited.amount_received)); + } + + #[test] + fn should_quarantine_pending_mint_without_changing_the_balance() { + init_state(); + let credited = credit_sweep_of_two_deposits(0); + let [to_quarantine, _] = credited.pending_mints(); + + to_quarantine.quarantine(); + + read_state(|s| assert_eq!(s.balance(), credited.amount_received)); + } + + #[test] + fn should_replay_the_recorded_events_including_created_at_time() { + const CREDITED_AT: u64 = 1_234_000_000; + init_state(); + let credited = credit_sweep_of_two_deposits(CREDITED_AT); + let [to_mint, sibling] = credited.pending_mints(); + to_mint.mint(42); + let recorded_events: Vec = with_event_iter(|events| events.collect()); + + let replayed = replay_events( + [ + Event { + timestamp: 0, + payload: EventType::Init(valid_init_args()), + }, + Event { + timestamp: 0, + payload: crate::test_fixtures::minter_public_key_fetched_event(), + }, + ] + .into_iter() + .chain(recorded_events), + ); + + assert_eq!( + replayed.deposits().pending_mints()[&sibling.deposit_id].created_at_time, + CREDITED_AT + ); + read_state(|live_state| assert_eq!(&replayed, live_state)); + } + + fn credit_sweep_of_two_deposits(credited_at: u64) -> CreditedSweepFlow { + init_schnorr_master_key(); + let deposits = [1, 2].map(|i| DepositFlow::queue(account(i), 1_000_000 * i as u64)); + SweepFlow::of(deposits) + .submit(signature(SWEEP_SIGNATURE_INDEX)) + .succeed() + .credit_at(credited_at) + } + #[test] fn should_quarantine_finalized_deposits_without_crediting_the_balance() { init_state(); diff --git a/minter/src/test_fixtures/flow/deposit.rs b/minter/src/test_fixtures/flow/deposit.rs new file mode 100644 index 000000000..ed7764f4e --- /dev/null +++ b/minter/src/test_fixtures/flow/deposit.rs @@ -0,0 +1,240 @@ +use crate::{ + state::read_state, + test_fixtures::{ + events::{ + credit_sweep_at, mint_swept_deposit, quarantine_pending_mint, queue, submit_sweep, + succeed_transaction, + }, + queued_deposit_of, + }, +}; +use cksol_types::DepositSolId; +use icrc_ledger_types::icrc1::account::Account; +use sol_rpc_types::Lamport; +use solana_address::Address; +use solana_signature::Signature; + +/// Entry point of the lifecycle of an automated deposit, applied event by event to the +/// state. Every stage holds what the test needs to assert on later. +pub struct DepositFlow; + +impl DepositFlow { + /// Queues a deposit of `account`, whose deposit address holds `sweepable_amount` on top + /// of the rent exemption threshold, under the next deposit id of the state. + pub fn queue(account: Account, sweepable_amount: Lamport) -> QueuedDepositFlow { + let deposit_id = read_state(|state| state.deposits().next_id()); + let deposit = queued_deposit_of(account, sweepable_amount); + queue(deposit_id, deposit); + QueuedDepositFlow { + deposit_id, + account, + address: deposit.address, + sweepable_amount, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct QueuedDepositFlow { + pub deposit_id: DepositSolId, + pub account: Account, + pub address: Address, + pub sweepable_amount: Lamport, +} + +impl QueuedDepositFlow { + /// Submits a sweep of this single deposit under `signature`. + pub fn sweep(self, signature: Signature) -> SubmittedSweepFlow { + SweepFlow::of([self]).submit(signature) + } +} + +/// The deposits of one sweep transaction, in the order of its transfers. +pub struct SweepFlow { + deposits: Vec, +} + +impl SweepFlow { + pub fn of(deposits: impl IntoIterator) -> Self { + Self { + deposits: deposits.into_iter().collect(), + } + } + + /// Submits the sweep of the deposits to the minter's main address under `signature`. + pub fn submit(self, signature: Signature) -> SubmittedSweepFlow { + let deposit_ids = self.deposits.iter().map(|deposit| deposit.deposit_id); + submit_sweep(signature, deposit_ids.collect()); + let expected_received = read_state(|state| { + state + .deposits() + .swept() + .get(&signature) + .expect("BUG: the sweep was just submitted") + .expected_received() + }); + SubmittedSweepFlow { + signature, + deposits: self.deposits, + expected_received, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SubmittedSweepFlow { + pub signature: Signature, + pub deposits: Vec, + pub expected_received: Lamport, +} + +impl SubmittedSweepFlow { + /// Reports the sweep transaction as finalized without error. + pub fn succeed(self) -> FinalizedSweepFlow { + succeed_transaction(self.signature); + FinalizedSweepFlow { + signature: self.signature, + deposits: self.deposits, + expected_received: self.expected_received, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FinalizedSweepFlow { + pub signature: Signature, + pub deposits: Vec, + pub expected_received: Lamport, +} + +impl FinalizedSweepFlow { + /// Credits the sweep with the amount its plan expected, at time zero. + pub fn credit(self) -> CreditedSweepFlow { + self.credit_at(0) + } + + /// Credits the sweep with the amount its plan expected, at `timestamp`, which becomes + /// the `created_at_time` of every pending mint. + pub fn credit_at(self, timestamp: u64) -> CreditedSweepFlow { + credit_sweep_at(self.signature, self.expected_received, timestamp); + let mints = read_state(|state| { + self.deposits + .iter() + .map(|deposit| { + let pending = state + .deposits() + .pending_mints() + .get(&deposit.deposit_id) + .expect("BUG: the credited deposit has no pending mint"); + PendingMintFlow { + deposit_id: deposit.deposit_id, + account: pending.account(), + amount_to_mint: pending.amount_to_mint, + sweep_signature: pending.sweep_signature(), + created_at_time: pending.created_at_time, + } + }) + .collect() + }); + CreditedSweepFlow { + signature: self.signature, + amount_received: self.expected_received, + mints, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CreditedSweepFlow { + pub signature: Signature, + pub amount_received: Lamport, + /// The pending mints of the sweep, in the order of its deposits. + pub mints: Vec, +} + +impl CreditedSweepFlow { + /// The pending mint of a sweep of exactly one deposit. + pub fn single_pending_mint(&self) -> PendingMintFlow { + match self.mints.as_slice() { + [mint] => *mint, + mints => panic!( + "BUG: expected the sweep {} to have exactly one pending mint, got {}", + self.signature, + mints.len() + ), + } + } + + /// The pending mints of a sweep of exactly `N` deposits, in the order of its deposits. + pub fn pending_mints(&self) -> [PendingMintFlow; N] { + self.mints + .clone() + .try_into() + .unwrap_or_else(|mints: Vec<_>| { + panic!( + "BUG: expected the sweep {} to have exactly {N} pending mints, got {}", + self.signature, + mints.len() + ) + }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PendingMintFlow { + pub deposit_id: DepositSolId, + pub account: Account, + pub amount_to_mint: Lamport, + pub sweep_signature: Signature, + pub created_at_time: u64, +} + +impl PendingMintFlow { + /// Records the mint of the deposit on the ledger under `block_index`, which releases + /// its account for a new deposit. + pub fn mint(self, block_index: u64) -> MintedDepositFlow { + mint_swept_deposit(self.deposit_id, block_index); + let minted = read_state(|state| { + *state + .deposits() + .minted() + .get(&self.deposit_id) + .expect("BUG: the deposit was just minted") + }); + MintedDepositFlow { + deposit_id: self.deposit_id, + account: self.account, + minted_amount: minted.minted_amount, + sweep_signature: self.sweep_signature, + mint_block_index: *minted.mint_block_index.get(), + } + } +} + +impl PendingMintFlow { + /// Quarantines the pending mint, which keeps its account in flight. + pub fn quarantine(self) -> QuarantinedDepositFlow { + quarantine_pending_mint(self.deposit_id); + QuarantinedDepositFlow { + deposit_id: self.deposit_id, + account: self.account, + sweep_signature: self.sweep_signature, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct QuarantinedDepositFlow { + pub deposit_id: DepositSolId, + pub account: Account, + pub sweep_signature: Signature, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct MintedDepositFlow { + pub deposit_id: DepositSolId, + pub account: Account, + pub minted_amount: Lamport, + pub sweep_signature: Signature, + pub mint_block_index: u64, +} diff --git a/minter/src/test_fixtures/flow/mod.rs b/minter/src/test_fixtures/flow/mod.rs new file mode 100644 index 000000000..4ed90426c --- /dev/null +++ b/minter/src/test_fixtures/flow/mod.rs @@ -0,0 +1 @@ +pub mod deposit; diff --git a/minter/src/test_fixtures/mod.rs b/minter/src/test_fixtures/mod.rs index 0d6fed865..f50a9e769 100644 --- a/minter/src/test_fixtures/mod.rs +++ b/minter/src/test_fixtures/mod.rs @@ -26,6 +26,7 @@ use solana_transaction_status_client_types::{ }; use std::{collections::VecDeque, str::FromStr}; +pub mod flow; pub mod runtime; pub mod signer; mod stubs; @@ -691,6 +692,10 @@ pub mod events { } pub fn credit_sweep(signature: Signature, amount_received: Lamport) { + credit_sweep_at(signature, amount_received, 0); + } + + pub fn credit_sweep_at(signature: Signature, amount_received: Lamport, timestamp: u64) { let mints = read_state(|state| { state .deposits() @@ -707,6 +712,29 @@ pub mod events { amount_received, mints, }, + &TestCanisterRuntime::new().add_times([timestamp, timestamp]), + ) + }); + } + + pub fn mint_swept_deposit(deposit_id: DepositSolId, mint_block_index: u64) { + mutate_state(|state| { + process_event( + state, + EventType::MintedSweptDeposit { + deposit_id, + mint_block_index: LedgerMintIndex::from(mint_block_index), + }, + &runtime(), + ) + }); + } + + pub fn quarantine_pending_mint(deposit_id: DepositSolId) { + mutate_state(|state| { + process_event( + state, + EventType::QuarantinedPendingMint { deposit_id }, &runtime(), ) }); @@ -1162,6 +1190,13 @@ pub mod arb { chain_code, } }), + (any::(), arb_ledger_mint_index()).prop_map(|(deposit_id, mint_block_index)| { + EventType::MintedSweptDeposit { + deposit_id, + mint_block_index, + } + }), + any::().prop_map(|deposit_id| EventType::QuarantinedPendingMint { deposit_id }), ] } diff --git a/minter/src/test_fixtures/runtime.rs b/minter/src/test_fixtures/runtime.rs index fbed5c6b7..8987418bd 100644 --- a/minter/src/test_fixtures/runtime.rs +++ b/minter/src/test_fixtures/runtime.rs @@ -6,11 +6,16 @@ use super::{ use crate::{ constants::GET_RECENT_BLOCK_MAX_TRIES, runtime::CanisterRuntime, signer::SchnorrSigner, }; -use candid::{CandidType, Principal}; +use async_trait::async_trait; +use candid::{ + CandidType, Principal, + utils::{ArgumentEncoder, decode_args, encode_args}, +}; use ic_canister_runtime::{IcError, Runtime, StubRuntime}; use ic_cdk::call::{CallPerformFailed, Error as CallError}; use ic_cdk_management_canister::{SchnorrPublicKeyArgs, SchnorrPublicKeyResult}; use icrc_ledger_types::icrc1::account::Account; +use serde::de::DeserializeOwned; use sol_rpc_types::{MultiRpcResult, RpcResult, Signature, Slot}; use std::{ future::Future, @@ -23,7 +28,7 @@ pub const TEST_CANISTER_ID: Principal = Principal::from_slice(&[0xCA; 10]); #[derive(Clone, Default)] pub struct TestCanisterRuntime { - inter_canister_call_runtime: StubRuntime, + inter_canister_call_runtime: RecordingStubRuntime, signer: MockSchnorrSigner, times: Stubs, expects_charges: bool, @@ -51,16 +56,28 @@ impl TestCanisterRuntime { } pub fn add_stub_response(mut self, response: Out) -> Self { - self.inter_canister_call_runtime = - self.inter_canister_call_runtime.add_stub_response(response); + self.inter_canister_call_runtime.stub = self + .inter_canister_call_runtime + .stub + .add_stub_response(response); self } pub fn add_stub_error(mut self, error: IcError) -> Self { - self.inter_canister_call_runtime = self.inter_canister_call_runtime.add_stub_error(error); + self.inter_canister_call_runtime.stub = + self.inter_canister_call_runtime.stub.add_stub_error(error); self } + /// The inter-canister update calls made through this runtime, in call order. + pub fn sent_update_calls(&self) -> Vec { + self.inter_canister_call_runtime + .sent_update_calls + .lock() + .unwrap() + .clone() + } + pub fn add_recent_block(mut self, result: RpcResult) -> Self { match result { Ok(slot) => self @@ -197,6 +214,63 @@ impl CanisterRuntime for TestCanisterRuntime { } } +/// Wraps [`StubRuntime`] to record the method and Candid-encoded arguments of every +/// update call, so that a test can assert exactly what was sent. +#[derive(Clone, Default)] +struct RecordingStubRuntime { + stub: StubRuntime, + sent_update_calls: Arc>>, +} + +/// An inter-canister update call recorded by [`TestCanisterRuntime`]. +#[derive(Clone, Debug, PartialEq)] +pub struct SentUpdateCall { + pub method: String, + args: Vec, +} + +impl SentUpdateCall { + /// Decodes the single Candid argument of the recorded call. + pub fn single_arg(&self) -> Arg { + let (arg,) = decode_args(&self.args).expect("Failed to decode the call argument"); + arg + } +} + +#[async_trait] +impl Runtime for RecordingStubRuntime { + async fn update_call( + &self, + id: Principal, + method: &str, + args: In, + cycles: u128, + ) -> Result + where + In: ArgumentEncoder + Send, + Out: CandidType + DeserializeOwned, + { + self.sent_update_calls.lock().unwrap().push(SentUpdateCall { + method: method.to_string(), + args: encode_args(args).expect("Failed to encode the call arguments"), + }); + self.stub.update_call(id, method, (), cycles).await + } + + async fn query_call( + &self, + id: Principal, + method: &str, + args: In, + ) -> Result + where + In: ArgumentEncoder + Send, + Out: CandidType + DeserializeOwned, + { + self.stub.query_call(id, method, args).await + } +} + /// Suspends the caller once, so that concurrent callers all reach the call before any of /// them sees its response, as they do on the IC. async fn suspend_like_an_inter_canister_call() { diff --git a/minter/templates/dashboard.html b/minter/templates/dashboard.html index d4053cca6..643fea569 100644 --- a/minter/templates/dashboard.html +++ b/minter/templates/dashboard.html @@ -112,7 +112,9 @@

Quarantined Swept Deposits

Deposit Id Account Sweep Txn + Cause Planned Amount (SOL) + Amount to Mint (SOL) @@ -121,7 +123,9 @@

Quarantined Swept Deposits

{{ deposit.deposit_id }} {{ deposit.account }} {% call solscan_tx_link(deposit.signature) %} + {{ deposit.cause }} {{ deposit.planned_amount }} + {{ deposit.amount_to_mint }} {% endfor %} {% if quarantined_swept_deposits_table.has_more_than_one_page() %}{{ quarantined_swept_deposits_table.pagination|safe }}{% endif %} @@ -129,6 +133,31 @@

Quarantined Swept Deposits

{% endif %} + {% if !minted_sweeps_table.is_empty() %} +

Minted Swept Deposits

+ + + + + + + + + + + {% for minted in minted_sweeps_table.current_page %} + + + + + + + {% endfor %} + {% if minted_sweeps_table.has_more_than_one_page() %}{{ minted_sweeps_table.pagination|safe }}{% endif %} + +
Deposit IdAccountMinted Amount (SOL)Mint Block Index
{{ minted.deposit_id }}{{ minted.account }}{{ minted.minted_amount }}{{ minted.mint_block_index }}
+ {% endif %} + {% if !deposits_table.is_empty() %}

Deposits