feat(minter): report the age of the oldest in-flight deposit

integration_tests/tests/tests.rs

@@ -1304,6 +1304,33 @@ mod metrics_tests {
             // Only the init and minter public key events should have been recorded
             .assert_contains_metric_matching(r#"total_event_count 2 \d+"#)
             .assert_contains_metric_matching(r#"minter_balance 0 \d+"#)
+            .assert_contains_metric_matching(r#"oldest_in_flight_deposit_age_seconds 0 \d+"#)
+            .assert_contains_metric_matching(r#"oldest_pending_mint_age_seconds 0 \d+"#)
+            .assert_contains_metric_matching(
+                r#"failed_credit_attempts\{reason="not_found"\} 0 \d+"#,
+            )
+            .assert_contains_metric_matching(
+                r#"failed_credit_attempts\{reason="rpc_error"\} 0 \d+"#,
+            )
+            .assert_contains_metric_matching(
+                r#"failed_credit_attempts\{reason="invalid_response"\} 0 \d+"#,
+            )
+            .assert_contains_metric_matching(
+                r#"failed_credit_attempts\{reason="unreadable"\} 0 \d+"#,
+            )
+            .assert_contains_metric_matching(r#"failed_credit_attempts\{reason="mismatch"\} 0 \d+"#)
+            .assert_contains_metric_matching(r#"failed_mint_attempts\{reason="expired"\} 0 \d+"#)
+            .assert_contains_metric_matching(r#"failed_mint_attempts\{reason="rejected"\} 0 \d+"#)
+            .assert_contains_metric_matching(
+                r#"failed_mint_attempts\{reason="ledger_error"\} 0 \d+"#,
+            )
+            .assert_contains_metric_matching(
+                r#"failed_mint_attempts\{reason="created_in_future"\} 0 \d+"#,
+            )
+            .assert_contains_metric_matching(r#"failed_mint_attempts\{reason="call_error"\} 0 \d+"#)
+            .assert_contains_metric_matching(
+                r#"failed_mint_attempts\{reason="unknown_outcome"\} 0 \d+"#,
+            )
             .into()
             .drop()
             .await;

minter/src/deposit/sweep/finalize/mod.rs

@@ -6,6 +6,7 @@ use crate::{
         Sweep, SweepSettlementError, audit::process_event, event::EventType, mutate_state,
         read_state,
     },
+    storage::{FailedCreditReason, record_failed_credit_attempt},
 };
 use canlog::log;
 use cksol_types_internal::log::Priority;
@@ -53,21 +54,26 @@ pub async fn credit_finalized_sweeps<R: CanisterRuntime>(runtime: &R) -> bool {
             Ok(Some(outcome)) => outcome,
             Ok(None) => {
                 log!(
-                    Priority::Info,
+                    Priority::Error,
                     "Finalized sweep {signature} was not returned by getTransaction, retrying later"
                 );
+                record_failed_credit_attempt(FailedCreditReason::NotFound);
+                continue;
+            }
+            Err(e) if e.is_response_untrustworthy() => {
+                log!(
+                    Priority::Error,
+                    "Failed to fetch finalized sweep {signature}: {e}, retrying later"
+                );
+                record_failed_credit_attempt(FailedCreditReason::InvalidResponse);
                 continue;
             }
             Err(e) => {
-                let priority = if e.is_response_untrustworthy() {
-                    Priority::Error
-                } else {
-                    Priority::Info
-                };
                 log!(
-                    priority,
+                    Priority::Info,
                     "Failed to fetch finalized sweep {signature}: {e}, retrying later"
                 );
+                record_failed_credit_attempt(FailedCreditReason::RpcError);
                 continue;
             }
         };
@@ -79,16 +85,18 @@ pub async fn credit_finalized_sweeps<R: CanisterRuntime>(runtime: &R) -> bool {
             },
             Err(SweepSettlementError::Unreadable(e)) => {
                 log!(
-                    Priority::Info,
+                    Priority::Error,
                     "Could not read the outcome of sweep {signature}: {e}, retrying later"
                 );
+                record_failed_credit_attempt(FailedCreditReason::Unreadable);
                 continue;
             }
             Err(SweepSettlementError::Mismatch(e)) => {
                 log!(
                     Priority::Error,
                     "Quarantining the deposits of sweep {signature}: {e}"
                 );
+                record_failed_credit_attempt(FailedCreditReason::Mismatch);
                 EventType::QuarantinedSweep { signature }
             }
         };

minter/src/deposit/sweep/finalize/tests.rs

@@ -3,7 +3,7 @@ use crate::{
     constants::MAX_CONCURRENT_RPC_CALLS,
     deposit::sweep::deposit_status,
     state::{event::EventType, read_state, reset_state},
-    storage::reset_events,
+    storage::{FailedCreditReason, failed_credit_attempt_count, reset_events},
     test_fixtures::{
         EventsAssert, GetTransactionResult, account, devnet_sweep,
         events::{queue, submit_sweep_to, succeed_transaction},
@@ -15,7 +15,9 @@ use crate::{
 use cksol_types::{DepositSolId, DepositSolStatus};
 use sol_rpc_types::Lamport;
 use solana_signature::Signature;
-use solana_transaction_status_client_types::EncodedConfirmedTransactionWithStatusMeta;
+use solana_transaction_status_client_types::{
+    EncodedConfirmedTransactionWithStatusMeta, EncodedTransaction, TransactionBinaryEncoding,
+};
 
 const FRESH_SWEEP_SIGNATURE_INDEX: usize = 0x80;
 
@@ -108,26 +110,49 @@ async fn should_credit_the_amount_received_by_the_main_account() {
             devnet_sweep::DEPOSITS.len()
         );
     });
+    for reason in FailedCreditReason::ALL {
+        assert_eq!(failed_credit_attempt_count(reason), 0, "{reason:?}");
+    }
 }
 
 #[tokio::test]
 async fn should_keep_deposits_finalized_until_the_outcome_can_be_read() {
     type Response = fn() -> GetTransactionResult;
-    let cases: [(&str, Response); 3] = [
-        ("the transaction is not returned", || {
-            GetTransactionResult::Consistent(Ok(None))
-        }),
-        ("fetching the transaction fails", || {
-            GetTransactionResult::Inconsistent(vec![])
-        }),
-        ("the metadata cannot be read", || {
-            let mut outcome = devnet_sweep::derived_outcome();
-            outcome.transaction.meta = None;
-            transaction_response(outcome)
-        }),
+    let cases: [(&str, Response, FailedCreditReason); 4] = [
+        (
+            "the transaction is not returned",
+            || GetTransactionResult::Consistent(Ok(None)),
+            FailedCreditReason::NotFound,
+        ),
+        (
+            "fetching the transaction fails",
+            || GetTransactionResult::Inconsistent(vec![]),
+            FailedCreditReason::RpcError,
+        ),
+        (
+            "the providers agree on a transaction that cannot be decoded",
+            || {
+                let mut outcome = devnet_sweep::derived_outcome();
+                outcome.transaction.transaction = EncodedTransaction::Binary(
+                    "not a transaction".to_string(),
+                    TransactionBinaryEncoding::Base64,
+                );
+                transaction_response(outcome)
+            },
+            FailedCreditReason::InvalidResponse,
+        ),
+        (
+            "the metadata cannot be read",
+            || {
+                let mut outcome = devnet_sweep::derived_outcome();
+                outcome.transaction.meta = None;
+                transaction_response(outcome)
+            },
+            FailedCreditReason::Unreadable,
+        ),
     ];
 
-    for (name, response) in cases {
+    for (name, response, expected_reason) in cases {
         setup();
         let sweep_signature = finalize_devnet_sweep();
         let events_before = EventsAssert::from_recorded();
@@ -154,6 +179,7 @@ async fn should_keep_deposits_finalized_until_the_outcome_can_be_read() {
             },
             "{name}"
         );
+        assert_eq!(failed_credit_attempt_count(expected_reason), 1, "{name}");
     }
 }
 
@@ -183,6 +209,7 @@ async fn should_quarantine_deposits_if_the_outcome_does_not_match_the_plan() {
             signature: sweep_signature.into()
         }
     );
+    assert_eq!(failed_credit_attempt_count(FailedCreditReason::Mismatch), 1);
 }
 
 fn setup() {

minter/src/deposit/sweep/mint/mod.rs

@@ -8,6 +8,7 @@ use crate::{
         PendingMint, State, TaskType, audit::process_event, event::EventType, mutate_state,
         read_state,
     },
+    storage::{FailedMintReason, record_failed_mint_attempt},
 };
 use canlog::log;
 use cksol_types::{DepositSolId, Memo, MintMemo};
@@ -84,11 +85,13 @@ async fn process_pending_mint<R: CanisterRuntime>(
             Priority::Error,
             "Quarantining deposit {deposit_id}: its pending mint is older than the deduplication window of the ledger"
         );
+        record_failed_mint_attempt(FailedMintReason::Expired);
         record_quarantined_pending_mint(runtime, deposit_id);
         return;
     }
 
     let quarantine_unless_defused = scopeguard::guard(deposit_id, |deposit_id| {
+        record_failed_mint_attempt(FailedMintReason::UnknownOutcome);
         record_quarantined_pending_mint(runtime, deposit_id);
     });
 
@@ -118,34 +121,49 @@ async fn process_pending_mint<R: CanisterRuntime>(
             ScopeGuard::into_inner(quarantine_unless_defused);
         }
         Ok(Err(TransferError::TooOld)) => {
+            ScopeGuard::into_inner(quarantine_unless_defused);
+            record_failed_mint_attempt(FailedMintReason::Expired);
             log!(
                 Priority::Error,
                 "Quarantining deposit {deposit_id}: the ledger rejected its mint as outside the deduplication window"
             );
+            record_quarantined_pending_mint(runtime, deposit_id);
         }
         Ok(Err(
             rejection @ (TransferError::BadFee { .. }
             | TransferError::BadBurn { .. }
             | TransferError::InsufficientFunds { .. }),
         )) => {
+            ScopeGuard::into_inner(quarantine_unless_defused);
+            record_failed_mint_attempt(FailedMintReason::Rejected);
             log!(
                 Priority::Error,
                 "Quarantining deposit {deposit_id}: the ledger definitively rejected its mint from the minting account: {rejection:?}"
             );
+            record_quarantined_pending_mint(runtime, deposit_id);
         }
         Ok(Err(
-            transient @ (TransferError::TemporarilyUnavailable
-            | TransferError::GenericError { .. }
-            | TransferError::CreatedInFuture { .. }),
+            transient
+            @ (TransferError::TemporarilyUnavailable | TransferError::GenericError { .. }),
         )) => {
             ScopeGuard::into_inner(quarantine_unless_defused);
+            record_failed_mint_attempt(FailedMintReason::LedgerError);
             log!(
                 Priority::Info,
                 "Failed to mint deposit {deposit_id}, retrying on the next round: {transient:?}"
             );
         }
+        Ok(Err(clock_skew @ TransferError::CreatedInFuture { .. })) => {
+            ScopeGuard::into_inner(quarantine_unless_defused);
+            record_failed_mint_attempt(FailedMintReason::CreatedInFuture);
+            log!(
+                Priority::Info,
+                "Failed to mint deposit {deposit_id}, retrying on the next round: {clock_skew:?}"
+            );
+        }
         Err(ic_error) => {
             ScopeGuard::into_inner(quarantine_unless_defused);
+            record_failed_mint_attempt(FailedMintReason::CallError);
             log!(
                 Priority::Info,
                 "Failed to mint deposit {deposit_id}, retrying on the next round: {ic_error}"