Show retry budget usage and why a retry was refused

src/client/apis/retryPoliciesApi.ts

@@ -4,6 +4,8 @@ import {
     RetryPoliciesSearchModel,
     RetryPolicyModel,
     RetryPolicyRow,
+    RetryGroupUsageRow,
+    RetryPolicyResetUsage,
     TestRetryPolicyRequest,
     TestRetryPolicyResponse
 } from "src/types/retryPolicies";
@@ -12,7 +14,7 @@ import {ApiPagedResponse} from "src/types/common";
 export const RetryPoliciesApi = createApi({
     baseQuery: customFetchBase,
     reducerPath: "RetryPoliciesApi",
-    tagTypes: ["retryPolicies"],
+    tagTypes: ["retryPolicies", "retryPolicyUsage"],
     endpoints: (builder) => ({
         retryPolicies: builder.query<ApiPagedResponse<RetryPolicyRow>, RetryPoliciesSearchModel>({
             providesTags: ['retryPolicies'],
@@ -40,7 +42,8 @@ export const RetryPoliciesApi = createApi({
             })
         }),
         updateRetryPolicy: builder.mutation<{}, { id: number } & RetryPolicyModel>({
-            invalidatesTags: ['retryPolicies'],
+            // Saving drops the counters of any removed group, so the usage panel must refetch.
+            invalidatesTags: ['retryPolicies', 'retryPolicyUsage'],
             query: body => ({
                 url: `RetryPolicies/${body.id}`,
                 method: "POST",
@@ -68,6 +71,22 @@ export const RetryPoliciesApi = createApi({
                 method: "POST",
                 body
             })
+        }),
+        retryPolicyUsage: builder.query<RetryGroupUsageRow[], number>({
+            providesTags: ['retryPolicyUsage'],
+            query: id => ({
+                url: `RetryPolicies/${id}/usage`,
+                method: "POST",
+                body: {}
+            })
+        }),
+        resetRetryPolicyUsage: builder.mutation<{}, { id: number } & RetryPolicyResetUsage>({
+            invalidatesTags: ['retryPolicyUsage'],
+            query: ({id, ...body}) => ({
+                url: `RetryPolicies/${id}/resetusage`,
+                method: "POST",
+                body
+            })
         })
     })
 });
@@ -81,4 +100,6 @@ export const {
     useDeleteRetryPolicyMutation,
     useRetryPoliciesLookupQuery,
     useTestRetryPolicyMutation,
+    useRetryPolicyUsageQuery,
+    useResetRetryPolicyUsageMutation,
 } = RetryPoliciesApi;

src/components/RetryPolicies/AddEditRetryGroupModal.tsx

@@ -169,7 +169,7 @@ const AddEditRetryGroupModal: React.FC<Props> = ({visible, onClose, onAdd, initi
                         <TextEditor type={"number"} value={group.budget?.maxAttemptsPerError}
                                     onChange={(v) => onChangeBudgetField("maxAttemptsPerError", Number(v))}/>
                     </FormField>
-                    <FormField title="Max attempts total" tooltip="Hard ceiling on total retries across all messages hitting this group, so a burst of failures can't overwhelm the downstream system." className="grow">
+                    <FormField title="Max attempts total" tooltip="Lifetime ceiling on retries across all messages hitting this group, counted separately for each integration. Once reached the group stops retrying for that integration until its counter is cleared." className="grow">
                         <TextEditor type={"number"} value={group.budget?.maxAttemptsTotal}
                                     onChange={(v) => onChangeBudgetField("maxAttemptsTotal", Number(v))}/>
                     </FormField>

src/components/RetryPolicies/RetryBudgetUsage.tsx

@@ -0,0 +1,113 @@
+import React from "react";
+import {useResetRetryPolicyUsageMutation, useRetryPolicyUsageQuery} from "src/client/apis/retryPoliciesApi";
+import Button from "src/components/common/forms/Button";
+import FormField from "src/components/common/forms/FormField";
+import Authorize from "src/components/common/authorize/authorize";
+import dayjs from "dayjs";
+
+interface Props {
+    policyId: number
+}
+
+// "Max attempts total" never resets on its own, so an integration that reaches its ceiling stops
+// being retried until someone clears the counter here. Without this panel that state is invisible.
+const RetryBudgetUsage: React.FC<Props> = ({policyId}) => {
+
+    const {data, isLoading, isError, refetch} = useRetryPolicyUsageQuery(policyId)
+    const [reset] = useResetRetryPolicyUsageMutation()
+
+    const rows = data ?? []
+    const exhaustedCount = rows.filter(r => r.exhausted).length
+
+    return (
+        <FormField title="Budget usage"
+                   tooltip="How much of each group's 'max attempts total' the integrations using this policy have spent. The total never resets on its own — clear it here to let a group retry again.">
+            <p className={"text-xs text-gray-500 mb-2"}>
+                Counted separately for each integration. Integrations that have never failed under this
+                policy do not appear.
+            </p>
+
+            {isLoading && <p className={"text-sm text-gray-400 italic px-2 py-3"}>Loading…</p>}
+
+            {/* Never fall through to the empty state on failure: "no budget spent" would claim
+                every group is untouched when the truth is that we could not find out. */}
+            {isError &&
+                <div className={"flex flex-row items-center justify-between gap-3 text-sm text-red-700 bg-red-50 border border-red-200 rounded px-3 py-2"}>
+                    <span>Could not load budget usage.</span>
+                    <Button variant={"secondary"} onClick={() => refetch()}>Try again</Button>
+                </div>}
+
+            {!isLoading && !isError && rows.length === 0 &&
+                <p className={"text-sm text-gray-400 italic px-2 py-3"}>
+                    No budget spent — every group has its full allowance.
+                </p>}
+
+            {!isError && rows.length > 0 && <div className={"flex flex-col gap-2"}>
+                {exhaustedCount > 0 &&
+                    <p className={"text-sm text-red-700 bg-red-50 border border-red-200 rounded px-3 py-2"}>
+                        {exhaustedCount === 1
+                            ? "1 integration has exhausted its budget and is no longer being retried."
+                            : `${exhaustedCount} integrations have exhausted their budget and are no longer being retried.`}
+                    </p>}
+
+                <table className="appearance-none min-w-full">
+                    <thead className="border-y bg-gray-50">
+                    <tr>
+                        <th scope="col" className="text-sm font-medium text-gray-900 px-6 py-2 text-left">Integration</th>
+                        <th scope="col" className="text-sm font-medium text-gray-900 px-6 py-2 text-left">Group</th>
+                        <th scope="col" className="text-sm font-medium text-gray-900 px-6 py-2 text-left">Used</th>
+                        <th scope="col" className="text-sm font-medium text-gray-900 px-6 py-2 text-left">Last retry</th>
+                        <th scope="col" className="text-sm font-medium text-gray-900 px-6 py-2 text-left"></th>
+                    </tr>
+                    </thead>
+                    <tbody>
+                    {rows.map((r) => (
+                        <tr key={`${r.subscriptionId}-${r.groupId}`} className="bg-white border-b">
+                            <td className="text-sm text-gray-900 font-semibold px-6 py-4 whitespace-nowrap">
+                                {r.subscriptionName}
+                            </td>
+                            <td className="text-sm text-gray-900 font-light px-6 py-4 whitespace-nowrap">
+                                {r.groupName}
+                            </td>
+                            <td className="text-sm px-6 py-4 whitespace-nowrap">
+                                <span className={r.exhausted ? "text-red-600 font-semibold" : "text-gray-900"}>
+                                    {r.attemptsUsed} / {r.maxAttemptsTotal}
+                                </span>
+                                {r.exhausted &&
+                                    <span className={"ml-2 inline-block bg-red-100 text-red-700 rounded px-2 py-0.5 text-xs"}>
+                                        Exhausted
+                                    </span>}
+                            </td>
+                            <td className="text-sm text-gray-900 font-light px-6 py-4 whitespace-nowrap">
+                                {dayjs(r.lastAttemptOn).format("YYYY-MM-DD HH:mm")}
+                            </td>
+                            <td className={"px-6 py-4"}>
+                                <Authorize roles={["Admin", "Member"]}>
+                                    <Button variant={"secondary"}
+                                            onClick={() => reset({
+                                                id: policyId,
+                                                subscriptionId: r.subscriptionId,
+                                                groupId: r.groupId
+                                            })}>
+                                        Reset
+                                    </Button>
+                                </Authorize>
+                            </td>
+                        </tr>
+                    ))}
+                    </tbody>
+                </table>
+
+                <Authorize roles={["Admin", "Member"]}>
+                    <div className={"flex flex-row-reverse"}>
+                        <Button variant={"secondary"} onClick={() => reset({id: policyId})}>
+                            Reset all
+                        </Button>
+                    </div>
+                </Authorize>
+            </div>}
+        </FormField>
+    );
+}
+
+export default RetryBudgetUsage;

src/components/RetryPolicy.tsx

@@ -7,6 +7,7 @@ import Authorize from "src/components/common/authorize/authorize";
 import React, {useEffect, useState} from "react";
 import {RetryPolicyModel} from "src/types/retryPolicies";
 import RetryGroupsEditor from "src/components/RetryPolicies/RetryGroupsEditor";
+import RetryBudgetUsage from "src/components/RetryPolicies/RetryBudgetUsage";
 import TestRetryPolicyModal from "src/components/RetryPolicies/TestRetryPolicyModal";
 import {MdPlayCircleOutline} from "react-icons/md";
 
@@ -62,6 +63,10 @@ const RetryPolicy = () => {
                                 onChange={(g) => onChange("groups", g)}/>
         </div>
 
+        <div className={"bg-white p-2 rounded-lg shadow-lg mt-5"}>
+            <RetryBudgetUsage policyId={Number(id)}/>
+        </div>
+
         <div className={"flex w-full flex-row-reverse gap-2 mt-8"}>
             <Authorize roles={["Admin", "Member"]}>
                 <Button onClick={onUpdate}>Save</Button>

src/components/exchanges/ExchangeList.tsx

@@ -224,6 +224,7 @@ export const ExchangeList: React.FC<Props> = ({
                     xid={showExceptionFor}
                     exception={data.find((i) => i.id == showExceptionFor)?.exception}
                     scheduledRetryOn={data.find((i) => i.id == showExceptionFor)?.scheduledRetryOn}
+                    retryBlockedReason={data.find((i) => i.id == showExceptionFor)?.retryBlockedReason}
                     onClose={() => setShowExceptionFor(null)}
                     onRefresh={refresh}
                 />

src/components/exchanges/RetryModal.tsx

@@ -11,9 +11,10 @@ type Props = {
     onClose: () => void
     xid?: string
     scheduledRetryOn?: string | null
+    retryBlockedReason?: string | null
     onRefresh?: () => void
 }
-const RetryModal: React.FC<Props> = ({exception, onClose, xid, scheduledRetryOn, onRefresh}) => {
+const RetryModal: React.FC<Props> = ({exception, onClose, xid, scheduledRetryOn, retryBlockedReason, onRefresh}) => {
 
     const [resetForRetry, setResetForRetry] = useState<boolean>(false);
     const [runNow] = useRunDelayedRetryNowMutation();
@@ -71,6 +72,12 @@ const RetryModal: React.FC<Props> = ({exception, onClose, xid, scheduledRetryOn,
                 Use "Run Now" to execute it immediately, or wait for it to run automatically.
             </div>
         }
+        {
+            !hasScheduledRetry && retryBlockedReason &&
+            <div className="flex gap-2 flex-col py-2 border border-amber-200 bg-amber-50 px-2 align-center rounded shadow-sm mb-2">
+                Not retried automatically — {retryBlockedReason}. Use &quot;Retry&quot; to run it manually.
+            </div>
+        }
         {
             exception &&
             <div className="flex gap-2 flex-col py-1 border bg-gray-50 px-2 align-center rounded shadow-sm  ">

src/types/retryPolicies.ts

@@ -97,6 +97,24 @@ export interface RetryPolicyRow {
     groupCount: number
 }
 
+// "Max attempts total" is counted per integration, so a policy shared by several
+// integrations reports one row for each that has spent any of its budget.
+export interface RetryGroupUsageRow {
+    subscriptionId: number
+    subscriptionName: string
+    groupId: string
+    groupName: string
+    attemptsUsed: number
+    maxAttemptsTotal: number
+    exhausted: boolean
+    lastAttemptOn: string
+}
+
+export interface RetryPolicyResetUsage {
+    subscriptionId?: number
+    groupId?: string
+}
+
 export interface RetryPoliciesSearchModel {
     limit?: number
     offset?: number

src/types/xchange.ts

@@ -39,6 +39,8 @@ export interface IXchange {
     correlationId: string;
     partnerId: number | null;
     scheduledRetryOn?: string | null;
+    // Why the retry policy declined another attempt, when it declined.
+    retryBlockedReason?: string | null;
 }