diff --git a/internal/api/handler_purchases.go b/internal/api/handler_purchases.go index 75f9ca125..268d53bd3 100644 --- a/internal/api/handler_purchases.go +++ b/internal/api/handler_purchases.go @@ -612,8 +612,13 @@ func (h *Handler) approveViaToken(ctx context.Context, req *events.LambdaFunctio // Check for Gmail-style pre-fire delay (issue #291 wave-2). // Token/email-link path: no authenticated session UUID is available, so the // scheduled transition is recorded as system-initiated (transitioned_by = NULL). + // Fail closed: a config-read error must NOT silently discard the configured + // free-cancel window and execute immediately. Return 500 so the caller can retry. globalCfg, cfgErr := h.config.GetGlobalConfig(ctx) - if cfgErr == nil && globalCfg.GetPurchaseDelay() > 0 { + if cfgErr != nil { + return nil, fmt.Errorf("failed to read global config for purchase delay check: %w", cfgErr) + } + if globalCfg.GetPurchaseDelay() > 0 { return h.approveWithDelay(ctx, execution, globalCfg.GetPurchaseDelay(), actor, nil) } // ApproveExecution now runs the purchase synchronously inside the @@ -699,8 +704,13 @@ func (h *Handler) approvePurchaseViaSession(ctx context.Context, req *events.Lam // Check for Gmail-style pre-fire delay (issue #291 wave-2). When // PurchaseDelayHours > 0 the SDK call is deferred; the user gets a // "scheduled, revoke before X" email and a window to cancel at $0. + // Fail closed: a config-read error must NOT silently discard the configured + // free-cancel window and execute immediately. Return 500 so the caller can retry. globalCfg, cfgErr := h.config.GetGlobalConfig(ctx) - if cfgErr == nil && globalCfg.GetPurchaseDelay() > 0 { + if cfgErr != nil { + return nil, fmt.Errorf("failed to read global config for purchase delay check: %w", cfgErr) + } + if globalCfg.GetPurchaseDelay() > 0 { return h.approveWithDelay(ctx, execution, globalCfg.GetPurchaseDelay(), session.Email, actor) } diff --git a/internal/api/handler_purchases_test.go b/internal/api/handler_purchases_test.go index bfb1919c2..e2ae7a721 100644 --- a/internal/api/handler_purchases_test.go +++ b/internal/api/handler_purchases_test.go @@ -372,6 +372,95 @@ func TestHandler_approvePurchase_SessionExecuteFailureSurfacesAs409(t *testing.T assert.Contains(t, ce.Error(), "could not be approved") } +// --- F3 regression: global-config read error must fail closed (not execute) --- + +// TestHandler_approveViaToken_GlobalConfigError_FailsClosed pins the F3 fix: +// when GetGlobalConfig returns an error during the email-link (token) approve +// path, the handler must return an error rather than silently executing the +// purchase (which would discard the configured free-cancel window). Pre-fix, +// both approve paths used "if cfgErr == nil && delay > 0 { delay }" and fell +// through to execute on any transient config error. +func TestHandler_approveViaToken_GlobalConfigError_FailsClosed(t *testing.T) { + ctx := context.Background() + execID := "f3f3f3f3-f3f3-f3f3-f3f3-f3f3f3f3f301" + contactEmail := "approver@example.com" + + mockConfig := new(MockConfigStore) + exec := approvalTestExec(execID, contactEmail, mockConfig) + mockConfig.On("GetExecutionByID", ctx, execID).Return(exec, nil) + // GetGlobalConfig fails on every call. authorizeApprovalAction consumes the + // first call best-effort (error silently ignored, globalNotify stays ""); + // approveViaToken must fail closed on its own call rather than executing + // the purchase without the configured delay. Matching all calls is correct + // because the first (best-effort) call also fails — contact_email on the + // per-account record still matches the approver, so authorizeApprovalAction + // succeeds despite the config failure. + mockConfig.On("GetGlobalConfig", ctx).Return(nil, errors.New("db transient error")) + + mockAuth := new(MockAuthService) + mockAuth.On("ValidateSession", ctx, "sess-tok").Return(&Session{Email: contactEmail}, nil) + // No approve-any / approve-own permissions — dispatch falls to token path. + mockAuth.On("HasPermissionAPI", ctx, "", "approve-any", "purchases").Return(false, nil).Maybe() + mockAuth.On("HasPermissionAPI", ctx, "", "approve-own", "purchases").Return(false, nil).Maybe() + + mockPurchase := new(MockPurchaseManager) + // Neither approve path must be reached. + + handler := &Handler{purchase: mockPurchase, config: mockConfig, auth: mockAuth} + req := &events.LambdaFunctionURLRequest{ + Headers: map[string]string{"authorization": "Bearer sess-tok"}, + } + + _, err := handler.approvePurchase(ctx, req, execID, "valid-token") + require.Error(t, err, "config error must propagate; must not execute immediately (F3 token path)") + assert.Contains(t, err.Error(), "failed to read global config") + mockPurchase.AssertNotCalled(t, "ApproveExecution", + mock.Anything, mock.Anything, mock.Anything, mock.Anything) + mockPurchase.AssertNotCalled(t, "ApproveAndExecute", + mock.Anything, mock.Anything, mock.Anything, mock.Anything) +} + +// TestHandler_approvePurchaseViaSession_GlobalConfigError_FailsClosed pins the +// F3 fix for the session (dashboard) approve path: same contract as the token +// path — a transient config error must not discard the free-cancel window. +func TestHandler_approvePurchaseViaSession_GlobalConfigError_FailsClosed(t *testing.T) { + ctx := context.Background() + execID := "f3f3f3f3-f3f3-f3f3-f3f3-f3f3f3f3f302" + adminEmail := "admin@example.com" + + mockConfig := new(MockConfigStore) + exec := &config.PurchaseExecution{ + ExecutionID: execID, + ApprovalToken: "valid-token", + Status: "pending", + Recommendations: []config.RecommendationRecord{{ID: "r1"}}, + } + mockConfig.On("GetExecutionByID", ctx, execID).Return(exec, nil) + // GetGlobalConfig always fails — approvePurchaseViaSession must return the + // error rather than executing the purchase without the configured delay. + mockConfig.On("GetGlobalConfig", ctx).Return(nil, errors.New("db transient error")) + + mockAuth := new(MockAuthService) + mockAuth.On("ValidateSession", ctx, "sess-tok").Return(&Session{Email: adminEmail}, nil) + mockAuth.grantAdmin() + // approvePurchaseViaSession enforces CSRF. + mockAuth.On("ValidateCSRFToken", ctx, "sess-tok", "").Return(nil) + + mockPurchase := new(MockPurchaseManager) + // ApproveAndExecute must not be called. + + handler := &Handler{purchase: mockPurchase, config: mockConfig, auth: mockAuth} + req := &events.LambdaFunctionURLRequest{ + Headers: map[string]string{"authorization": "Bearer sess-tok"}, + } + + _, err := handler.approvePurchase(ctx, req, execID, "") + require.Error(t, err, "config error must propagate; must not execute immediately (F3 session path)") + assert.Contains(t, err.Error(), "failed to read global config") + mockPurchase.AssertNotCalled(t, "ApproveAndExecute", + mock.Anything, mock.Anything, mock.Anything, mock.Anything) +} + // --- Regression tests for issue #609 (orphan-account guard) --- // TestHandler_approvePurchase_AzureOrphanRejects409 is the regression guard diff --git a/internal/config/store_postgres.go b/internal/config/store_postgres.go index cff7b69c2..25068892a 100644 --- a/internal/config/store_postgres.go +++ b/internal/config/store_postgres.go @@ -1391,7 +1391,7 @@ func (s *PostgresStore) GetExecutionByPlanAndDate(ctx context.Context, planID st } if len(executions) == 0 { - return nil, fmt.Errorf("execution not found for plan %s at %v", planID, scheduledDate) + return nil, fmt.Errorf("%w: plan %s at %v", ErrNotFound, planID, scheduledDate) } return &executions[0], nil diff --git a/internal/config/store_postgres_pgxmock_test.go b/internal/config/store_postgres_pgxmock_test.go index e29285cd4..ee5239acf 100644 --- a/internal/config/store_postgres_pgxmock_test.go +++ b/internal/config/store_postgres_pgxmock_test.go @@ -2476,3 +2476,40 @@ func TestPGXMock_TransitionExecutionStatus_ProbeHardErrorNotMappedToNotFound(t * assert.ErrorIs(t, err, dbErr) assert.NoError(t, mock.ExpectationsWereMet()) } + +// ─── F2 regression: GetExecutionByPlanAndDate zero-rows wraps ErrNotFound ──── + +// TestPGXMock_GetExecutionByPlanAndDate_NotFoundWrapsErrNotFound is the +// regression test for F2: a zero-row result from GetExecutionByPlanAndDate must +// return an error wrapping config.ErrNotFound so that getOrCreateExecution can +// distinguish "no existing execution" from a real store failure and create a new +// one. Before the fix the function returned a plain fmt.Errorf, which caused +// getOrCreateExecution to treat the not-found case as a fatal error, making the +// create-execution branch unreachable against the real store. +func TestPGXMock_GetExecutionByPlanAndDate_NotFoundWrapsErrNotFound(t *testing.T) { + mock := newMock(t) + store := storeWith(mock) + ctx := context.Background() + + scheduledDate := time.Date(2025, 3, 1, 0, 0, 0, 0, time.UTC) + + // Return an empty row set — simulates the "no existing execution" case. + cols := []string{ + "plan_id", "execution_id", "status", "step_number", "scheduled_date", + "notification_sent", "approval_token", "recommendations", + "total_upfront_cost", "estimated_savings", "completed_at", "error", "expires_at", + "cloud_account_id", "source", "approved_by", "cancelled_by", "capacity_percent", + "created_by_user_id", "retry_execution_id", "retry_attempt_n", + "approval_token_expires_at", + "executed_by_user_id", "executed_at", "pre_approval_skip_reason", + "idempotency_key", "scheduled_execution_at", + } + emptyRows := pgxmock.NewRows(cols) + mock.ExpectQuery("SELECT").WithArgs(pgxmock.AnyArg(), pgxmock.AnyArg()).WillReturnRows(emptyRows) + + _, err := store.GetExecutionByPlanAndDate(ctx, "plan-missing", scheduledDate) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrNotFound), + "zero-row GetExecutionByPlanAndDate must wrap ErrNotFound so getOrCreateExecution can create a new execution; got: %v", err) + assert.NoError(t, mock.ExpectationsWereMet()) +} diff --git a/internal/purchase/coverage_extra_test.go b/internal/purchase/coverage_extra_test.go index e806fd726..1a044efcb 100644 --- a/internal/purchase/coverage_extra_test.go +++ b/internal/purchase/coverage_extra_test.go @@ -265,8 +265,9 @@ func TestHandleExecutePurchase_SaveError(t *testing.T) { mockSTS := new(MockSTSClient) plan := &config.PurchasePlan{ - ID: "plan-save-err", - Name: "Plan", + ID: "plan-save-err", + Name: "Plan", + AutoPurchase: true, } rec := config.RecommendationRecord{ diff --git a/internal/purchase/manager.go b/internal/purchase/manager.go index 60906d372..df5f03432 100644 --- a/internal/purchase/manager.go +++ b/internal/purchase/manager.go @@ -489,6 +489,84 @@ func (m *Manager) RecoverStrandedApprovals(ctx context.Context) (int, error) { return recovered, nil } +// executableByScheduler reports whether a pending or notified execution may be +// auto-executed by the cron sweep or an SQS execute_purchase message, without +// an explicit human approval action (fail closed on money paths). +// +// Rules: +// - source="web" rows must wait for the token-link approval path; the +// scheduler and SQS paths must never bypass that gate. +// - All other pending/notified rows require the owning plan to have +// AutoPurchase=true. A plan-fetch error is propagated so the caller can +// fail closed rather than defaulting to "execute". +// +// "approved" rows are handled by the session/token approval paths and +// RecoverStrandedApprovals; this helper is only called for pending/notified. +func (m *Manager) executableByScheduler(ctx context.Context, exec *config.PurchaseExecution) (bool, error) { + if exec.Source == "web" { + return false, nil + } + plan, err := m.config.GetPurchasePlan(ctx, exec.PlanID) + if err != nil { + return false, fmt.Errorf("failed to fetch plan %s for AutoPurchase gate: %w", exec.PlanID, err) + } + return plan.AutoPurchase, nil +} + +// processOneExecution runs the full gate+claim+execute pipeline for a single +// due pending/notified execution. It updates the Processed/Executed/Failed +// counters and Errors slice on the supplied ProcessResult in place. +// Extracted from ProcessScheduledPurchases to keep that function under the +// gocyclo:10 threshold. +func (m *Manager) processOneExecution(ctx context.Context, exec config.PurchaseExecution, result *ProcessResult) { + // AutoPurchase gate: pending/notified rows are only eligible for + // automatic execution when the owning plan has AutoPurchase=true AND + // the row was not web-submitted (those must go through the token-link + // path). Fail closed: a plan-fetch error counts as a failure rather + // than defaulting to "execute" (no silent money action on error). + eligible, gateErr := m.executableByScheduler(ctx, &exec) + if gateErr != nil { + result.Failed++ + result.Errors = append(result.Errors, fmt.Sprintf("%s: AutoPurchase gate check failed: %v", exec.ExecutionID, gateErr)) + return + } + if !eligible { + logging.Infof("Skipping execution %s (AutoPurchase=false or source=web; requires explicit approval)", exec.ExecutionID) + return + } + + result.Processed++ + logging.Infof("Executing scheduled purchase: %s", exec.ExecutionID) + + // Atomically claim the row before executing (issue #1013). Overlapping + // cron ticks (a tick that runs longer than the interval, EventBridge + // duplicate/overlapping deliveries, or cron racing the SQS path) would + // otherwise both execute the same due row. claimAndExecute CASes the row + // to "running" and only the winner runs; a lost claim is skipped without + // re-executing. + claimed, execErr := m.claimAndExecute(ctx, &exec) + if !claimed { + // execErr != nil here is a real DB error during the claim (count as + // failed); execErr == nil is a benign CAS race-loss (skip silently). + if execErr != nil { + result.Failed++ + result.Errors = append(result.Errors, fmt.Sprintf("%s: claim failed: %v", exec.ExecutionID, execErr)) + } + return + } + + // A multi-account run where at least one account committed is a success + // for ack purposes (issue #1014): the per-account rows own the truth and + // re-running would double-buy. Only a genuine failure (nothing + // committed) is counted/surfaced. + if isMultiAccountAckable(execErr) { + result.Executed++ + return + } + result.Failed++ + result.Errors = append(result.Errors, fmt.Sprintf("%s: %v", exec.ExecutionID, execErr)) +} + // ProcessScheduledPurchases checks for and executes scheduled purchases. func (m *Manager) ProcessScheduledPurchases(ctx context.Context) (*ProcessResult, error) { logging.Info("Processing scheduled purchases...") @@ -509,10 +587,7 @@ func (m *Manager) ProcessScheduledPurchases(ctx context.Context) (*ProcessResult } now := time.Now() - processed := 0 - executed := 0 - failed := 0 - var errs []string + result := &ProcessResult{Recovered: recovered} for i := range executions { exec := executions[i] @@ -531,44 +606,8 @@ func (m *Manager) ProcessScheduledPurchases(ctx context.Context) (*ProcessResult continue } - processed++ - - logging.Infof("Executing scheduled purchase: %s", exec.ExecutionID) - - // Atomically claim the row before executing (issue #1013). Overlapping - // cron ticks (a tick that runs longer than the interval, EventBridge - // duplicate/overlapping deliveries, or cron racing the SQS path) would - // otherwise both execute the same due row. claimAndExecute CASes the row - // to "running" and only the winner runs; a lost claim is skipped without - // re-executing. - claimed, execErr := m.claimAndExecute(ctx, &exec) - if !claimed { - // execErr != nil here is a real DB error during the claim (count as - // failed); execErr == nil is a benign CAS race-loss (skip silently). - if execErr != nil { - failed++ - errs = append(errs, fmt.Sprintf("%s: claim failed: %v", exec.ExecutionID, execErr)) - } - continue - } - - // A multi-account run where at least one account committed is a success - // for ack purposes (issue #1014): the per-account rows own the truth and - // re-running would double-buy. Only a genuine failure (nothing - // committed) is counted/surfaced. - if isMultiAccountAckable(execErr) { - executed++ - continue - } - failed++ - errs = append(errs, fmt.Sprintf("%s: %v", exec.ExecutionID, execErr)) + m.processOneExecution(ctx, exec, result) } - return &ProcessResult{ - Processed: processed, - Executed: executed, - Failed: failed, - Recovered: recovered, - Errors: errs, - }, nil + return result, nil } diff --git a/internal/purchase/manager_test.go b/internal/purchase/manager_test.go index d50d27dc6..edbc2a6f6 100644 --- a/internal/purchase/manager_test.go +++ b/internal/purchase/manager_test.go @@ -197,8 +197,9 @@ func TestManager_ProcessScheduledPurchases_DuePurchase(t *testing.T) { } plan := &config.PurchasePlan{ - ID: "plan-456", - Name: "Test Plan", + ID: "plan-456", + Name: "Test Plan", + AutoPurchase: true, // required by executableByScheduler gate (F1 fix) RampSchedule: config.RampSchedule{ CurrentStep: 0, TotalSteps: 4, @@ -210,9 +211,11 @@ func TestManager_ProcessScheduledPurchases_DuePurchase(t *testing.T) { claimedExec.Status = "running" mockStore.On("GetStaleApprovedExecutions", ctx, mock.Anything).Return([]config.PurchaseExecution{}, nil) mockStore.On("GetPendingExecutions", ctx).Return(executions, nil) + // executableByScheduler calls GetPurchasePlan first (AutoPurchase gate), then + // executePurchase calls it again to build purchase options — two calls total. + mockStore.On("GetPurchasePlan", ctx, "plan-456").Return(plan, nil) mockStore.On("TransitionExecutionStatus", ctx, "exec-123", []string{"approved", "pending", "notified"}, "running", (*string)(nil)).Return(&claimedExec, nil) - mockStore.On("GetPurchasePlan", ctx, "plan-456").Return(plan, nil).Once() mockStore.On("SavePurchaseHistory", ctx, mock.AnythingOfType("*config.PurchaseHistoryRecord")).Return(nil) mockEmail.On("SendPurchaseConfirmation", ctx, mock.AnythingOfType("email.NotificationData")).Return(nil) mockStore.On("SavePurchaseExecution", ctx, mock.AnythingOfType("*config.PurchaseExecution")).Return(nil) @@ -302,13 +305,21 @@ func TestManager_ProcessScheduledPurchases_ExecutionFails(t *testing.T) { }, } + autoPurchasePlan := &config.PurchasePlan{ + ID: "plan-456", + AutoPurchase: true, + } claimedExec := executions[0] claimedExec.Status = "running" mockStore.On("GetStaleApprovedExecutions", ctx, mock.Anything).Return([]config.PurchaseExecution{}, nil) mockStore.On("GetPendingExecutions", ctx).Return(executions, nil) + // First call: executableByScheduler gate check (AutoPurchase=true -> proceed). + // Second call: executePurchase fetches the plan config and returns error to + // simulate a purchase failure — this is what drives the row to "failed". + mockStore.On("GetPurchasePlan", ctx, "plan-456").Return(autoPurchasePlan, nil).Once() + mockStore.On("GetPurchasePlan", ctx, "plan-456").Return(nil, errors.New("plan not found")).Once() mockStore.On("TransitionExecutionStatus", ctx, "exec-123", []string{"approved", "pending", "notified"}, "running", (*string)(nil)).Return(&claimedExec, nil) - mockStore.On("GetPurchasePlan", ctx, "plan-456").Return(nil, errors.New("plan not found")).Once() mockStore.On("SavePurchaseExecution", ctx, mock.AnythingOfType("*config.PurchaseExecution")).Return(nil) // updatePlanProgress is NOT called when execution fails @@ -327,6 +338,137 @@ func TestManager_ProcessScheduledPurchases_ExecutionFails(t *testing.T) { mockStore.AssertExpectations(t) } +// ─── F1 regression: AutoPurchase gate (adversarial review follow-up) ───────── + +// TestManager_ProcessScheduledPurchases_PendingAutoFalseSkipped is the +// regression test for F1: a pending execution whose owning plan has +// AutoPurchase=false must NOT be executed by the cron sweep. Before the fix +// the sweep executed all pending/notified rows regardless of the plan setting. +func TestManager_ProcessScheduledPurchases_PendingAutoFalseSkipped(t *testing.T) { + ctx := context.Background() + mockStore := new(MockConfigStore) + mockEmail := new(MockEmailSender) + + pastDate := time.Now().Add(-1 * time.Hour) + executions := []config.PurchaseExecution{ + { + ExecutionID: "exec-noauto", + PlanID: "plan-noauto", + Status: "pending", + ScheduledDate: pastDate, + }, + } + plan := &config.PurchasePlan{ + ID: "plan-noauto", + AutoPurchase: false, // gate must block execution + } + + mockStore.On("GetStaleApprovedExecutions", ctx, mock.Anything).Return([]config.PurchaseExecution{}, nil) + mockStore.On("GetPendingExecutions", ctx).Return(executions, nil) + mockStore.On("GetPurchasePlan", ctx, "plan-noauto").Return(plan, nil) + // TransitionExecutionStatus must NOT be called — the row must be skipped. + + manager := &Manager{ + config: mockStore, + email: mockEmail, + dashboardURL: "https://dashboard.example.com", + } + + result, err := manager.ProcessScheduledPurchases(ctx) + require.NoError(t, err) + assert.Equal(t, 0, result.Processed, "AutoPurchase=false rows must not be processed") + assert.Equal(t, 0, result.Executed) + assert.Equal(t, 0, result.Failed) + + mockStore.AssertExpectations(t) + mockStore.AssertNotCalled(t, "TransitionExecutionStatus", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) +} + +// TestManager_ProcessScheduledPurchases_WebSourceSkipped verifies that a +// pending execution with Source="web" is never auto-executed by the cron +// sweep, even when the plan has AutoPurchase=true. Web-submitted rows must +// wait for the token-link approval path. +func TestManager_ProcessScheduledPurchases_WebSourceSkipped(t *testing.T) { + ctx := context.Background() + mockStore := new(MockConfigStore) + mockEmail := new(MockEmailSender) + + pastDate := time.Now().Add(-1 * time.Hour) + executions := []config.PurchaseExecution{ + { + ExecutionID: "exec-web", + PlanID: "plan-web", + Status: "pending", + Source: "web", + ScheduledDate: pastDate, + }, + } + + mockStore.On("GetStaleApprovedExecutions", ctx, mock.Anything).Return([]config.PurchaseExecution{}, nil) + mockStore.On("GetPendingExecutions", ctx).Return(executions, nil) + // executableByScheduler short-circuits on Source="web" — GetPurchasePlan must NOT be called. + // TransitionExecutionStatus must NOT be called. + + manager := &Manager{ + config: mockStore, + email: mockEmail, + dashboardURL: "https://dashboard.example.com", + } + + result, err := manager.ProcessScheduledPurchases(ctx) + require.NoError(t, err) + assert.Equal(t, 0, result.Processed, "web-sourced rows must not be auto-executed") + assert.Equal(t, 0, result.Executed) + assert.Equal(t, 0, result.Failed) + + mockStore.AssertExpectations(t) + mockStore.AssertNotCalled(t, "GetPurchasePlan", mock.Anything, mock.Anything) + mockStore.AssertNotCalled(t, "TransitionExecutionStatus", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) +} + +// TestManager_ProcessScheduledPurchases_GateCheckErrorCounted verifies that a +// plan-fetch error during the AutoPurchase gate check counts as a failure and +// does NOT proceed to claim+execute the row. +func TestManager_ProcessScheduledPurchases_GateCheckErrorCounted(t *testing.T) { + ctx := context.Background() + mockStore := new(MockConfigStore) + mockEmail := new(MockEmailSender) + + pastDate := time.Now().Add(-1 * time.Hour) + executions := []config.PurchaseExecution{ + { + ExecutionID: "exec-gate-err", + PlanID: "plan-gate-err", + Status: "pending", + ScheduledDate: pastDate, + }, + } + + mockStore.On("GetStaleApprovedExecutions", ctx, mock.Anything).Return([]config.PurchaseExecution{}, nil) + mockStore.On("GetPendingExecutions", ctx).Return(executions, nil) + mockStore.On("GetPurchasePlan", ctx, "plan-gate-err").Return(nil, errors.New("db timeout")) + + manager := &Manager{ + config: mockStore, + email: mockEmail, + dashboardURL: "https://dashboard.example.com", + } + + result, err := manager.ProcessScheduledPurchases(ctx) + require.NoError(t, err) + assert.Equal(t, 0, result.Processed, "gate-check error rows are not yet processed") + assert.Equal(t, 0, result.Executed) + assert.Equal(t, 1, result.Failed, "gate-check error must count as a failure") + assert.Len(t, result.Errors, 1) + assert.Contains(t, result.Errors[0], "AutoPurchase gate check failed") + + mockStore.AssertExpectations(t) + mockStore.AssertNotCalled(t, "TransitionExecutionStatus", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) +} + // TestManager_RecoverStrandedApprovals_FailsStrandedRow is the regression test // for issue #632 safe-fail path: an Azure Savings Plans execution flipped to // "approved" whose synchronous purchase run was interrupted before it finalized diff --git a/internal/purchase/messages.go b/internal/purchase/messages.go index 9da26e234..9ccf226f3 100644 --- a/internal/purchase/messages.go +++ b/internal/purchase/messages.go @@ -80,6 +80,29 @@ func (m *Manager) ProcessMessage(ctx context.Context, body string) error { } // handleExecutePurchase processes an execute_purchase message. +// checkAutoExecuteGate enforces the AutoPurchase and source gate for +// pending/notified rows on the SQS execute_purchase path. Extracted from +// handleExecutePurchase to keep that function under the gocyclo:10 limit. +// Returns nil when execution may proceed; a non-nil error means the message +// should be dead-lettered (the row is not eligible for auto-execution). +// +// "approved" rows bypass the gate: they have already passed the human +// approval path and are always eligible for execution. +func (m *Manager) checkAutoExecuteGate(ctx context.Context, execution *config.PurchaseExecution) error { + if execution.Status != "pending" && execution.Status != "notified" { + return nil + } + eligible, err := m.executableByScheduler(ctx, execution) + if err != nil { + return fmt.Errorf("AutoPurchase gate check failed for execution %s: %w", execution.ExecutionID, err) + } + if !eligible { + return fmt.Errorf("execution %s (status=%s source=%q) is not eligible for auto-execution: AutoPurchase=false or source=web — approve via the approval link", + execution.ExecutionID, execution.Status, execution.Source) + } + return nil +} + func (m *Manager) handleExecutePurchase(ctx context.Context, msg *AsyncMessage) error { if msg.ExecutionID == "" { return fmt.Errorf("execution_id required for execute_purchase message") @@ -95,6 +118,13 @@ func (m *Manager) handleExecutePurchase(ctx context.Context, msg *AsyncMessage) logging.Infof("Executing purchase from async message: %s", msg.ExecutionID) + // AutoPurchase gate: pending/notified rows require AutoPurchase=true on the + // owning plan; web-sourced rows must use the token-link approval path. + // "approved" rows bypass the gate (already human-approved). Fail closed. + if err := m.checkAutoExecuteGate(ctx, execution); err != nil { + return err + } + // Atomically claim the row before touching the cloud (issue #1013). SQS // delivery is at-least-once: a redelivered execute_purchase message (slow // purchase whose visibility timeout expired, partial-batch replay, operator diff --git a/internal/purchase/messages_test.go b/internal/purchase/messages_test.go index bec9a539f..2e9a55867 100644 --- a/internal/purchase/messages_test.go +++ b/internal/purchase/messages_test.go @@ -7,6 +7,7 @@ import ( "github.com/LeanerCloud/CUDly/internal/config" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) func TestManager_ProcessMessage(t *testing.T) { @@ -106,6 +107,54 @@ func TestManager_ProcessMessage(t *testing.T) { assert.Contains(t, err.Error(), "execution not found") }) + // ─── F1 regression: AutoPurchase gate on SQS execute_purchase path ──────── + + t.Run("execute_purchase pending web-source rejected (F1)", func(t *testing.T) { + // A web-submitted pending row must not be auto-executed via SQS; + // it must wait for the token-link approval path. + mockStore := new(MockConfigStore) + manager := &Manager{config: mockStore, dashboardURL: "https://example.com"} + execution := &config.PurchaseExecution{ + ExecutionID: "exec-web", + PlanID: "plan-1", + Status: "pending", + Source: "web", + } + mockStore.On("GetExecutionByID", ctx, "exec-web").Return(execution, nil) + // GetPurchasePlan must NOT be called (Source=web short-circuits) + // TransitionExecutionStatus must NOT be called + + err := manager.ProcessMessage(ctx, `{"type": "execute_purchase", "execution_id": "exec-web"}`) + assert.Error(t, err) + assert.Contains(t, err.Error(), "not eligible for auto-execution") + mockStore.AssertNotCalled(t, "GetPurchasePlan", mock.Anything, mock.Anything) + mockStore.AssertNotCalled(t, "TransitionExecutionStatus", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + }) + + t.Run("execute_purchase pending AutoPurchase=false rejected (F1)", func(t *testing.T) { + // A pending row under a plan with AutoPurchase=false must not be + // auto-executed via SQS. + mockStore := new(MockConfigStore) + manager := &Manager{config: mockStore, dashboardURL: "https://example.com"} + execution := &config.PurchaseExecution{ + ExecutionID: "exec-noauto", + PlanID: "plan-noauto", + Status: "pending", + } + plan := &config.PurchasePlan{ID: "plan-noauto", AutoPurchase: false} + mockStore.On("GetExecutionByID", ctx, "exec-noauto").Return(execution, nil) + mockStore.On("GetPurchasePlan", ctx, "plan-noauto").Return(plan, nil) + // TransitionExecutionStatus must NOT be called + + err := manager.ProcessMessage(ctx, `{"type": "execute_purchase", "execution_id": "exec-noauto"}`) + assert.Error(t, err) + assert.Contains(t, err.Error(), "not eligible for auto-execution") + mockStore.AssertNotCalled(t, "TransitionExecutionStatus", + mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything) + mockStore.AssertExpectations(t) + }) + t.Run("execute_purchase wrong status", func(t *testing.T) { mockStore := new(MockConfigStore) mockEmail := new(MockEmailSender) diff --git a/internal/purchase/money_path_regression_test.go b/internal/purchase/money_path_regression_test.go index eaaaf374b..8b85e0b2e 100644 --- a/internal/purchase/money_path_regression_test.go +++ b/internal/purchase/money_path_regression_test.go @@ -247,7 +247,7 @@ func TestSQSRedeliveryDoesNotDoubleExecute(t *testing.T) { mockStore.SavePurchaseExecutionFn = func(_ context.Context, _ *config.PurchaseExecution) error { return nil } mockStore.On("SavePurchaseHistory", ctx, mock.AnythingOfType("*config.PurchaseHistoryRecord")).Return(nil) mockEmail.On("SendPurchaseConfirmation", ctx, mock.AnythingOfType("email.NotificationData")).Return(nil) - mockStore.On("GetPurchasePlan", ctx, mock.Anything).Return(&config.PurchasePlan{Name: "p"}, nil).Maybe() + mockStore.On("GetPurchasePlan", ctx, mock.Anything).Return(&config.PurchasePlan{Name: "p", AutoPurchase: true}, nil).Maybe() mockFactory.On("CreateAndValidateProvider", mock.Anything, "aws", mock.Anything).Return(mockProviderInst, nil) mockProviderInst.On("GetServiceClient", mock.Anything, common.ServiceEC2, mock.Anything).Return(mockServiceClient, nil) @@ -303,14 +303,14 @@ func TestMultiAccountPartialSuccessIsAcked(t *testing.T) { {Provider: "aws", Service: "ec2", ResourceType: "m5.large", Region: "us-east-1", Count: 1, UpfrontCost: 300, Selected: true}, }, } - plan := &config.PurchasePlan{ID: "plan-x", Name: "Plan X"} + plan := &config.PurchasePlan{ID: "plan-x", Name: "Plan X", AutoPurchase: true} mockStore.On("GetExecutionByID", ctx, "root-partial").Return(exec, nil) running := *exec running.Status = "running" mockStore.On("TransitionExecutionStatus", ctx, "root-partial", []string{"approved", "pending", "notified"}, "running", (*string)(nil)).Return(&running, nil) - mockStore.On("GetPurchasePlan", ctx, "plan-x").Return(plan, nil) + mockStore.On("GetPurchasePlan", ctx, "plan-x").Return(plan, nil).Maybe() // GetPlanAccounts is served by the Fn hook, not a testify expectation. mockStore.GetPlanAccountsFn = func(_ context.Context, _ string) ([]config.CloudAccount, error) { return accounts, nil diff --git a/internal/purchase/notifications.go b/internal/purchase/notifications.go index 531ba1531..f084a23b1 100644 --- a/internal/purchase/notifications.go +++ b/internal/purchase/notifications.go @@ -2,6 +2,7 @@ package purchase import ( "context" + "errors" "fmt" "time" @@ -108,15 +109,18 @@ func (m *Manager) sendPlanNotification(ctx context.Context, plan *config.Purchas // getOrCreateExecution gets existing execution or creates new one. func (m *Manager) getOrCreateExecution(ctx context.Context, plan *config.PurchasePlan) (*config.PurchaseExecution, error) { - // Check for existing execution for this date to prevent duplicates + // Check for existing execution for this date to prevent duplicates. + // GetExecutionByPlanAndDate wraps ErrNotFound on zero rows; any other + // error is a real store failure and must propagate. existing, err := m.config.GetExecutionByPlanAndDate(ctx, plan.ID, *plan.NextExecutionDate) - if err != nil { - return nil, fmt.Errorf("failed to check for existing execution: %w", err) - } - if existing != nil { + switch { + case err == nil && existing != nil: logging.Debugf("Found existing execution %s for plan %s on %s", existing.ExecutionID, plan.ID, plan.NextExecutionDate) return existing, nil + case err != nil && !errors.Is(err, config.ErrNotFound): + return nil, fmt.Errorf("failed to check for existing execution: %w", err) } + // ErrNotFound (or nil error with nil row): no existing execution for this plan+date; create a new one. approvalToken, err := common.GenerateApprovalToken() if err != nil { diff --git a/internal/purchase/notifications_test.go b/internal/purchase/notifications_test.go index f4a296822..ea8d352ca 100644 --- a/internal/purchase/notifications_test.go +++ b/internal/purchase/notifications_test.go @@ -3,6 +3,7 @@ package purchase import ( "context" "errors" + "fmt" "testing" "time" @@ -295,6 +296,42 @@ func TestManager_GetOrCreateExecution_LookupError(t *testing.T) { mockStore.AssertExpectations(t) } +// TestManager_GetOrCreateExecution_CreatesOnErrNotFound is the F2 regression +// guard: when GetExecutionByPlanAndDate wraps ErrNotFound (zero rows), the +// create branch must fire rather than treating it as a hard error. +// Pre-fix, the store returned a plain fmt.Errorf on zero rows, so getOrCreateExecution +// treated it as a fatal error and the create branch was unreachable. +func TestManager_GetOrCreateExecution_CreatesOnErrNotFound(t *testing.T) { + ctx := context.Background() + mockStore := new(MockConfigStore) + + nextExec := time.Now().Add(24 * time.Hour) + plan := &config.PurchasePlan{ + ID: "plan-f2", + Name: "F2 Plan", + NextExecutionDate: &nextExec, + RampSchedule: config.RampSchedule{CurrentStep: 2}, + } + + // Store returns ErrNotFound (wrapped), matching the post-fix store behavior. + notFoundErr := fmt.Errorf("%w: plan plan-f2 at %v", config.ErrNotFound, nextExec) + mockStore.On("GetExecutionByPlanAndDate", ctx, "plan-f2", nextExec).Return(nil, notFoundErr) + mockStore.On("SavePurchaseExecution", ctx, mock.AnythingOfType("*config.PurchaseExecution")).Return(nil) + + manager := &Manager{config: mockStore, dashboardURL: "https://example.com"} + + execution, err := manager.getOrCreateExecution(ctx, plan) + require.NoError(t, err, "ErrNotFound must trigger the create path, not a hard error (F2)") + require.NotNil(t, execution) + assert.Equal(t, "plan-f2", execution.PlanID) + assert.Equal(t, "pending", execution.Status) + assert.Equal(t, 2, execution.StepNumber) + assert.NotEmpty(t, execution.ExecutionID) + assert.NotEmpty(t, execution.ApprovalToken) + + mockStore.AssertExpectations(t) +} + func TestManager_SendUpcomingPurchaseNotifications_WithNotification(t *testing.T) { ctx := context.Background() mockStore := new(MockConfigStore) diff --git a/internal/purchase/reaper.go b/internal/purchase/reaper.go index f760a0102..aa8c1c596 100644 --- a/internal/purchase/reaper.go +++ b/internal/purchase/reaper.go @@ -215,8 +215,17 @@ func (m *Manager) reapOne(ctx context.Context, exec *config.PurchaseExecution, r // Best-effort: if this save fails (network blip, etc.) the row is // still in "failed" — operator just sees a generic failure without // the reaper attribution. Bump Errored so ops can track it. - transitioned.Error = fmt.Sprintf("reaped after %dm in %s state — executor did not complete; safe to retry", - ageMinutes, prevStatus) + // + // "safe to retry" is only appended when every recommendation in the + // execution uses an idempotent provider API. Azure savings-plans use a + // timestamp-based alias name with no server-side idempotency key, so + // retrying a reaped Azure SP row risks creating a duplicate commitment. + safeMsg := "" + if allRecsSafeToRedrive(exec) { + safeMsg = "; safe to retry" + } + transitioned.Error = fmt.Sprintf("reaped after %dm in %s state — executor did not complete%s", + ageMinutes, prevStatus, safeMsg) if saveErr := m.config.SavePurchaseExecution(ctx, transitioned); saveErr != nil { logging.Errorf("purchase reaper: failed to persist canonical error for execution %s (already flipped to failed): %v", exec.ExecutionID, saveErr) diff --git a/internal/purchase/reaper_test.go b/internal/purchase/reaper_test.go index 1904658db..d41223175 100644 --- a/internal/purchase/reaper_test.go +++ b/internal/purchase/reaper_test.go @@ -23,10 +23,9 @@ func newReaperManager(store *MockConfigStore) *Manager { } } -// stuckExec builds a representative stuck-approved execution. Tests -// override Status / ExecutionID as needed. The age claim (`updated_at` -// being old) is implicit — the SELECT in ListStuckExecutions is what -// enforces it, and the test mocks return whatever the test wants. +// stuckExec builds a representative stuck execution with no recommendations. +// allRecsSafeToRedrive returns false for empty recs, so no "safe to retry" +// is appended to the reaper's canonical error message. func stuckExec(id, status string) config.PurchaseExecution { return config.PurchaseExecution{ PlanID: "plan-1", @@ -37,12 +36,25 @@ func stuckExec(id, status string) config.PurchaseExecution { } } +// stuckExecWithAWSRecs builds a stuck execution with a single AWS EC2 rec. +// allRecsSafeToRedrive returns true for pure-AWS recs, so the reaper's +// canonical error includes "safe to retry" (F4 fix). +func stuckExecWithAWSRecs(id, status string) config.PurchaseExecution { + e := stuckExec(id, status) + e.Recommendations = []config.RecommendationRecord{ + {Provider: "aws", Service: "ec2", ResourceType: "m5.large", Count: 1}, + } + return e +} + func TestReapStuckExecutions_StaleApprovedFlippedToFailed(t *testing.T) { ctx := context.Background() store := new(MockConfigStore) reapAfter := 10 * time.Minute - row := stuckExec("exec-A", "approved") + // AWS recs are idempotent (ClientToken / tag-guard), so "safe to retry" + // must appear in the canonical error for operator guidance (F4 fix). + row := stuckExecWithAWSRecs("exec-A", "approved") transitioned := row transitioned.Status = failedStatus @@ -389,3 +401,70 @@ func TestParseReapAfterFromEnv_NonStandardButValidGoDuration(t *testing.T) { got := ParseReapAfterFromEnv() assert.Equal(t, 2*time.Hour+30*time.Minute, got) } + +// ─── F4 regression: safe-to-retry gate (adversarial review follow-up) ──────── + +// TestReapStuckExecutions_AzureSPNotSafeToRetry is the regression test for F4: +// a stranded Azure savings-plans execution must NOT receive "safe to retry" in +// its canonical error message. Azure SP purchases use a timestamp-based alias +// name with no server-side idempotency key, so an operator retry would create +// a duplicate savings plan. +func TestReapStuckExecutions_AzureSPNotSafeToRetry(t *testing.T) { + ctx := context.Background() + store := new(MockConfigStore) + reapAfter := 10 * time.Minute + + row := stuckExec("exec-azsp", "approved") + row.Recommendations = []config.RecommendationRecord{ + {Provider: "azure", Service: "savingsplans", Count: 1}, + } + transitioned := row + transitioned.Status = failedStatus + + store.On("ListStuckExecutions", ctx, stuckStatuses, reapAfter). + Return([]config.PurchaseExecution{row}, nil) + store.On("TransitionExecutionStatus", ctx, "exec-azsp", stuckStatuses, failedStatus, (*string)(nil)). + Return(&transitioned, nil) + store.On("SavePurchaseExecution", ctx, mock.MatchedBy(func(e *config.PurchaseExecution) bool { + // "safe to retry" MUST NOT appear — Azure SP is not idempotent. + return e.ExecutionID == "exec-azsp" && + e.Status == failedStatus && + strings.Contains(e.Error, "reaped after") && + !strings.Contains(e.Error, "safe to retry") + })).Return(nil) + + mgr := newReaperManager(store) + result, err := mgr.ReapStuckExecutions(ctx, reapAfter) + require.NoError(t, err) + assert.Equal(t, 1, result.Reaped) + store.AssertExpectations(t) +} + +// TestReapStuckExecutions_EmptyRecsNotSafeToRetry verifies that executions +// with no recommendations are not stamped "safe to retry": an empty rec-set +// means allRecsSafeToRedrive returns false (unknown purchase state). +func TestReapStuckExecutions_EmptyRecsNotSafeToRetry(t *testing.T) { + ctx := context.Background() + store := new(MockConfigStore) + reapAfter := 10 * time.Minute + + row := stuckExec("exec-empty", "running") // stuckExec has no recs + transitioned := row + transitioned.Status = failedStatus + + store.On("ListStuckExecutions", ctx, stuckStatuses, reapAfter). + Return([]config.PurchaseExecution{row}, nil) + store.On("TransitionExecutionStatus", ctx, "exec-empty", stuckStatuses, failedStatus, (*string)(nil)). + Return(&transitioned, nil) + store.On("SavePurchaseExecution", ctx, mock.MatchedBy(func(e *config.PurchaseExecution) bool { + return e.ExecutionID == "exec-empty" && + e.Status == failedStatus && + !strings.Contains(e.Error, "safe to retry") + })).Return(nil) + + mgr := newReaperManager(store) + result, err := mgr.ReapStuckExecutions(ctx, reapAfter) + require.NoError(t, err) + assert.Equal(t, 1, result.Reaped) + store.AssertExpectations(t) +}