Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 60 additions & 3 deletions internal/api/handler_purchases_run_now_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,20 @@ import (
// runNowService buys every recommendation except the call numbered failOn.
type runNowService struct {
fanoutServiceClient
calls atomic.Int32
failOn int32
calls atomic.Int32
failOn int32
onPurchase func(context.Context)
}

func (s *runNowService) PurchaseCommitment(ctx context.Context, rec common.Recommendation, opts common.PurchaseOptions) (common.PurchaseResult, error) {
if s.calls.Add(1) == s.failOn {
call := s.calls.Add(1)
if s.onPurchase != nil {
s.onPurchase(ctx)
}
if err := ctx.Err(); err != nil {
return common.PurchaseResult{}, err
}
if call == s.failOn {
return common.PurchaseResult{}, errors.New("provider rejected the purchase")
}
return s.fanoutServiceClient.PurchaseCommitment(ctx, rec, opts)
Expand Down Expand Up @@ -145,3 +153,52 @@ func TestRunNow_LostClaimIsConflict(t *testing.T) {
assert.Nil(t, row.ExecutedAt)
assert.Zero(t, service.calls.Load())
}

func TestRunNow_CanceledRequestAfterClaimPersistsTerminalOutcome(t *testing.T) {
for _, tc := range []struct {
name string
failOn int32
status string
history int
}{
{name: "completed", status: "completed", history: 1},
{name: "failed", failOn: 1, status: "failed"},
} {
t.Run(tc.name, func(t *testing.T) {
f, service := newRunNowFixture(t, tc.failOn, fanoutRecs())
requestCtx, cancel := context.WithCancel(f.ctx)
defer cancel()
var claimStatus string
var claimErr error
service.onPurchase = func(context.Context) {
row, err := f.store.GetExecutionByID(f.ctx, f.execution.ExecutionID)
claimErr = err
if row != nil {
claimStatus = row.Status
}
cancel()
}
_, runErr := f.handler.runPlannedPurchase(requestCtx, f.request, f.execution.ExecutionID)
require.NoError(t, claimErr)
require.Equal(t, "running", claimStatus)
require.ErrorIs(t, requestCtx.Err(), context.Canceled)
if tc.failOn == 0 {
require.NoError(t, runErr)
} else {
require.Error(t, runErr)
}
verifyCtx, verifyCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer verifyCancel()
row, err := f.store.GetExecutionByID(verifyCtx, f.execution.ExecutionID)
require.NoError(t, err)
require.Equal(t, tc.status, row.Status)
var histories int
require.NoError(t, f.pool.QueryRow(verifyCtx, "SELECT count(*) FROM purchase_history WHERE plan_id=$1", f.planID).Scan(&histories))
require.Equal(t, tc.history, histories)
freshRequest := *f.request
_, retryErr := f.handler.runPlannedPurchase(verifyCtx, &freshRequest, f.execution.ExecutionID)
require.Error(t, retryErr)
assert.EqualValues(t, 1, service.calls.Load())
})
}
}
4 changes: 2 additions & 2 deletions internal/api/handler_purchases_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6674,9 +6674,9 @@ func TestHandler_approvePurchaseViaSession_FourEyesOn_DifferentApproverSucceeds(
// refused at execution (#609), and this test is about the 4-eyes check.
approved := &config.PurchaseExecution{ExecutionID: execID, Status: "approved"}
mockConfig.On("GetExecutionByID", ctx, execID).Return(exec, nil)
mockConfig.On("GetGlobalConfig", ctx).Return(fourEyesCfgOn(), nil)
mockConfig.On("GetGlobalConfig", mock.Anything).Return(fourEyesCfgOn(), nil)
mockConfig.On("TransitionExecutionStatus", ctx, execID, []string{"pending", "notified"}, "running", &approverID).Return(approved, nil)
mockConfig.On("SavePurchaseExecution", ctx, mock.AnythingOfType("*config.PurchaseExecution")).Return(nil)
mockConfig.On("SavePurchaseExecution", mock.Anything, mock.AnythingOfType("*config.PurchaseExecution")).Return(nil)

mockAuth := new(MockAuthService)
mockAuth.On("ValidateSession", ctx, "sess-tok").Return(&Session{UserID: approverID, Email: approverEmail}, nil)
Expand Down
4 changes: 2 additions & 2 deletions internal/purchase/approvals_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -546,7 +546,7 @@ func TestManager_ApproveAndExecute_FourEyesOn_AllowsDifferentApprover(t *testing
Recommendations: approvalTestRecs(),
}

store.On("GetGlobalConfig", ctx).Return(fourEyesCfgOnForManager(), nil)
store.On("GetGlobalConfig", mock.Anything).Return(fourEyesCfgOnForManager(), nil)
store.On("GetExecutionByID", ctx, "exec-direct-diff").Return(execution, nil)
store.On("TransitionExecutionStatus", ctx, "exec-direct-diff", approveFromStatuses, "running", &approverUUID).Return(updated, nil)
stubExecuteChain(t, store, sender, "plan-fourEyes")
Expand Down Expand Up @@ -714,7 +714,7 @@ func TestManager_ApproveExecution_FourEyesOn_DifferentActor_Allowed(t *testing.T
}

store.On("GetExecutionByID", ctx, "exec-sqs-diff").Return(execution, nil)
store.On("GetGlobalConfig", ctx).Return(fourEyesCfgOnForManager(), nil)
store.On("GetGlobalConfig", mock.Anything).Return(fourEyesCfgOnForManager(), nil)
store.On("GetUserEmailByID", ctx, creatorID).Return("creator@example.com", nil)
store.On("TransitionExecutionStatus", ctx, "exec-sqs-diff", approveFromStatuses, "running", (*string)(nil)).Return(updated, nil)
stubExecuteChain(t, store, sender, "plan-fourEyes")
Expand Down
3 changes: 2 additions & 1 deletion internal/purchase/claim_guard_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,8 @@ func TestApprove_ClaimsRunningAndStampsGuarded(t *testing.T) {
store.On("GetExecutionByID", mock.Anything, "exec-g").Return(&config.PurchaseExecution{ExecutionID: "exec-g", PlanID: "plan-g", Status: "pending"}, nil).Maybe()
store.On("TransitionExecutionStatus", ctx, "exec-g", approveFromStatuses, "running", (*string)(nil)).Return(updated, nil)
stubExecuteChain(t, store, sender, "plan-g")
store.On("SaveExecutionIfStatus", ctx, mock.Anything, []string{"running"}).Return(nil)
store.On("SaveExecutionIfStatus", ctx, mock.Anything, []string{"running"}).Return(nil).Once()
store.On("SaveExecutionIfStatus", mock.Anything, mock.Anything, []string{"running"}).Return(nil).Once()

_, _, err := manager.ApproveAndExecute(ctx, "exec-g", "approver@example.invalid", nil)
require.NoError(t, err)
Expand Down
9 changes: 5 additions & 4 deletions internal/purchase/claim_pin_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,12 +159,12 @@ func TestClaimPin_ApprovedRowSkipsCapRecheck(t *testing.T) {
ctx := context.Background()
exec := dueRecExec("approved", upfrontRec(1e9))
store := new(MockConfigStore)
store.On("GetPurchasePlan", ctx, exec.PlanID).Return(autoPlan(nil), nil)
store.On("GetPlanAccounts", ctx, exec.PlanID).Return(nil, errors.New("accounts unavailable"))
store.On("GetPurchasePlan", mock.Anything, exec.PlanID).Return(autoPlan(nil), nil)
store.On("GetPlanAccounts", mock.Anything, exec.PlanID).Return(nil, errors.New("accounts unavailable"))
claimed := exec
claimed.Status = "running"
store.On("TransitionExecutionStatusPinned", ctx, exec.ExecutionID, mock.Anything, "running", (*string)(nil), false).Return(&claimed, nil)
store.On("SavePurchaseExecution", ctx, mock.Anything).Return(nil)
store.On("SavePurchaseExecution", mock.Anything, mock.Anything).Return(nil)
manager := &Manager{config: store, email: new(MockEmailSender), providerFactory: new(MockProviderFactory)}

didClaim, err := manager.claimAndExecute(ctx, &exec)
Expand Down Expand Up @@ -209,11 +209,12 @@ func TestClaimPin_RecLessRowStillFailsInsteadOfBuying(t *testing.T) {
store, factory := new(MockConfigStore), new(MockProviderFactory)
exec := dueRecExec("pending")
manager := newPinManager(ctx, store, factory, exec, autoPlan(nil), autoPlan(nil))
store.On("GetPurchasePlan", mock.Anything, exec.PlanID).Return(autoPlan(nil), nil)
claimed := exec
claimed.Status = "running"
store.On("TransitionExecutionStatusPinned", ctx, "exec-gate", mock.Anything, "running", (*string)(nil), true).Return(&claimed, nil)
var saved *config.PurchaseExecution
store.On("SavePurchaseExecutionTx", ctx, mock.Anything, mock.Anything).
store.On("SavePurchaseExecutionTx", mock.Anything, mock.Anything, mock.Anything).
Run(func(args mock.Arguments) { saved = args.Get(2).(*config.PurchaseExecution) }).Return(nil)
result, err := manager.ProcessScheduledPurchases(ctx)
require.NoError(t, err)
Expand Down
39 changes: 20 additions & 19 deletions internal/purchase/coverage_extra_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -240,12 +240,12 @@ func TestHandleExecutePurchase_ApprovedStatus(t *testing.T) {
runningExec.Status = "running"
mockStore.On("TransitionExecutionStatus", ctx, "exec-approved",
[]string{"approved", "pending", "notified"}, "running", (*string)(nil)).Return(&runningExec, nil)
mockStore.On("GetPurchasePlan", ctx, "plan-approved").Return(plan, nil)
mockEmail.On("SendPurchaseConfirmation", ctx, mock.AnythingOfType("email.NotificationData")).Return(nil)
mockStore.On("SavePurchaseExecution", ctx, mock.AnythingOfType("*config.PurchaseExecution")).Return(nil)
mockStore.On("CompletePlanStep", ctx, "plan-approved", 1).Return(nil)
mockStore.On("SavePurchaseHistory", ctx, mock.AnythingOfType("*config.PurchaseHistoryRecord")).Return(nil)
mockSTS.On("GetCallerIdentity", ctx, mock.Anything).Return(nil, errors.New("sts error"))
mockStore.On("GetPurchasePlan", mock.Anything, "plan-approved").Return(plan, nil)
mockEmail.On("SendPurchaseConfirmation", mock.Anything, mock.AnythingOfType("email.NotificationData")).Return(nil)
mockStore.On("SavePurchaseExecution", mock.Anything, mock.AnythingOfType("*config.PurchaseExecution")).Return(nil)
mockStore.On("CompletePlanStep", mock.Anything, "plan-approved", 1).Return(nil)
mockStore.On("SavePurchaseHistory", mock.Anything, mock.AnythingOfType("*config.PurchaseHistoryRecord")).Return(nil)
mockSTS.On("GetCallerIdentity", mock.Anything, mock.Anything).Return(nil, errors.New("sts error"))
mockProv := new(MockProvider)
mockSvc := new(MockServiceClient)
mockFactory.On("CreateAndValidateProvider", mock.Anything, "aws", mock.Anything).Return(mockProv, nil)
Expand Down Expand Up @@ -326,16 +326,17 @@ func TestHandleExecutePurchase_SaveError(t *testing.T) {
Recommendations: []config.RecommendationRecord{rec},
}

mockStore.On("GetGlobalConfig", ctx).Return(autoPurchaseOnCfg(), nil)
mockStore.On("GetGlobalConfig", ctx).Return(autoPurchaseOnCfg(), nil).Once()
mockStore.On("GetGlobalConfig", mock.Anything).Return(autoPurchaseOnCfg(), nil)
mockStore.On("GetExecutionByID", ctx, "exec-save-err").Return(exec, nil)
// claimAndExecute CASes the row to "running" before executing (issue #1013).
runningExec := *exec
runningExec.Status = "running"
mockStore.On("TransitionExecutionStatus", ctx, "exec-save-err",
[]string{"approved", "pending", "notified"}, "running", (*string)(nil)).Return(&runningExec, nil)
mockStore.On("GetPurchasePlan", ctx, "plan-save-err").Return(plan, nil)
mockStore.On("SavePurchaseExecution", ctx, mock.AnythingOfType("*config.PurchaseExecution")).Return(errors.New("save failed"))
mockSTS.On("GetCallerIdentity", ctx, mock.Anything).Return(nil, errors.New("sts error"))
mockStore.On("GetPurchasePlan", mock.Anything, "plan-save-err").Return(plan, nil)
mockStore.On("SavePurchaseExecution", mock.Anything, mock.AnythingOfType("*config.PurchaseExecution")).Return(errors.New("save failed"))
mockSTS.On("GetCallerIdentity", mock.Anything, mock.Anything).Return(nil, errors.New("sts error"))

// Provider factory returns error → purchase fails
mockFactory.On("CreateAndValidateProvider", mock.Anything, "aws", mock.Anything).Return(nil, errors.New("provider error"))
Expand Down Expand Up @@ -409,10 +410,10 @@ func TestProcessMessage_ApproveHappyPath(t *testing.T) {
// regression that drops it (e.g. passes "" or exec.ExecutionID by
// mistake) would mismatch the mock expectation and fail the test.
plan := &config.PurchasePlan{ID: planID, Name: "test-plan"}
mockStore.On("GetPurchasePlan", ctx, planID).Return(plan, nil)
mockEmail.On("SendPurchaseConfirmation", ctx, mock.Anything).Return(nil)
mockStore.On("SavePurchaseExecution", ctx, mock.AnythingOfType("*config.PurchaseExecution")).Return(nil)
mockStore.On("CompletePlanStep", ctx, planID, 1).Return(nil)
mockStore.On("GetPurchasePlan", mock.Anything, planID).Return(plan, nil)
mockEmail.On("SendPurchaseConfirmation", mock.Anything, mock.Anything).Return(nil)
mockStore.On("SavePurchaseExecution", mock.Anything, mock.AnythingOfType("*config.PurchaseExecution")).Return(nil)
mockStore.On("CompletePlanStep", mock.Anything, planID, 1).Return(nil)

manager := &Manager{
config: mockStore,
Expand Down Expand Up @@ -514,14 +515,14 @@ func TestProcessMessage_ApproveFourEyesOn_DifferentApproverSucceeds(t *testing.T
// row TransitionExecutionStatus returns) in place.
mockStore.On("GetExecutionByID", ctx, "exec-appv-diff").Return(exec, nil).Times(3)
mockStore.On("GetCloudAccount", ctx, accountID).Return(account, nil)
mockStore.On("GetGlobalConfig", ctx).Return(fourEyesCfgOnForManager(), nil)
mockStore.On("GetGlobalConfig", mock.Anything).Return(fourEyesCfgOnForManager(), nil)
mockStore.On("GetUserEmailByID", ctx, creatorID).Return("owner@example.com", nil)
mockStore.On("TransitionExecutionStatus", ctx, "exec-appv-diff", []string{"pending", "notified"}, "running", (*string)(nil)).Return(approved, nil)
plan := &config.PurchasePlan{ID: planID, Name: "test-plan"}
mockStore.On("GetPurchasePlan", ctx, planID).Return(plan, nil)
mockEmail.On("SendPurchaseConfirmation", ctx, mock.Anything).Return(nil)
mockStore.On("SavePurchaseExecution", ctx, mock.AnythingOfType("*config.PurchaseExecution")).Return(nil)
mockStore.On("CompletePlanStep", ctx, planID, 1).Return(nil)
mockStore.On("GetPurchasePlan", mock.Anything, planID).Return(plan, nil)
mockEmail.On("SendPurchaseConfirmation", mock.Anything, mock.Anything).Return(nil)
mockStore.On("SavePurchaseExecution", mock.Anything, mock.AnythingOfType("*config.PurchaseExecution")).Return(nil)
mockStore.On("CompletePlanStep", mock.Anything, planID, 1).Return(nil)

manager := &Manager{
config: mockStore,
Expand Down
174 changes: 174 additions & 0 deletions internal/purchase/execute_detached_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
package purchase

import (
"context"
"errors"
"sync/atomic"
"testing"
"time"

"github.com/LeanerCloud/cloud-commitments-go/pkg/common"
"github.com/LeanerCloud/cloud-commitments-platform/internal/config"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/sts"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)

// detachFixture wires a direct single-account AWS purchase (issue #706).
type detachFixture struct {
manager *Manager
exec *config.PurchaseExecution
svc *MockServiceClient
saved []string
history int
saveCtx context.Context
saveRemaining time.Duration
histCtx context.Context
historyRemaining time.Duration
}

func newDetachFixture(t *testing.T) *detachFixture {
t.Helper()
f := &detachFixture{svc: new(MockServiceClient)}
store := new(MockConfigStore)
factory := new(MockProviderFactory)
prov := new(MockProvider)
stsMock := new(MockSTSClient)
f.exec = &config.PurchaseExecution{
ExecutionID: "exec-detach", StepNumber: 1, Status: "running",
Recommendations: []config.RecommendationRecord{
{Provider: "aws", Service: "ec2", ResourceType: "m5.large", Region: "us-east-1", Count: 1, UpfrontCost: 500, Selected: true},
},
}
store.SavePurchaseExecutionFn = func(ctx context.Context, e *config.PurchaseExecution) error {
f.saveCtx = ctx
if deadline, ok := ctx.Deadline(); ok {
f.saveRemaining = time.Until(deadline)
}
if err := ctx.Err(); err != nil {
return err
}
f.saved = append(f.saved, e.Status)
return nil
}
store.On("SavePurchaseHistory", mock.Anything, mock.Anything).Run(func(a mock.Arguments) {
f.history++
f.histCtx = a.Get(0).(context.Context)
if deadline, ok := f.histCtx.Deadline(); ok {
f.historyRemaining = time.Until(deadline)
}
}).Return(nil)
email := new(MockEmailSender)
email.On("SendPurchaseConfirmation", mock.Anything, mock.Anything).Return(nil)
stsMock.On("GetCallerIdentity", mock.Anything, mock.Anything).Return(&stsOut, nil)
factory.On("CreateAndValidateProvider", mock.Anything, "aws", mock.Anything).Return(prov, nil)
prov.On("GetServiceClient", mock.Anything, common.ServiceEC2, "us-east-1").Return(f.svc, nil)
f.manager = &Manager{config: store, email: email, stsClient: stsMock, providerFactory: factory, dashboardURL: "https://d.example.com"}
return f
}

func TestExecuteAndFinalize_SlowRunGetsFreshTerminalSaveBudget(t *testing.T) {
t.Setenv("CUDLY_MAX_ACCOUNT_PARALLELISM", "1")
f := newDetachFixture(t)
f.exec.Recommendations = append(f.exec.Recommendations, f.exec.Recommendations[0])
f.exec.Recommendations[1].ResourceType = "m5.xlarge"
var active atomic.Int32
var overlapped atomic.Bool
var providerCanceled atomic.Bool
f.svc.On("PurchaseCommitment", mock.Anything, mock.Anything, mock.Anything).
Run(func(args mock.Arguments) {
if active.Add(1) != 1 {
overlapped.Store(true)
}
defer active.Add(-1)
ctx := args.Get(0).(context.Context)
timer := time.NewTimer(16 * time.Second)
defer timer.Stop()
select {
case <-timer.C:
case <-ctx.Done():
providerCanceled.Store(true)
}
}).Return(common.PurchaseResult{Success: true, CommitmentID: "ri-slow"}, nil).Twice()
started := time.Now()
err := f.manager.executeAndFinalize(context.Background(), f.exec)
assert.Greater(t, time.Since(started), 30*time.Second)
assert.False(t, overlapped.Load())
assert.False(t, providerCanceled.Load(), "provider call must fit its own budget")
f.svc.AssertNumberOfCalls(t, "PurchaseCommitment", 2)
assert.Equal(t, 2, f.history)
require.NoError(t, err)
assert.Equal(t, []string{"completed"}, f.saved)
assert.Greater(t, f.saveRemaining, 29*time.Second)
assert.LessOrEqual(t, f.saveRemaining, 30*time.Second)
}

var stsOut = sts.GetCallerIdentityOutput{Account: aws.String("123456789012")}

// A committed purchase must remain auditable after request cancellation.
func TestExecuteAndFinalize_ClientDisconnectMidPurchase_StillRecorded(t *testing.T) {
f := newDetachFixture(t)
ctx, cancel := context.WithCancel(context.Background())
f.svc.On("PurchaseCommitment", mock.Anything, mock.Anything, mock.Anything).
Run(func(mock.Arguments) { cancel() }).
Return(common.PurchaseResult{Success: true, CommitmentID: "ri-1"}, nil).Once()

err := f.manager.executeAndFinalize(ctx, f.exec)

require.NoError(t, err)
f.svc.AssertNumberOfCalls(t, "PurchaseCommitment", 1)
assert.Equal(t, []string{"completed"}, f.saved)
assert.Equal(t, 1, f.history)
}

// Test B: a caller ctx whose deadline already passed (30s request timeout)
// must not reach the provider or the save; both get their own bounded budget.
func TestExecuteAndFinalize_ExpiredCallerDeadline_UsesOwnBudgets(t *testing.T) {
f := newDetachFixture(t)
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
defer cancel()
var provDeadline time.Time
var provContextErr error
f.svc.On("PurchaseCommitment", mock.Anything, mock.Anything, mock.Anything).
Run(func(a mock.Arguments) {
c := a.Get(0).(context.Context)
provContextErr = c.Err()
provDeadline, _ = c.Deadline()
}).
Return(common.PurchaseResult{Success: true, CommitmentID: "ri-2"}, nil).Once()

require.NoError(t, f.manager.executeAndFinalize(ctx, f.exec))
require.NoError(t, provContextErr)

// per-rec cap (30s) still bounds the provider call inside the 4m run budget.
assert.WithinDuration(t, time.Now(), provDeadline, 30*time.Second)
_, ok := f.saveCtx.Deadline()
require.True(t, ok, "terminal save needs a deadline")
// Literal durations on purpose: comparing against the constants would
// pass whatever their values are.
assert.LessOrEqual(t, f.saveRemaining, 30*time.Second)
assert.Greater(t, f.saveRemaining, 29*time.Second)
// SavePurchaseHistory runs inside executePurchase, so it sees the run ctx.
_, ok = f.histCtx.Deadline()
require.True(t, ok, "run ctx needs a deadline")
assert.LessOrEqual(t, f.historyRemaining, 4*time.Minute)
assert.Greater(t, f.historyRemaining, 3*time.Minute+59*time.Second)
assert.Equal(t, []string{"completed"}, f.saved)
}

// Test C: a failed purchase with a canceled caller still persists "failed".
func TestExecuteAndFinalize_ProviderErrorWithCanceledCaller_PersistsFailed(t *testing.T) {
f := newDetachFixture(t)
ctx, cancel := context.WithCancel(context.Background())
f.svc.On("PurchaseCommitment", mock.Anything, mock.Anything, mock.Anything).
Run(func(mock.Arguments) { cancel() }).
Return(common.PurchaseResult{}, errors.New("provider rejected")).Once()

err := f.manager.executeAndFinalize(ctx, f.exec)

require.Error(t, err)
assert.NotErrorIs(t, err, config.ErrAuditLoss)
assert.Equal(t, []string{"failed"}, f.saved)
}
Loading
Loading