From 514b608c4c15813dbcbdcbba1ceec45556fa7817 Mon Sep 17 00:00:00 2001 From: creegon2 <315603738+creegon2@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:49:57 -0700 Subject: [PATCH 01/11] feat(core): add residual reclamation ledger states and evidence Add forward-only reclaim responsibility, evidence, target and disposition columns with constraints and scan indexes. Preserve legacy responsibility and fail clearly on historical cleared_at values. Reserve migration 0051 after upstream main 0050; update both migration READMEs and add normal migration-helper tests. --- ...ent_run_residual_reclaim_migration_test.go | 221 ++++++++++++++++++ .../0051_agent_run_residual_reclamation.sql | 71 ++++++ internal/core/migrations/README.en.md | 5 +- internal/core/migrations/README.md | 5 +- 4 files changed, 300 insertions(+), 2 deletions(-) create mode 100644 integration/agent_run_residual_reclaim_migration_test.go create mode 100644 internal/core/migrations/0051_agent_run_residual_reclamation.sql diff --git a/integration/agent_run_residual_reclaim_migration_test.go b/integration/agent_run_residual_reclaim_migration_test.go new file mode 100644 index 00000000..70ac4331 --- /dev/null +++ b/integration/agent_run_residual_reclaim_migration_test.go @@ -0,0 +1,221 @@ +package integration + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" +) + +const residualReclaimMigration = "0051_agent_run_residual_reclamation.sql" + +func seedResidualMigrationRow(t *testing.T, pool *sql.DB, seed skeletonSeed, reason string) { + t.Helper() + _, err := pool.Exec(`INSERT INTO agent_run_residuals + (run_id,tenant_id,project_id,workspace_id,runtime_generation,responsibility,reason,session_ended_at) + VALUES($1,$2,$3,$4,27,'{"lateFacts":{"original":{"state":"ended"}},"original":"keep"}',$5,'2026-01-02T03:04:05Z')`, + seed.runID, seed.tenantID, seed.projectID, seed.runWorkspaceID, reason) + must(t, err) +} + +// Compare the complete old ledger and business rows, rather than only the newly added defaults. +func residualMigrationHistory(t *testing.T, pool *sql.DB) string { + t.Helper() + var snapshot string + must(t, pool.QueryRow(`SELECT jsonb_build_object( + 'residuals',(SELECT jsonb_agg(to_jsonb(r)-ARRAY['reclaim_state','disposition','reclaim_id','reclaim_mode', + 'reclaim_operation_id','reclaim_authorization','reclaim_targets','reclaim_evidence','reclaim_version', + 'reclaim_checked_at','reclaim_conflicted_at'] ORDER BY run_id) FROM agent_run_residuals r), + 'runs',(SELECT jsonb_agg(to_jsonb(r) ORDER BY id) FROM issue_runs r), + 'workspaces',(SELECT jsonb_agg(to_jsonb(w) ORDER BY id) FROM workspaces w), + 'operations',(SELECT jsonb_agg(to_jsonb(o) ORDER BY id) FROM operations o))::text`).Scan(&snapshot)) + return snapshot +} + +func TestMigration0051ResidualReclamationFreshAndConstraints(t *testing.T) { + pool, _ := testSchema(t, "test_residual50f_") + must(t, newStoreOnSchema(t, pool).Migrate(context.Background())) + seed := seedAgentIssueRunSkeleton(t, pool) + seedResidualMigrationRow(t, pool, seed, "agent_start_failed") + var columns int + must(t, pool.QueryRow(`SELECT count(*) FROM information_schema.columns + WHERE table_schema=current_schema() AND table_name='agent_run_residuals' + AND column_name IN ('reclaim_state','disposition','reclaim_id','reclaim_mode','reclaim_operation_id', + 'reclaim_authorization','reclaim_targets','reclaim_evidence','reclaim_version','reclaim_checked_at','reclaim_conflicted_at')`).Scan(&columns)) + if columns != 11 { + t.Fatalf("fresh schema has %d reclamation columns, want 11", columns) + } + var scanDefinition, scanPredicate string + must(t, pool.QueryRow(`SELECT pg_get_indexdef(indexrelid),pg_get_expr(indpred,indrelid) + FROM pg_index WHERE indexrelid='agent_run_residuals_reclaim_scan'::regclass`).Scan(&scanDefinition, &scanPredicate)) + if !strings.Contains(scanDefinition, "(reclaim_checked_at NULLS FIRST, created_at, run_id)") || + !strings.Contains(scanPredicate, "reconciling") || !strings.Contains(scanPredicate, "reclaiming") || strings.Contains(scanPredicate, "cleared") { + t.Fatalf("residual scan must order pending rows fairly: %s WHERE %s", scanDefinition, scanPredicate) + } + var initialized bool + must(t, pool.QueryRow(`SELECT reclaim_state='reconciling' AND disposition IS NULL AND reclaim_id IS NULL + AND reclaim_mode IS NULL AND reclaim_operation_id IS NULL AND reclaim_authorization='{}'::jsonb + AND reclaim_targets='[]'::jsonb AND reclaim_evidence='[]'::jsonb AND reclaim_version=0 + AND reclaim_checked_at IS NULL AND reclaim_conflicted_at IS NULL AND cleared_at IS NULL + FROM agent_run_residuals WHERE run_id=$1`, seed.runID).Scan(&initialized)) + if !initialized { + t.Fatal("fresh residual did not receive empty reconciling defaults") + } + op := uuid.NewString() + _, err := pool.Exec(`INSERT INTO operations(id,tenant_id,actor_user_id,project_id,workspace_id,kind,state,step,request,idempotency_key,request_hash) + VALUES($1,$2,$3,$4,$5,'delete_workspace','succeeded','done','{}','residual-migration','fixture-request')`, + op, seed.tenantID, seed.userID, seed.projectID, seed.runWorkspaceID) + must(t, err) + claim := "reclaim_state='reclaiming',reclaim_id='" + uuid.NewString() + "',reclaim_mode='owned_delete',reclaim_authorization='{\"source\":\"test\"}',reclaim_operation_id='" + op + "'" + for _, tc := range []struct { + name, set string + }{ + {"unknown state", "reclaim_state='unknown'"}, + {"null state", "reclaim_state=NULL"}, + {"unknown disposition", "disposition='unknown'"}, + {"unknown mode", "reclaim_mode='unknown'"}, + {"authorization array", "reclaim_authorization='[]'"}, + {"authorization null", "reclaim_authorization=NULL"}, + {"targets object", "reclaim_targets='{}'"}, + {"targets null", "reclaim_targets=NULL"}, + {"evidence object", "reclaim_evidence='{}'"}, + {"evidence null", "reclaim_evidence=NULL"}, + {"negative version", "reclaim_version=-1"}, + {"null version", "reclaim_version=NULL"}, + {"reconciling id", "reclaim_id='" + uuid.NewString() + "'"}, + {"reconciling mode", "reclaim_mode='controlled'"}, + {"reconciling operation", "reclaim_operation_id='" + op + "'"}, + {"reconciling authorization", "reclaim_authorization='{\"source\":\"test\"}'"}, + {"missing id", "reclaim_state='reclaiming',reclaim_mode='controlled',reclaim_authorization='{\"source\":\"test\"}'"}, + {"missing mode", "reclaim_state='reclaiming',reclaim_id='" + uuid.NewString() + "',reclaim_authorization='{\"source\":\"test\"}'"}, + {"empty authorization", "reclaim_state='reclaiming',reclaim_id='" + uuid.NewString() + "',reclaim_mode='controlled'"}, + {"owned delete without operation", "reclaim_state='reclaiming',reclaim_id='" + uuid.NewString() + "',reclaim_mode='owned_delete',reclaim_authorization='{\"source\":\"test\"}'"}, + {"cleared missing timestamp", strings.Replace(claim, "reclaiming", "cleared", 1) + ",disposition='released'"}, + {"cleared missing disposition", strings.Replace(claim, "reclaiming", "cleared", 1) + ",cleared_at=now()"}, + {"reconciling timestamp", "cleared_at=now()"}, + {"reconciling disposition", "disposition='released'"}, + {"reclaiming timestamp", claim + ",cleared_at=now()"}, + {"reclaiming disposition", claim + ",disposition='released'"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := pool.Exec("UPDATE agent_run_residuals SET "+tc.set+" WHERE run_id=$1", seed.runID) + code := "23514" + if strings.Contains(tc.name, "null") { + code = "23502" + } + wantPGError(t, err, code) + }) + } + _, err = pool.Exec("UPDATE agent_run_residuals SET "+strings.Replace(claim, op, uuid.NewString(), 1)+" WHERE run_id=$1", seed.runID) + wantPGError(t, err, "23503") + _, err = pool.Exec("UPDATE agent_run_residuals SET "+claim+" WHERE run_id=$1", seed.runID) + must(t, err) + other := seed + other.runID, other.runWorkspaceID = newBoundRun(t, pool, seed) + seedResidualMigrationRow(t, pool, other, "workspace_unavailable") + _, err = pool.Exec(`UPDATE agent_run_residuals SET reclaim_state='reclaiming',reclaim_mode='controlled', + reclaim_authorization='{"source":"test"}',reclaim_id=(SELECT reclaim_id FROM agent_run_residuals WHERE run_id=$1) + WHERE run_id=$2`, seed.runID, other.runID) + wantPGError(t, err, "23505") + _, err = pool.Exec(`UPDATE agent_run_residuals SET reclaim_state='reclaiming',reclaim_mode='controlled', + reclaim_authorization='{"source":"test"}',reclaim_id=$1,reclaim_operation_id=$2 WHERE run_id=$3`, uuid.NewString(), op, other.runID) + wantPGError(t, err, "23505") + // Controlled responsibility may omit an operation; all three explicit dispositions are legal. + _, err = pool.Exec(`UPDATE agent_run_residuals SET reclaim_state='reclaiming',reclaim_mode='controlled', + reclaim_authorization='{"source":"test"}',reclaim_id=$1 WHERE run_id=$2`, uuid.NewString(), other.runID) + must(t, err) + for _, disposition := range []string{"released", "terminated_data_retained", "externally_confirmed"} { + _, err = pool.Exec(`UPDATE agent_run_residuals SET reclaim_state='cleared',cleared_at=now(),disposition=$1 WHERE run_id=$2`, disposition, other.runID) + must(t, err) + } +} + +func TestMigration0051ResidualReclamationUpgradePreservesHistory(t *testing.T) { + pool, _ := testSchema(t, "test_residual50u_") + applyMigrationsUpTo(t, pool, migrationsBefore(t, residualReclaimMigration)) + seed := seedAgentIssueRunSkeleton(t, pool) + seedResidualMigrationRow(t, pool, seed, "agent_start_failed") + other := seed + other.runID, other.runWorkspaceID = newBoundRun(t, pool, seed) + seedResidualMigrationRow(t, pool, other, "workspace_unavailable") + before := residualMigrationHistory(t, pool) + must(t, newStoreOnSchema(t, pool).Migrate(context.Background())) + if after := residualMigrationHistory(t, pool); before != after { + t.Fatal("reclamation upgrade rewrote legacy residual or business facts") + } + var count int + must(t, pool.QueryRow(`SELECT count(*) FROM agent_run_residuals WHERE reclaim_state='reconciling' + AND reclaim_authorization='{}'::jsonb AND reclaim_targets='[]'::jsonb AND reclaim_evidence='[]'::jsonb + AND reclaim_version=0 AND reclaim_id IS NULL AND reclaim_mode IS NULL AND reclaim_operation_id IS NULL + AND disposition IS NULL AND reclaim_checked_at IS NULL AND reclaim_conflicted_at IS NULL`).Scan(&count)) + if count != 2 { + t.Fatalf("upgrade initialized %d legacy residuals, want both D4 and D5 without invented evidence", count) + } +} + +func TestMigration0051ResidualReclamationReplayPreservesProgress(t *testing.T) { + pool, _ := testSchema(t, "test_residual50r_") + store := newStoreOnSchema(t, pool) + must(t, store.Migrate(context.Background())) + base := seedAgentIssueRunSkeleton(t, pool) + for _, state := range []string{"reconciling", "reclaiming", "cleared"} { + seed := base + if state != "reconciling" { + seed.runID, seed.runWorkspaceID = newBoundRun(t, pool, base) + } + seedResidualMigrationRow(t, pool, seed, "agent_start_failed") + if state != "reconciling" { + _, err := pool.Exec(`UPDATE agent_run_residuals SET reclaim_state=$1,reclaim_id=$2,reclaim_mode='controlled', + reclaim_authorization='{"source":"test"}',reclaim_targets='[{"sandboxId":"original"}]', + reclaim_evidence='[{"eventKey":"original"}]',reclaim_version=7,reclaim_checked_at=now(),reclaim_conflicted_at=now(), + cleared_at=CASE WHEN $1='cleared' THEN now() END,disposition=CASE WHEN $1='cleared' THEN 'released' END + WHERE run_id=$3`, state, uuid.NewString(), seed.runID) + must(t, err) + } + } + var before, after string + query := "SELECT jsonb_agg(to_jsonb(r) ORDER BY run_id)::text FROM agent_run_residuals r" + must(t, pool.QueryRow(query).Scan(&before)) + // Exercise both runner replay and direct reentrant DDL on already progressed rows. + must(t, store.Migrate(context.Background())) + ddl, err := os.ReadFile(filepath.Join("..", "internal", "core", "migrations", residualReclaimMigration)) + must(t, err) + tx, err := pool.Begin() + must(t, err) + defer func() { _ = tx.Rollback() }() + _, err = tx.Exec(string(ddl)) + must(t, err) + must(t, tx.Commit()) + must(t, pool.QueryRow(query).Scan(&after)) + if before != after { + t.Fatal("migration replay reset reclamation progress or evidence") + } +} + +func TestMigration0051ResidualReclamationRejectsLegacyClearedRows(t *testing.T) { + pool, _ := testSchema(t, "test_residual50a2_") + applyMigrationsUpTo(t, pool, migrationsBefore(t, residualReclaimMigration)) + seed := seedAgentIssueRunSkeleton(t, pool) + seedResidualMigrationRow(t, pool, seed, "agent_start_failed") + _, err := pool.Exec("UPDATE agent_run_residuals SET cleared_at='2026-01-03T03:04:05Z' WHERE run_id=$1", seed.runID) + must(t, err) + before := residualMigrationHistory(t, pool) + err = newStoreOnSchema(t, pool).Migrate(context.Background()) + if err == nil || !strings.Contains(err.Error(), "non-null cleared_at") { + t.Fatalf("legacy cleared row must produce the explicit A2 compatibility error, got %v", err) + } + if after := residualMigrationHistory(t, pool); before != after { + t.Fatal("failed A2 migration changed original data") + } + var columns, recorded int + must(t, pool.QueryRow(`SELECT count(*) FROM information_schema.columns + WHERE table_schema=current_schema() AND table_name='agent_run_residuals' AND column_name='reclaim_state'`).Scan(&columns)) + must(t, pool.QueryRow("SELECT count(*) FROM schema_migrations WHERE version=$1", residualReclaimMigration).Scan(&recorded)) + if columns != 0 || recorded != 0 { + t.Fatalf("failed A2 migration left schema/data state: columns=%d recorded=%d", columns, recorded) + } +} diff --git a/internal/core/migrations/0051_agent_run_residual_reclamation.sql b/internal/core/migrations/0051_agent_run_residual_reclamation.sql new file mode 100644 index 00000000..a01605e3 --- /dev/null +++ b/internal/core/migrations/0051_agent_run_residual_reclamation.sql @@ -0,0 +1,71 @@ +-- The residual row remains the sole reclamation authority. New defaults initialize legacy D4/D5 +-- rows without rewriting their reason, responsibility, timestamps, or business settlement. +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid='agent_run_residuals'::regclass AND attname='reclaim_state' AND NOT attisdropped + ) AND EXISTS (SELECT 1 FROM agent_run_residuals WHERE cleared_at IS NOT NULL) THEN + RAISE EXCEPTION 'residual reclamation migration cannot initialize legacy rows with non-null cleared_at; explicit compatibility handling is required (A2)'; + END IF; +END $$; + +ALTER TABLE agent_run_residuals + ADD COLUMN IF NOT EXISTS reclaim_state text NOT NULL DEFAULT 'reconciling', + ADD COLUMN IF NOT EXISTS disposition text, + ADD COLUMN IF NOT EXISTS reclaim_id uuid, + ADD COLUMN IF NOT EXISTS reclaim_mode text, + ADD COLUMN IF NOT EXISTS reclaim_operation_id uuid, + ADD COLUMN IF NOT EXISTS reclaim_authorization jsonb NOT NULL DEFAULT '{}', + ADD COLUMN IF NOT EXISTS reclaim_targets jsonb NOT NULL DEFAULT '[]', + ADD COLUMN IF NOT EXISTS reclaim_evidence jsonb NOT NULL DEFAULT '[]', + ADD COLUMN IF NOT EXISTS reclaim_version bigint NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS reclaim_checked_at timestamptz, + ADD COLUMN IF NOT EXISTS reclaim_conflicted_at timestamptz; + +-- Named constraints make direct DDL replay safe too. They describe valid row shapes, not the +-- history of state transitions; conditional writes in the owning transaction enforce that history. +DO $reclaim_constraints$ +DECLARE + item record; +BEGIN + FOR item IN SELECT * FROM (VALUES + ('agent_run_residuals_reclaim_state_check', + $$CHECK (reclaim_state IN ('reconciling','reclaiming','cleared'))$$), + ('agent_run_residuals_disposition_check', + $$CHECK (disposition IS NULL OR disposition IN ('released','terminated_data_retained','externally_confirmed'))$$), + ('agent_run_residuals_reclaim_mode_check', + $$CHECK (reclaim_mode IS NULL OR reclaim_mode IN ('owned_delete','controlled'))$$), + ('agent_run_residuals_reclaim_json_check', + $$CHECK (jsonb_typeof(reclaim_authorization)='object' + AND jsonb_typeof(reclaim_targets)='array' AND jsonb_typeof(reclaim_evidence)='array')$$), + ('agent_run_residuals_reclaim_version_check', $$CHECK (reclaim_version>=0)$$), + ('agent_run_residuals_reclaim_responsibility_check', + $$CHECK ( + (reclaim_state='reconciling' AND reclaim_id IS NULL AND reclaim_mode IS NULL + AND reclaim_operation_id IS NULL AND reclaim_authorization='{}'::jsonb) + OR (reclaim_state IN ('reclaiming','cleared') AND reclaim_id IS NOT NULL + AND reclaim_mode IS NOT NULL AND reclaim_authorization<>'{}'::jsonb + AND (reclaim_mode<>'owned_delete' OR reclaim_operation_id IS NOT NULL)))$$), + ('agent_run_residuals_reclaim_completion_check', + $$CHECK ( + (reclaim_state='cleared' AND cleared_at IS NOT NULL AND disposition IS NOT NULL) + OR (reclaim_state<>'cleared' AND cleared_at IS NULL AND disposition IS NULL))$$), + ('agent_run_residuals_reclaim_operation_fk', + $$FOREIGN KEY (reclaim_operation_id) REFERENCES operations(id)$$) + ) AS definitions(name, definition) + LOOP + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid='agent_run_residuals'::regclass AND conname=item.name) THEN + EXECUTE format('ALTER TABLE agent_run_residuals ADD CONSTRAINT %I %s', item.name, item.definition); + END IF; + END LOOP; +END $reclaim_constraints$; + +CREATE UNIQUE INDEX IF NOT EXISTS agent_run_residuals_reclaim_id_unique + ON agent_run_residuals(reclaim_id) WHERE reclaim_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS agent_run_residuals_reclaim_operation_unique + ON agent_run_residuals(reclaim_operation_id) WHERE reclaim_operation_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS agent_run_residuals_reclaim_scan + ON agent_run_residuals(reclaim_checked_at NULLS FIRST, created_at, run_id) + WHERE reclaim_state IN ('reconciling','reclaiming'); diff --git a/internal/core/migrations/README.en.md b/internal/core/migrations/README.en.md index 784b4580..8e914c21 100644 --- a/internal/core/migrations/README.en.md +++ b/internal/core/migrations/README.en.md @@ -6,7 +6,7 @@ SQL is embedded in the binary through `embed.FS` in `core/store.go`. `Store.Migr ## File index -There are currently 51 SQL files. +There are currently 52 SQL files. | SQL file | Schema change | |---|---| @@ -61,6 +61,7 @@ There are currently 51 SQL files. | [0047_thread_preview_positions.sql](0047_thread_preview_positions.sql) | Node history positions for settled Thread entries | | [0048_task_conversation_manual_pause.sql](0048_task_conversation_manual_pause.sql) | Manual pause for task conversations | | [0049_thread_commands_workflow_run.sql](0049_thread_commands_workflow_run.sql) | WorkflowRun session-command ownership and node scope | +| [0051_agent_run_residual_reclamation.sql](0051_agent_run_residual_reclamation.sql) | Residual reclamation responsibility, appended evidence and disposition | ## Immutability and upgrades @@ -74,4 +75,6 @@ There are currently 51 SQL files. - 0049 lets `thread_commands` accept a WorkflowRun owner and requires a graph node on workflow commands; each node holds at most one `end_session`, so the scheduler can replay it safely. Every old row carries `run_id` and is not backfilled. Semantics are in [workflows](../../../docs/workflows/workflows.en.md); coverage is in the `TestWorkflowSessionEndsAfterSuccessfulTurn` DB tests. - New migrations must verify fresh databases, upgrades from the preceding schema, repeated application and key constraints. See [migration_upgrade_path_test.go](../../../integration/migration_upgrade_path_test.go) and domain upgrade tests in integration. +0051 only adds residual-ledger columns: defaults initialize existing D4/D5 rows as `reconciling`, without backfilling targets or evidence, rewriting old `reason`, `responsibility` or business settlement, or backfilling D8. Legacy rows with non-null `cleared_at` cause an explicit migration error and rollback pending compatibility handling (A2). Reentrant DDL preserves progressed rows. The schema accepts `released`, `terminated_data_retained` and `externally_confirmed`; this step's automatic path implements only `released`, leaving the other dispositions to later controlled entry points. The authorization transaction checks the operation's tenant and Workspace relationships; a UUID foreign key alone does not establish resource scope. See [agent_run_residual_reclaim_migration_test.go](../../../integration/agent_run_residual_reclaim_migration_test.go) for fresh, upgrade, DDL replay and constraint coverage. + See [Persistence](../../../docs/core/persistence.en.md) for ownership/transactions, and [Workflows](../../../docs/workflows/workflows.en.md) / [External Git line](../../../docs/repositories/external-git.en.md) for runtime semantics. diff --git a/internal/core/migrations/README.md b/internal/core/migrations/README.md index 822be9ca..3a6a49fa 100644 --- a/internal/core/migrations/README.md +++ b/internal/core/migrations/README.md @@ -6,7 +6,7 @@ SQL 通过 `core/store.go` 的 `embed.FS` 编入二进制。`Store.Migrate` 在 ## 文件索引 -当前共 51 个 SQL 文件。 +当前共 52 个 SQL 文件。 | SQL 文件 | schema 变化 | |---|---| @@ -61,6 +61,7 @@ SQL 通过 `core/store.go` 的 `embed.FS` 编入二进制。`Store.Migrate` 在 | [0047_thread_preview_positions.sql](0047_thread_preview_positions.sql) | 定型 Thread 条目的 Node 历史位置 | | [0048_task_conversation_manual_pause.sql](0048_task_conversation_manual_pause.sql) | 任务对话的手动暂停 | | [0049_thread_commands_workflow_run.sql](0049_thread_commands_workflow_run.sql) | WorkflowRun 的会话命令归属与节点范围 | +| [0051_agent_run_residual_reclamation.sql](0051_agent_run_residual_reclamation.sql) | 残留回收责任、追加证据和处置状态 | ## 不可变性与升级 @@ -74,4 +75,6 @@ SQL 通过 `core/store.go` 的 `embed.FS` 编入二进制。`Store.Migrate` 在 - 0049 让 `thread_commands` 接受 WorkflowRun 归属,并要求工作流命令带图节点;每个节点至多一条 `end_session`,调度器可安全重放。旧行全部带 `run_id`,不回填。语义见 [工作流](../../../docs/workflows/workflows.md),覆盖见 `TestWorkflowSessionEndsAfterSuccessfulTurn` 等 DB 测试。 - 新增迁移须验证空库、前一 schema 升级、重复应用与关键约束。覆盖见 [migration_upgrade_path_test.go](../../../integration/migration_upgrade_path_test.go) 和 integration 的领域升级测试。 +0051 只给残留账本增列:存量 D4/D5 行通过默认值初始化为 `reconciling`,不回填目标或证据,不改旧 `reason`、`responsibility` 或业务结算,也不回填 D8。历史行若已有非空 `cleared_at`,迁移明确报错并回滚,等待显式兼容处理(A2)。DDL 可重入,不会重置已推进条目。数据库接受 `released`、`terminated_data_retained`、`externally_confirmed` 三种处置;本步自动路径只实现 `released`,其余处置留给后续受控入口。operation 的租户和 Workspace 关联由授权事务比对,UUID 外键本身不证明资源范围。空库、升级、DDL 重放和约束覆盖见 [agent_run_residual_reclaim_migration_test.go](../../../integration/agent_run_residual_reclaim_migration_test.go)。 + 数据归属与事务规则见 [持久化](../../../docs/core/persistence.md),运行语义见 [工作流](../../../docs/workflows/workflows.md) 和 [外部 Git 线](../../../docs/repositories/external-git.md)。 From 3418d79deec9bea9f1ad7b2b857104f1f58f0af7 Mon Sep 17 00:00:00 2001 From: creegon2 <315603738+creegon2@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:51:04 -0700 Subject: [PATCH 02/11] feat(core): take over residual facts with durable conflict audits Append identity-bound provisioning, execution, delivery and runtime receipts without rewriting historical responsibility. Rescue original verified deliveries after business settlement and cleanup. Commit contradictions and a durable block before returning residual_fact_conflict; keep canonical facts and idempotent replays. Add real PostgreSQL and bufconn intake coverage using existing RPCs only. --- .../agent_run_provisioning_fact_test.go | 28 +- .../agent_run_residual_delivery_test.go | 240 ++++++++++++++++++ integration/agent_run_residual_facts_test.go | 129 ++++++++++ internal/core/agent_run_provisioning_fact.go | 72 +----- .../agent_run_provisioning_fact_db_test.go | 104 ++++++-- internal/core/agent_run_residual_fact.go | 215 ++++++++++++++++ internal/core/clone.go | 6 + internal/core/control.go | 36 ++- internal/core/node_execution.go | 7 + internal/core/revision.go | 16 ++ internal/core/runtime_control.go | 2 +- internal/core/runtime_control_delivery.go | 11 + internal/core/store.go | 2 + 13 files changed, 755 insertions(+), 113 deletions(-) create mode 100644 integration/agent_run_residual_delivery_test.go create mode 100644 integration/agent_run_residual_facts_test.go create mode 100644 internal/core/agent_run_residual_fact.go diff --git a/integration/agent_run_provisioning_fact_test.go b/integration/agent_run_provisioning_fact_test.go index f67c6098..148c1898 100644 --- a/integration/agent_run_provisioning_fact_test.go +++ b/integration/agent_run_provisioning_fact_test.go @@ -45,6 +45,8 @@ func TestRecordProvisioningFactReceiptsLateFactsOverGRPCWithoutResurrectingRun(t _, e = f.store.Pool.Exec(`UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '1 second' WHERE id=$1`, run) must(t, e) must(t, f.store.GiveUpBlockedWorkspaceCreationsOnce(context.Background())) + var responsibility string + must(t, f.store.Pool.QueryRow(`SELECT responsibility::text FROM agent_run_residuals WHERE run_id=$1`, run).Scan(&responsibility)) var phase, status, threadState sql.NullString must(t, f.store.Pool.QueryRow(`SELECT phase, status, thread_state FROM issue_runs WHERE id=$1`, run).Scan(&phase, &status, &threadState)) if phase.String != "done" || status.String != "failed" || threadState.Valid { @@ -90,9 +92,12 @@ func TestRecordProvisioningFactReceiptsLateFactsOverGRPCWithoutResurrectingRun(t if open := f.scalar(`SELECT count(*) FROM workspaces WHERE id=$1 AND admission_open`, workspace); open != 0 { t.Fatal("late creation success reopened admission") } - if n := f.scalar(`SELECT count(*) FROM jsonb_object_keys((SELECT responsibility->'lateFacts' FROM agent_run_residuals WHERE run_id=$1))`, run); n != 2 { + if n := f.scalar(`SELECT jsonb_array_length(reclaim_evidence) FROM agent_run_residuals WHERE run_id=$1`, run); n != 2 { t.Fatalf("ledger receipts wrong: %d", n) } + if n := f.scalar(`SELECT count(*) FROM agent_run_residuals WHERE run_id=$1 AND responsibility=$2::jsonb`, run, responsibility); n != 1 { + t.Fatal("late facts changed the original residual responsibility") + } } // The gRPC contract carries a fact's target in dedicated fields, so the submission fingerprint @@ -122,6 +127,8 @@ func TestRecordProvisioningFactBindsTargetAndProgressesOverGRPC(t *testing.T) { _, e = f.store.Pool.Exec(`UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '1 second' WHERE id=$1`, run) must(t, e) must(t, f.store.GiveUpBlockedWorkspaceCreationsOnce(context.Background())) + var responsibility string + must(t, f.store.Pool.QueryRow(`SELECT responsibility::text FROM agent_run_residuals WHERE run_id=$1`, run).Scan(&responsibility)) // A recorded creation fact... _, e = c.operations.RecordProvisioningFact(c.ctx, &controlpb.RecordProvisioningFactRequest{SubmissionId: "bound-creation", Epoch: c.epoch, OperationId: operation, Kind: controlpb.ProvisioningFactKind_PROVISIONING_FACT_KIND_CREATION, Outcome: controlpb.ProvisioningOutcome_PROVISIONING_OUTCOME_READY}) @@ -147,14 +154,17 @@ func TestRecordProvisioningFactBindsTargetAndProgressesOverGRPC(t *testing.T) { if terminal.GetEffect().GetState() != controlpb.EffectState_EFFECT_STATE_SUCCEEDED { t.Fatalf("terminal after running not receipted: %s", terminal.GetEffect().GetState()) } - // The ledger holds the terminal on top of the retained observation; nothing reopened. - var state string - var prior int - must(t, f.store.Pool.QueryRow(`SELECT responsibility->'lateFacts'->('effect:'||$2)->>'state', - jsonb_array_length(responsibility->'lateFacts'->('effect:'||$2)->'prior') - FROM agent_run_residuals WHERE run_id=$1`, run, effectID).Scan(&state, &prior)) - if state != "succeeded" || prior != 1 { - t.Fatalf("ledger receipt wrong: state=%s prior=%d", state, prior) + // Provisional and terminal observations remain independent append-only evidence entries. + var states string + var entries int + must(t, f.store.Pool.QueryRow(`SELECT string_agg(e->'fact'->>'state', ',' ORDER BY ordinal), count(*) + FROM agent_run_residuals r, jsonb_array_elements(r.reclaim_evidence) WITH ORDINALITY AS facts(e,ordinal) + WHERE r.run_id=$1 AND e->>'key'='effect:'||$2`, run, effectID).Scan(&states, &entries)) + if states != "running,succeeded" || entries != 2 { + t.Fatalf("ledger observations wrong: states=%s entries=%d", states, entries) + } + if n := f.scalar(`SELECT count(*) FROM agent_run_residuals WHERE run_id=$1 AND responsibility=$2::jsonb`, run, responsibility); n != 1 { + t.Fatal("late observations changed the original residual responsibility") } if n := f.scalar(`SELECT count(*) FROM workspaces WHERE id=$1 AND admission_open`, workspace); n != 0 { t.Fatal("late terminal reopened admission") diff --git a/integration/agent_run_residual_delivery_test.go b/integration/agent_run_residual_delivery_test.go new file mode 100644 index 00000000..30673930 --- /dev/null +++ b/integration/agent_run_residual_delivery_test.go @@ -0,0 +1,240 @@ +package integration + +import ( + "strings" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/status" + + "github.com/wanglongan587/cloud/internal/controlpb" +) + +// seedResidualTerminalIntake fixes only the test's pre-existing business/ledger state. The original +// execution, frozen work input/target and receipt sequence were registered through production +// dispatch before this fixture runs. In particular this does not claim to execute or prove the +// reclamation completion topology: the cleared branch is a fact-intake precondition only. +func seedResidualTerminalIntake(t *testing.T, f *fixture, scene liveThreadScene, state string) { + t.Helper() + reclaimID, operationID := uuid.NewString(), uuid.NewString() + tx, err := f.store.Pool.Begin() + must(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + _, err = tx.Exec(`UPDATE issue_runs SET phase='done',status='failed',thread_state='ended',idle_since=NULL, + failure_reason='agent_start_failed',completed_at=clock_timestamp(),version=version+1, + result='{"deliveryState":"failed","testBoundary":"preexisting_business_terminal"}'::jsonb WHERE id=$1`, scene.runID) + must(t, err) + _, err = tx.Exec(`UPDATE workspaces SET admission_open=false,admission_epoch=admission_epoch+1, + desired_state='deleted',observed_state=CASE WHEN $2='cleared' THEN 'deleted' ELSE 'deleting' END, + deleted_at=CASE WHEN $2='cleared' THEN clock_timestamp() ELSE NULL END,version=version+1 WHERE issue_run_id=$1`, scene.runID, state) + must(t, err) + _, err = tx.Exec(`INSERT INTO operations(id,tenant_id,actor_user_id,project_id,workspace_id,kind,state,step, + request,idempotency_key,request_hash,controller_epoch) + SELECT $2::uuid,w.tenant_id,w.creator_user_id,w.project_id,w.id,'delete_workspace', + CASE WHEN $4='cleared' THEN 'succeeded' ELSE 'running' END, + CASE WHEN $4='cleared' THEN 'done' ELSE 'quiesce' END, + jsonb_build_object('reclaimId',$3::uuid::text),'terminal-intake-'||$3::uuid::text,'terminal-intake-fixture',1 + FROM workspaces w WHERE w.issue_run_id=$1::uuid`, scene.runID, operationID, reclaimID, state) + must(t, err) + _, err = tx.Exec(`INSERT INTO agent_run_residuals(run_id,tenant_id,project_id,workspace_id,runtime_generation, + responsibility,reason,reclaim_state,reclaim_id,reclaim_mode,reclaim_operation_id,reclaim_authorization, + reclaim_targets,cleared_at,disposition) + SELECT $1::uuid,w.tenant_id,w.project_id,w.id,w.runtime_generation, + jsonb_build_object('executions',(SELECT jsonb_agg(jsonb_build_object('executionId',e.execution_id)) + FROM node_executions e WHERE e.workspace_id=w.id)), + 'agent_start_failed',$2,$3::uuid,'owned_delete',$4::uuid, + jsonb_build_object('kind','preexisting_intake_fixture','runId',$1::uuid::text,'workspaceId',w.id), + (SELECT jsonb_agg(jsonb_build_object('kind','sandbox','id',s.id,'workspaceId',w.id,'generation',s.generation)) + FROM sandbox_instances s WHERE s.workspace_id=w.id), + CASE WHEN $2='cleared' THEN clock_timestamp() ELSE NULL END, + CASE WHEN $2='cleared' THEN 'released' ELSE NULL END + FROM workspaces w WHERE w.issue_run_id=$1::uuid`, scene.runID, state, reclaimID, operationID) + must(t, err) + must(t, tx.Commit()) +} + +// residualIntakeSnapshot deliberately excludes the ledger/effect/result facts being accepted. It +// covers the whole business row and every dispatch counter that a late fact must not resurrect. +func residualIntakeSnapshot(t *testing.T, f *fixture, scene liveThreadScene) string { + t.Helper() + var snapshot string + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object( + 'run',to_jsonb(r),'workspace',jsonb_build_object('admission',w.admission_open,'desired',w.desired_state, + 'observed',w.observed_state,'deleted',w.deleted_at,'generation',w.runtime_generation,'version',w.version), + 'operations',(SELECT count(*) FROM operations WHERE workspace_id=w.id), + 'ensures',(SELECT count(*) FROM external_effects WHERE workspace_id=w.id AND kind='sandbox_ensure'), + 'sandboxes',(SELECT count(*) FROM sandbox_instances WHERE workspace_id=w.id), + 'executions',(SELECT count(*) FROM node_executions WHERE workspace_id=w.id), + 'work',(SELECT count(*) FROM execution_work WHERE run_id=r.id), + 'activities',(SELECT count(*) FROM issue_activities WHERE issue_id=r.issue_id))::text + FROM issue_runs r JOIN workspaces w ON w.id=r.workspace_id WHERE r.id=$1`, scene.runID).Scan(&snapshot)) + return snapshot +} + +func TestResidualSessionTerminalTakeoverOverGRPC(t *testing.T) { + for _, state := range []string{"reclaiming", "cleared"} { + t.Run(state, func(t *testing.T) { + f := setup(t) + scene := seedLiveThreadScene(t, f) + scene.start(t, f) + f.runningThread(t, scene) + seedResidualTerminalIntake(t, f, scene, state) + residualTerminalIntakeCase(t, f, scene, scene.executionID, 2, false) + }) + } +} + +func TestResidualVerifiedDeliveryTakeoverOverGRPC(t *testing.T) { + for _, state := range []string{"reclaiming", "cleared"} { + t.Run(state, func(t *testing.T) { + f := setup(t) + scene := deliveringScene(t, f) + execution := f.deliverRevision(t, scene) + seedResidualTerminalIntake(t, f, scene, state) + residualTerminalIntakeCase(t, f, scene, execution, 1, true) + }) + } +} + +func residualTerminalIntakeCase(t *testing.T, f *fixture, scene liveThreadScene, execution string, sequence uint64, delivery bool) { + t.Helper() + before := residualIntakeSnapshot(t, f, scene) + in := f.deliveryInputOf(execution) + result := sessionEndResult(scene, controlpb.AgentSessionEndReason_AGENT_SESSION_END_REASON_USER_ENDED, "") + conflict := sessionEndResult(scene, controlpb.AgentSessionEndReason_AGENT_SESSION_END_REASON_INTERRUPTED, "") + kind := "agent_session" + if delivery { + kind = "deliver_revision" + result = deliveredResult(scene, in, revisionFinalCommit) + conflict = deliveredResult(scene, in, strings.Repeat("2", 40)) + } + request := func(submission string, fact *controlpb.ExecutionResult) *controlpb.TakeOverNodeEventRequest { + return &controlpb.TakeOverNodeEventRequest{ + SubmissionId: submission, Epoch: 1, + OperationId: scene.runID, ExecutionId: execution, Sequence: sequence, + Result: fact, Event: []byte("original residual terminal event"), + } + } + ctx := asController("ctrl-a") + + // Naming a real execution is insufficient: its original work target must still resolve to + // the original stable sandbox. An unrelated UUID is rejected before any result or receipt. + var target string + must(t, f.store.Pool.QueryRow("SELECT target::text FROM execution_work WHERE execution_id=$1", execution).Scan(&target)) + _, err := f.store.Pool.Exec(`UPDATE execution_work SET target=jsonb_set(target,'{sandboxInstanceId}',to_jsonb($2::text)) WHERE execution_id=$1`, execution, uuid.NewString()) + must(t, err) + response, err := f.executions.TakeOverNodeEvent(ctx, request("residual-wrong-target", result)) + if err == nil || response != nil { + t.Fatalf("foreign original target was acknowledged: response=%v error=%v", response, err) + } + _, err = f.store.Pool.Exec("UPDATE execution_work SET target=$2::jsonb WHERE execution_id=$1", execution, target) + must(t, err) + if got := f.scalar(`SELECT count(*) FROM node_event_receipts WHERE execution_id=$1 AND sequence=$2`, execution, sequence); got != 0 { + t.Fatalf("invalid target created a receipt: %d", got) + } + if delivery && len(f.objects.probed()) != 0 { + t.Fatal("invalid original target reached object verification") + } + + // A successful transport response is the ACK. Force the ledger write to fail, proving the + // canonical result, event receipt and (for delivery) verified Revision cannot commit before it. + _, err = f.store.Pool.Exec(`CREATE FUNCTION reject_residual_intake_evidence() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'injected residual evidence persistence failure'; END $$; + CREATE TRIGGER reject_residual_intake_evidence BEFORE UPDATE OF reclaim_evidence ON agent_run_residuals + FOR EACH ROW EXECUTE FUNCTION reject_residual_intake_evidence()`) + must(t, err) + response, err = f.executions.TakeOverNodeEvent(ctx, request("residual-terminal", result)) + if err == nil || response != nil { + t.Fatalf("failed evidence persistence was acknowledged: response=%v error=%v", response, err) + } + var rolledBack bool + must(t, f.store.Pool.QueryRow(`SELECT e.result IS NULL + AND NOT EXISTS(SELECT 1 FROM node_event_receipts WHERE execution_id=e.execution_id AND sequence=$2) + AND NOT EXISTS(SELECT 1 FROM revision_verifications WHERE execution_id=e.execution_id) + AND NOT EXISTS(SELECT 1 FROM revisions WHERE execution_id=e.execution_id) + AND r.reclaim_evidence='[]'::jsonb + FROM node_executions e JOIN agent_run_residuals r ON r.run_id=e.operation_id + WHERE e.execution_id=$1`, execution, sequence).Scan(&rolledBack)) + if !rolledBack { + t.Fatal("failed evidence persistence left a result, receipt, Revision or evidence") + } + _, err = f.store.Pool.Exec("DROP TRIGGER reject_residual_intake_evidence ON agent_run_residuals; DROP FUNCTION reject_residual_intake_evidence()") + must(t, err) + + response, err = f.executions.TakeOverNodeEvent(ctx, request("residual-terminal", result)) + must(t, err) + if response.GetRecord().GetExecutionId() != execution || response.GetRecord().GetResult() == nil { + t.Fatalf("successful ACK lost its original execution/result: %v", response) + } + var accepted bool + must(t, f.store.Pool.QueryRow(`SELECT e.result->>'outcome'=$3 AND e.last_event_sequence=$2 + AND EXISTS(SELECT 1 FROM node_event_receipts WHERE execution_id=e.execution_id AND sequence=$2) + AND EXISTS(SELECT 1 FROM jsonb_array_elements(r.reclaim_evidence) x + WHERE x->>'key'='execution:'||e.execution_id AND x->'fact'->>'kind'=$4 + AND x->'fact'->>'state'='succeeded' AND x->>'receivedAt' IS NOT NULL) + FROM node_executions e JOIN agent_run_residuals r ON r.run_id=e.operation_id + WHERE e.execution_id=$1`, execution, sequence, map[bool]string{false: "agent_session_ended", true: "revision_delivered"}[delivery], kind).Scan(&accepted)) + if !accepted { + t.Fatal("ACK preceded durable canonical result, matching receipt, or residual evidence") + } + if got := f.receiptEvent(execution, int64(sequence)); got != "original residual terminal event" { + t.Fatalf("original event bytes changed: %q", got) + } + + if delivery { + row := f.revisionRow(scene.runID) + project, repository := f.runPlacement(scene.runID) + if row == nil || row.RunID != scene.runID || row.TenantID != scene.tenantID || row.WorkspaceID != f.runWorkspaceID(scene.runID) || row.ProjectID != project || row.RepositoryURL != repository || row.FinalCommit != revisionFinalCommit || row.HistoryKey != in.S("historyKey") || row.BundleKey != in.S("bundleKey") { + t.Fatalf("late verified delivery registered wrong original Revision: %#v", row) + } + if verdict := f.revisionVerification(execution); verdict.S("outcome") != "delivered" { + t.Fatalf("actual object verification missing: %v", verdict) + } + // Both the failed transaction's attempt and its successful retry verified the two declared + // objects via the existing real-HTTP HEAD fixture. No pre-seeded verification verdict is used. + probes := f.objects.probed() + if len(probes) != 4 || probes[0].S("key") != in.S("historyKey") || probes[1].S("key") != in.S("bundleKey") || probes[2].S("key") != in.S("historyKey") || probes[3].S("key") != in.S("bundleKey") { + t.Fatalf("wrong actual storage verification requests: %v", probes) + } + } + + var evidenceVersion int64 + var canonical string + must(t, f.store.Pool.QueryRow(`SELECT r.reclaim_version,e.result::text FROM agent_run_residuals r + JOIN node_executions e ON e.operation_id=r.run_id WHERE e.execution_id=$1`, execution).Scan(&evidenceVersion, &canonical)) + // Replays use the same persisted event identity. A different submission must also be idempotent, + // and successful delivery replay must not need to contact object storage again. + _, err = f.executions.TakeOverNodeEvent(ctx, request("residual-terminal", result)) + must(t, err) + _, err = f.executions.TakeOverNodeEvent(ctx, request("residual-terminal-replay", result)) + must(t, err) + var replayStable bool + must(t, f.store.Pool.QueryRow(`SELECT reclaim_version=$2 AND + (SELECT count(*) FROM node_event_receipts WHERE execution_id=$1 AND sequence=$3)=1 + FROM agent_run_residuals WHERE run_id=$4`, execution, evidenceVersion, sequence, scene.runID).Scan(&replayStable)) + if !replayStable { + t.Fatal("same original terminal duplicated evidence or receipt") + } + + response, err = f.executions.TakeOverNodeEvent(ctx, request("residual-terminal-conflict", conflict)) + if response != nil || status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("canonical conflict did not return committed residual conflict: response=%v error=%v", response, err) + } + var audited bool + must(t, f.store.Pool.QueryRow(`SELECT r.reclaim_conflicted_at IS NOT NULL AND e.result=$2::jsonb + AND EXISTS(SELECT 1 FROM jsonb_array_elements(r.reclaim_evidence) x + WHERE x->>'key'='execution:'||e.execution_id AND x->'fact'->>'kind'='conflict' + AND x->'fact'->'original'->'result'=$2::jsonb AND x->'fact'->'incoming'->'result'<>$2::jsonb) + AND (SELECT count(*) FROM node_event_receipts WHERE execution_id=e.execution_id AND sequence=$3)=1 + FROM node_executions e JOIN agent_run_residuals r ON r.run_id=e.operation_id + WHERE e.execution_id=$1`, execution, canonical, sequence).Scan(&audited)) + if !audited { + t.Fatal("conflict response rolled back audit or rewrote canonical result/receipt") + } + if delivery && len(f.objects.probed()) != 4 { + t.Fatal("known replay or canonical conflict repeated object verification") + } + if after := residualIntakeSnapshot(t, f, scene); after != before { + t.Fatalf("late facts changed business terminal/version, admission, resource identity or dispatch counts:\nbefore=%s\nafter=%s", before, after) + } +} diff --git a/integration/agent_run_residual_facts_test.go b/integration/agent_run_residual_facts_test.go new file mode 100644 index 00000000..5a1b375b --- /dev/null +++ b/integration/agent_run_residual_facts_test.go @@ -0,0 +1,129 @@ +package integration + +import ( + "context" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + "github.com/wanglongan587/cloud/internal/controlpb" +) + +// These scenes stage each ledger prestate to exercise the real fact entrypoint over bufconn and +// PostgreSQL. Staging reclaiming/cleared is not evidence that the recovery loop reaches them. +func TestResidualProvisioningFactsAcrossReclaimStatesOverGRPC(t *testing.T) { + for _, state := range []string{"reconciling", "reclaiming", "cleared"} { + t.Run(state, func(t *testing.T) { + f := setup(t) + f.bindBusinessHooks() + created := f.create("residual-facts-" + state) + f.drain() + run := f.insertAgentRun(t, created.O("resource").S("id")) + provisioned, err := f.store.CreateRunWorkspace(context.Background(), run) + must(t, err) + workspace, operation := provisioned.O("workspace").S("id"), provisioned.O("operation").S("id") + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET phase='provisioning',status='dispatched',workspace_id=$2,version=version+1 WHERE id=$1`, run, workspace) + must(t, err) + c := newGRPCController(t, f) + snapshot := c.claim() + if snapshot.GetOperation().GetId() != operation { + t.Fatalf("claim picked %s, want %s", snapshot.GetOperation().GetId(), operation) + } + planned, err := c.operations.PlanEffect(c.ctx, &controlpb.PlanEffectRequest{SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, Version: snapshot.GetOperation().GetVersion(), Kind: controlpb.EffectKind_EFFECT_KIND_SANDBOX_ENSURE, WorkspaceId: workspace}) + must(t, err) + effect := planned.GetEffect().GetId() + clone, plugin := "late-clone-"+state, "late-plugin-"+state + _, err = f.store.Pool.Exec(`INSERT INTO clone_executions(execution_id,operation_id,workspace_id,node_id,input,dispatched_epoch,node_operation_id) + VALUES($1,$2,$3,'node-late','{"kind":"clone"}',1,$1::text)`, clone, operation, workspace) + must(t, err) + _, err = f.store.Pool.Exec(`INSERT INTO node_executions(execution_id,kind,operation_id,workspace_id,node_id,input,dispatched_epoch,node_operation_id) + VALUES($1,'install_plugins',$2,$3,'node-late','{"kind":"install_plugins"}',1,$1::text)`, plugin, operation, workspace) + must(t, err) + _, err = c.operations.DeferOperation(c.ctx, &controlpb.DeferOperationRequest{SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, Version: planned.GetOperation().GetVersion(), State: controlpb.DeferState_DEFER_STATE_BLOCKED, Reason: controlpb.DeferReason_DEFER_REASON_NODE_UNAVAILABLE, RetrySeconds: 60}) + must(t, err) + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '1 second' WHERE id=$1`, run) + must(t, err) + must(t, f.store.GiveUpBlockedWorkspaceCreationsOnce(context.Background())) + if state != "reconciling" { + _, err = f.store.Pool.Exec(`UPDATE agent_run_residuals SET reclaim_state=$2,reclaim_id=$3,reclaim_mode='controlled', + reclaim_authorization='{"source":"test-prestate"}',cleared_at=CASE WHEN $2='cleared' THEN now() END, + disposition=CASE WHEN $2='cleared' THEN 'externally_confirmed' END WHERE run_id=$1`, run, state, operation) + must(t, err) + } + var responsibility string + must(t, f.store.Pool.QueryRow(`SELECT responsibility::text FROM agent_run_residuals WHERE run_id=$1`, run).Scan(&responsibility)) + facts := []*controlpb.RecordProvisioningFactRequest{ + { + Epoch: c.epoch, OperationId: operation, Kind: controlpb.ProvisioningFactKind_PROVISIONING_FACT_KIND_EFFECT, EffectId: effect, + State: controlpb.EffectState_EFFECT_STATE_SUCCEEDED, ExternalId: "substrate-late", + Evidence: &controlpb.EffectEvidence{Evidence: &controlpb.EffectEvidence_SandboxEnsured{SandboxEnsured: &controlpb.SandboxEnsured{SandboxInstanceId: effect, NodeId: "node-late"}}}, + }, + { + Epoch: c.epoch, OperationId: operation, Kind: controlpb.ProvisioningFactKind_PROVISIONING_FACT_KIND_CLONE, ExecutionId: clone, + ExecutionResult: &controlpb.ExecutionResult{Node: &controlpb.NodeIdentity{NodeId: "node-late", NodeIncarnationId: "inc-late"}, Outcome: &controlpb.ExecutionResult_CloneFailed{CloneFailed: &controlpb.CloneFailed{Reason: controlpb.CloneFailureReason_CLONE_FAILURE_REASON_BRANCH_NOT_FOUND}}}, + }, + { + Epoch: c.epoch, OperationId: operation, Kind: controlpb.ProvisioningFactKind_PROVISIONING_FACT_KIND_PLUGIN, ExecutionId: plugin, + ExecutionResult: &controlpb.ExecutionResult{Node: &controlpb.NodeIdentity{NodeId: "node-late", NodeIncarnationId: "inc-late"}, Outcome: &controlpb.ExecutionResult_PluginsFailed{PluginsFailed: &controlpb.PluginsFailed{Reason: controlpb.PluginsFailureReason_PLUGINS_FAILURE_REASON_PLUGIN_ROOT_UNAVAILABLE}}}, + }, + {Epoch: c.epoch, OperationId: operation, Kind: controlpb.ProvisioningFactKind_PROVISIONING_FACT_KIND_CREATION, Outcome: controlpb.ProvisioningOutcome_PROVISIONING_OUTCOME_READY}, + } + for _, fact := range facts { + fact.SubmissionId = c.sub() + _, err = c.operations.RecordProvisioningFact(c.ctx, fact) + must(t, err) + _, err = c.operations.RecordProvisioningFact(c.ctx, fact) + must(t, err) + replay := proto.Clone(fact).(*controlpb.RecordProvisioningFactRequest) + replay.SubmissionId = c.sub() + _, err = c.operations.RecordProvisioningFact(c.ctx, replay) + must(t, err) + } + var canonicalClone, canonicalPlugin string + must(t, f.store.Pool.QueryRow(`SELECT result::text FROM clone_executions WHERE execution_id=$1`, clone).Scan(&canonicalClone)) + must(t, f.store.Pool.QueryRow(`SELECT result::text FROM node_executions WHERE execution_id=$1`, plugin).Scan(&canonicalPlugin)) + conflicts := make([]*controlpb.RecordProvisioningFactRequest, len(facts)) + for i, fact := range facts { + conflicts[i] = proto.Clone(fact).(*controlpb.RecordProvisioningFactRequest) + conflicts[i].SubmissionId = c.sub() + } + failure := "late contradiction" + conflicts[0].State, conflicts[0].Evidence, conflicts[0].Failure = controlpb.EffectState_EFFECT_STATE_FAILED, nil, &failure + conflicts[1].ExecutionResult.Outcome = &controlpb.ExecutionResult_CloneFailed{CloneFailed: &controlpb.CloneFailed{Reason: controlpb.CloneFailureReason_CLONE_FAILURE_REASON_INTERRUPTED}} + conflicts[2].ExecutionResult.Outcome = &controlpb.ExecutionResult_PluginsFailed{PluginsFailed: &controlpb.PluginsFailed{Reason: controlpb.PluginsFailureReason_PLUGINS_FAILURE_REASON_INTERRUPTED}} + conflicts[3].Outcome = controlpb.ProvisioningOutcome_PROVISIONING_OUTCOME_FAILED + for _, conflict := range conflicts { + for _, submission := range []string{conflict.SubmissionId, conflict.SubmissionId, c.sub()} { + conflict.SubmissionId = submission + _, err = c.operations.RecordProvisioningFact(c.ctx, conflict) + expectStatus(t, err, codes.Aborted, controlpb.ErrorCode_ERROR_CODE_CONFLICT) + if status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("conflict did not return its stable residual code: %v", err) + } + } + } + // Read after the rejected RPCs: all conflicts must already be committed, deduplicated, + // and separate from the frozen responsibility and canonical execution results. + var retained bool + must(t, f.store.Pool.QueryRow(`SELECT r.responsibility=$2::jsonb AND r.reclaim_state=$3 + AND (r.cleared_at IS NOT NULL)=($3='cleared') AND r.reclaim_conflicted_at IS NOT NULL + AND jsonb_array_length(r.reclaim_evidence)=8 + AND (SELECT count(*) FROM jsonb_array_elements(r.reclaim_evidence) e WHERE e->'fact'->>'kind'='conflict' + AND e->'fact' ? 'incoming' AND e->'fact' ? 'original' AND e->>'source'<>'' AND e->>'receivedAt'<>'')=4 + AND (SELECT count(DISTINCT e->>'key') FROM jsonb_array_elements(r.reclaim_evidence) e WHERE e->>'key'=ANY($7::text[]))=4 + AND (SELECT result=$5::jsonb FROM clone_executions WHERE execution_id=$4) + AND (SELECT result=$6::jsonb FROM node_executions WHERE execution_id=$8) + AND (SELECT state='succeeded' AND external_id='substrate-late' + AND result=jsonb_build_object('sandboxInstanceId',$9::text,'nodeId','node-late') FROM external_effects WHERE id=$9::uuid) + AND (SELECT phase='done' AND status='failed' FROM issue_runs WHERE id=$1) + AND (SELECT NOT admission_open FROM workspaces WHERE id=$10) + FROM agent_run_residuals r WHERE r.run_id=$1`, run, responsibility, state, clone, canonicalClone, canonicalPlugin, + "{effect:"+effect+",clone:"+clone+",plugin:"+plugin+",creation}", plugin, effect, workspace).Scan(&retained)) + if !retained { + t.Fatal("late fact intake changed original state, lost durable conflict evidence, or duplicated a replay") + } + }) + } +} diff --git a/internal/core/agent_run_provisioning_fact.go b/internal/core/agent_run_provisioning_fact.go index eaf120ce..f7ebed8d 100644 --- a/internal/core/agent_run_provisioning_fact.go +++ b/internal/core/agent_run_provisioning_fact.go @@ -24,7 +24,7 @@ func provisioningFact(t *transaction, r *ControlRequest) Object { w := t.one("SELECT * FROM workspaces WHERE id=$1", o.S("workspaceId")) require(w != nil && w.S("issueRunId") != "", 409, "operation_not_abandoned") run := w.S("issueRunId") - require(t.one("SELECT run_id FROM agent_run_residuals WHERE run_id=$1 AND cleared_at IS NULL", run) != nil, + require(t.one("SELECT run_id FROM agent_run_residuals WHERE run_id=$1", run) != nil, 409, "operation_not_abandoned") switch r.Body.S("kind") { @@ -59,9 +59,7 @@ func provisioningEffectFact(t *transaction, r *ControlRequest, o Object, run str factOp["controllerEpoch"] = r.Body.N("epoch") out := effectResult(t, r, factOp) e := out.O("effect") - appendResidualFact(t, run, "effect:"+e.S("id"), Object{ - "kind": "effect", "effectKind": e.S("kind"), "state": e.S("state"), "externalId": e.S("externalId"), "result": e.O("result"), - }) + // effectResult appends the same validated fact on both the live and late intake paths. return Object{"operation": t.one("SELECT * FROM operations WHERE id=$1", o.S("id")), "effect": e} } @@ -86,9 +84,7 @@ func provisioningExecutionFact(t *transaction, r *ControlRequest, run string) Ob require(e.S("kind") == "install_plugins" || e.S("kind") == "remove_plugins", 409, "invalid_execution") stored = nodeExecutionResult(t, factRequest, false) } - appendResidualFact(t, run, r.Body.S("kind")+":"+stored.S("executionId"), Object{ - "kind": r.Body.S("kind"), "state": executionState(stored), "result": stored.O("result"), - }) + // The execution applier owns the ledger receipt as well as the canonical result. return Object{"operation": t.one("SELECT * FROM operations WHERE id=$1", r.OperationID), "execution": stored} } @@ -121,65 +117,3 @@ func provisioningCreationFact(t *transaction, r *ControlRequest, run string) Obj appendResidualFact(t, run, "creation", fact) return Object{"operation": t.one("SELECT * FROM operations WHERE id=$1", r.OperationID)} } - -// appendResidualFact receipts one late fact onto the abandoned creation's residual ledger. The -// fact key is its stable identity; the same fact replays as a no-op. A provisional observation -// (running, absent/planned, pending) does not lock the identity: the real world progresses, so a -// later fact — above all the real terminal — takes over and the superseded observation is kept -// as prior evidence. A recorded terminal is final: only its identical replay is accepted and any -// different fact for the identity is a conflict, so a retried submission can never stack -// duplicates or rewrite an outcome. The fact is durable before the caller's ACK: this write is -// part of the submission's transaction. -func appendResidualFact(t *transaction, run, key string, fact Object) { - row := t.one("SELECT responsibility FROM agent_run_residuals WHERE run_id=$1 AND cleared_at IS NULL", run) - require(row != nil, 409, "operation_not_abandoned") - resp := row.O("responsibility") - facts := Object{} - for k, v := range resp.O("lateFacts") { - if m, ok := v.(map[string]any); ok { - facts[k] = Object(m) - } - } - var prev Object - switch v := facts[key].(type) { - case map[string]any: - prev = Object(v) - case Object: - prev = v - } - if prev != nil { - if prev.S("state") == fact.S("state") && prev.S("externalId") == fact.S("externalId") && jsonText(prev.O("result")) == jsonText(fact.O("result")) { - return - } - // The recorded outcome is final: a different fact contradicts it. - require(!residualFactTerminal(prev), 409, "fact_conflict") - // A provisional observation is superseded, not contradicted: keep it as prior evidence - // while the later fact becomes the identity's current receipt. - superseded := Object{} - for k, v := range prev { - if k != "prior" { - superseded[k] = v - } - } - prior := []any{} - if old, ok := prev["prior"].([]any); ok { - prior = append(prior, old...) - } - fact["prior"] = append(prior, superseded) - } - fact["receivedAt"] = t.one("SELECT clock_timestamp()::text AS received_at").S("receivedAt") - facts[key] = fact - resp["lateFacts"] = facts - t.exec("UPDATE agent_run_residuals SET responsibility=$2 WHERE run_id=$1", run, jsonText(resp)) -} - -// residualFactTerminal reports whether one ledger fact is an outcome rather than an observation: -// effects and executions end at succeeded/failed, creations at ready/failed; running, -// absent/planned and pending are states the real world can still move on from. -func residualFactTerminal(fact Object) bool { - switch fact.S("state") { - case "succeeded", "failed", "ready": - return true - } - return false -} diff --git a/internal/core/agent_run_provisioning_fact_db_test.go b/internal/core/agent_run_provisioning_fact_db_test.go index 7d3cdd4a..d1005285 100644 --- a/internal/core/agent_run_provisioning_fact_db_test.go +++ b/internal/core/agent_run_provisioning_fact_db_test.go @@ -2,6 +2,7 @@ package core import ( "context" + "errors" "strings" "testing" ) @@ -15,8 +16,9 @@ import ( // deadline, abandonment committed. The lease stays seeded for the fact submissions. type abandonedCreation struct { provisioningSeed - claims *Claims - effect string + claims *Claims + effect string + responsibility string } // seedAbandonedCreation parks a creation in blocked (freezing the deadline), optionally plans one @@ -61,7 +63,30 @@ func seedAbandonedCreation(t *testing.T, store *Store, withEffect bool) abandone if phase.String != "done" || status != "failed" || reason != "workspace_unavailable" { t.Fatalf("seed did not abandon the creation: %s/%s/%s", phase.String, status, reason) } - return abandonedCreation{provisioningSeed: p, claims: claims, effect: effect} + var responsibility string + if err := store.Pool.QueryRow(`SELECT responsibility::text FROM agent_run_residuals WHERE run_id=$1`, p.run).Scan(&responsibility); err != nil { + t.Fatal(err) + } + return abandonedCreation{provisioningSeed: p, claims: claims, effect: effect, responsibility: responsibility} +} + +func assertProvisioningResponsibilityUnchanged(t *testing.T, store *Store, scene abandonedCreation) { + t.Helper() + var unchanged bool + if err := store.Pool.QueryRow(`SELECT responsibility=$2::jsonb FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.responsibility).Scan(&unchanged); err != nil { + t.Fatal(err) + } + if !unchanged { + t.Fatal("late facts changed the original residual responsibility") + } +} + +func assertResidualFactConflict(t *testing.T, err error) { + t.Helper() + var fault *Fault + if !errors.As(err, &fault) || fault.Status != 409 || fault.Code != "residual_fact_conflict" { + t.Fatalf("want stable residual_fact_conflict Fault409, got %v", err) + } } // submitFact drives the provisioning_fact action; a non-nil error is returned as-is for conflict @@ -101,21 +126,36 @@ func TestProvisioningFactReceiptsLateEffectWithoutResurrecting(t *testing.T) { } var facts int var state, external string - if err := store.Pool.QueryRow(`SELECT (SELECT count(*) FROM jsonb_object_keys(responsibility->'lateFacts')), - responsibility->'lateFacts'->('effect:'||$2)->>'state', responsibility->'lateFacts'->('effect:'||$2)->>'externalId' - FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.effect).Scan(&facts, &state, &external); err != nil { + if err := store.Pool.QueryRow(`SELECT jsonb_array_length(reclaim_evidence), + reclaim_evidence->0->'fact'->>'state', reclaim_evidence->0->'fact'->>'externalId' + FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&facts, &state, &external); err != nil { t.Fatalf("ledger receipt read failed: %v", err) } if facts != 1 || state != "succeeded" || external != "sub-123" { t.Fatalf("ledger receipt wrong: count=%d state=%s external=%s", facts, state, external) } - // A different fact for the same effect identity is refused — by the live path's own evidence - // rules here (a succeeded effect cannot become failed), which is exactly the guarantee the - // entry reuses — and the ledger keeps the original receipt. + // A conflicting fact is durable before the stable error is returned. The canonical effect + // remains unchanged, and both a lost-response replay and a new submission deduplicate it. conflicting := Object{"epoch": 1, "kind": "effect", "effectId": scene.effect, "state": "failed", "externalId": "sub-123", "result": Object{"error": "substrate said no"}} - if _, err := submitFact(t, store, scene, "fact-3", conflicting); err == nil { - t.Fatal("contradicting fact accepted") + for _, submission := range []string{"fact-3", "fact-3", "fact-4"} { + _, err := submitFact(t, store, scene, submission, conflicting) + assertResidualFactConflict(t, err) + } + var committed bool + if err := store.Pool.QueryRow(`SELECT jsonb_array_length(r.reclaim_evidence)=2 AND r.reclaim_conflicted_at IS NOT NULL + AND r.reclaim_evidence->1->>'key'='effect:'||$2 + AND r.reclaim_evidence->1->'fact'->>'kind'='conflict' + AND r.reclaim_evidence->1->'fact'->'incoming'->>'state'='failed' + AND r.reclaim_evidence->1->'fact'->'original'->>'state'='succeeded' + AND e.state='succeeded' AND e.external_id='sub-123' + AND e.result=jsonb_build_object('sandboxInstanceId',$2::text,'nodeId','node-late') + FROM agent_run_residuals r JOIN external_effects e ON e.id=$2::uuid WHERE r.run_id=$1`, scene.run, scene.effect).Scan(&committed); err != nil { + t.Fatal(err) + } + if !committed { + t.Fatal("conflict was not retained exactly once before its error, or changed the canonical effect") } + assertProvisioningResponsibilityUnchanged(t, store, scene) phase, status, reason, _, _, _ := runFields(t, store, scene.run) if phase.String != "done" || status != "failed" || reason != "workspace_unavailable" { t.Fatalf("late fact resurrected the run: %s/%s/%s", phase.String, status, reason) @@ -163,16 +203,23 @@ func TestProvisioningFactReceiptsLateCloneAndCreationOutcomes(t *testing.T) { if _, err := submitFact(t, store, scene, "creation-1", Object{"epoch": 1, "kind": "creation", "outcome": "ready"}); err != nil { t.Fatalf("creation fact submission replay failed: %v", err) } - if _, err := submitFact(t, store, scene, "creation-2", Object{"epoch": 1, "kind": "creation", "outcome": "failed"}); err == nil || !strings.Contains(err.Error(), "fact_conflict") { - t.Fatalf("contradicting creation outcome accepted: %v", err) + for _, submission := range []string{"creation-2", "creation-2", "creation-3"} { + _, err := submitFact(t, store, scene, submission, Object{"epoch": 1, "kind": "creation", "outcome": "failed"}) + assertResidualFactConflict(t, err) } var creationState string - if err := store.Pool.QueryRow(`SELECT responsibility->'lateFacts'->'creation'->>'state' FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&creationState); err != nil { + var conflicts int + if err := store.Pool.QueryRow(`SELECT + (SELECT e->'fact'->>'state' FROM jsonb_array_elements(reclaim_evidence) e WHERE e->>'key'='creation' AND e->'fact'->>'kind'='creation'), + (SELECT count(*) FROM jsonb_array_elements(reclaim_evidence) e WHERE e->>'key'='creation' AND e->'fact'->>'kind'='conflict' + AND e->'fact'->'incoming'->>'state'='failed' AND e->'fact'->'original'->>'state'='ready' AND reclaim_conflicted_at IS NOT NULL) + FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&creationState, &conflicts); err != nil { t.Fatal(err) } - if creationState != "ready" { - t.Fatalf("creation fact not receipted: %s", creationState) + if creationState != "ready" || conflicts != 1 { + t.Fatalf("creation fact or durable deduplicated conflict wrong: state=%s conflicts=%d", creationState, conflicts) } + assertProvisioningResponsibilityUnchanged(t, store, scene) // A late success is evidence only: the run keeps its business terminal and the Workspace keeps // its admission fence. var admissionOpen bool @@ -286,22 +333,22 @@ func TestProvisioningEffectFactProgressesFromProvisionalToTerminal(t *testing.T) if _, err := submitFact(t, store, scene, "progress-replay", terminal); err != nil { t.Fatalf("identical terminal must replay: %v", err) } - // One receipt for the identity: the terminal on top, both provisional observations retained. - var state string - var prior int - if err := store.Pool.QueryRow(`SELECT responsibility->'lateFacts'->('effect:'||$2)->>'state', - jsonb_array_length(responsibility->'lateFacts'->('effect:'||$2)->'prior') - FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.effect).Scan(&state, &prior); err != nil { + // Each observation remains an independent append-only entry in arrival order. + var states string + var entries int + if err := store.Pool.QueryRow(`SELECT string_agg(e->'fact'->>'state', ',' ORDER BY ordinal), count(*) + FROM agent_run_residuals r, jsonb_array_elements(r.reclaim_evidence) WITH ORDINALITY AS facts(e,ordinal) + WHERE r.run_id=$1 AND e->>'key'='effect:'||$2`, scene.run, scene.effect).Scan(&states, &entries); err != nil { t.Fatalf("ledger receipt read failed: %v", err) } - if state != "succeeded" || prior != 2 { - t.Fatalf("ledger receipt wrong: state=%s prior=%d", state, prior) + if states != "planned,running,succeeded" || entries != 3 { + t.Fatalf("ledger observations wrong: states=%s entries=%d", states, entries) } // A terminal is final: a contradicting outcome stays refused, by the live path's own evidence // rules for the effect row, and the ledger keeps the recorded terminal. - if _, err := submitFact(t, store, scene, "progress-conflict", Object{"epoch": 1, "kind": "effect", "effectId": scene.effect, "state": "failed", "externalId": "sub-123", "result": Object{"error": "substrate said no"}}); err == nil { - t.Fatal("contradicting terminal accepted") - } + _, err = submitFact(t, store, scene, "progress-conflict", Object{"epoch": 1, "kind": "effect", "effectId": scene.effect, "state": "failed", "externalId": "sub-123", "result": Object{"error": "substrate said no"}}) + assertResidualFactConflict(t, err) + assertProvisioningResponsibilityUnchanged(t, store, scene) phase, status, reason, _, _, _ := runFields(t, store, scene.run) if phase.String != "done" || status != "failed" || reason != "workspace_unavailable" { t.Fatalf("progression resurrected the run: %s/%s/%s", phase.String, status, reason) @@ -341,7 +388,8 @@ func TestProvisioningFactSubmissionBindsItsTarget(t *testing.T) { // The ledger still holds exactly the two facts receipted for their original targets. var facts int - if err := store.Pool.QueryRow(`SELECT count(*) FROM jsonb_object_keys((SELECT responsibility->'lateFacts' FROM agent_run_residuals WHERE run_id=$1))`, scene.run).Scan(&facts); err != nil || facts != 2 { + if err := store.Pool.QueryRow(`SELECT jsonb_array_length(reclaim_evidence) FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&facts); err != nil || facts != 2 { t.Fatalf("ledger receipts wrong: count=%d err=%v", facts, err) } + assertProvisioningResponsibilityUnchanged(t, store, scene) } diff --git a/internal/core/agent_run_residual_fact.go b/internal/core/agent_run_residual_fact.go new file mode 100644 index 00000000..fcf1257b --- /dev/null +++ b/internal/core/agent_run_residual_fact.go @@ -0,0 +1,215 @@ +package core + +import "fmt" + +// residualForWorkspace also finds cleared responsibilities: cleanup never closes the original +// identity's evidence intake. Soft-deleted Workspace rows remain the identity authority. +func residualForWorkspace(t *transaction, workspaceID string) Object { + return t.one("SELECT * FROM agent_run_residuals WHERE workspace_id=$1", workspaceID) +} + +func residualObjects(value any) []Object { + var out []Object + switch values := value.(type) { + case []Object: + return values + case []any: + for _, item := range values { + switch v := item.(type) { + case map[string]any: + out = append(out, Object(v)) + case Object: + out = append(out, v) + } + } + } + return out +} + +// appendResidualFact records one validated fact without editing the original responsibility. +// False means a contradictory terminal was durably audited; callers must not apply it to the +// canonical execution/effect. Control returns the stable error only after this transaction commits. +func appendResidualFact(t *transaction, run, key string, fact Object) bool { + row := t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", run) + require(row != nil, 409, "operation_not_abandoned") + var previous Object + entries := residualObjects(row["reclaimEvidence"]) + for _, entry := range entries { + if entry.S("key") != key { + continue + } + old := entry.O("fact") + if old.S("kind") == "conflict" { + if jsonText(old.O("incoming")) == jsonText(fact) { + t.residualFactConflict = true + return false + } + continue + } + if jsonText(old) == jsonText(fact) { + return true + } + previous = old + } + // Pre-migration lateFacts stay intact. They may supply the original terminal, never a reason + // to rewrite the legacy JSON or to infer cleanup eligibility during migration. + if previous == nil { + previous = row.O("responsibility").O("lateFacts").O(key) + } + if len(previous) > 0 && residualFactTerminal(previous) && !sameResidualFact(previous, fact) { + return appendResidualConflict(t, row, key, previous, fact) + } + appendResidualEntry(t, row, key, fact, false) + return true +} + +func appendResidualConflict(t *transaction, row Object, key string, previous, incoming Object) bool { + t.residualFactConflict = true + for _, entry := range residualObjects(row["reclaimEvidence"]) { + old := entry.O("fact") + if entry.S("key") == key && old.S("kind") == "conflict" && jsonText(old.O("incoming")) == jsonText(incoming) { + return false + } + } + appendResidualEntry(t, row, key, Object{"kind": "conflict", "state": "conflict", "original": previous, "incoming": incoming}, true) + return false +} + +func appendResidualEntry(t *transaction, row Object, key string, fact Object, conflict bool) { + source := fact.O("source") + if len(source) == 0 { + source = t.residualFactSource + if len(source) == 0 { + source = Object{"kind": "cloud_accepted_fact", "identity": key} + } + } + entry := Object{ + "key": key, "fact": fact, "source": source, + "receivedAt": t.one("SELECT clock_timestamp()::text AS received_at").S("receivedAt"), + } + t.exec(`UPDATE agent_run_residuals SET reclaim_evidence=reclaim_evidence||jsonb_build_array($2::jsonb), + reclaim_version=reclaim_version+1, + reclaim_conflicted_at=CASE WHEN $3 THEN COALESCE(reclaim_conflicted_at,clock_timestamp()) ELSE reclaim_conflicted_at END + WHERE run_id=$1`, row.S("runId"), jsonText(entry), conflict) +} + +func residualEffectRow(t *transaction, e Object) Object { + if row := residualForWorkspace(t, e.S("workspaceId")); row != nil { + return row + } + // Some legacy preallocated ensure plans have no Workspace binding on the effect row. + return t.one(`SELECT ar.* FROM agent_run_residuals ar JOIN operations o ON o.workspace_id=ar.workspace_id + WHERE o.id=$1 AND o.tenant_id=ar.tenant_id AND o.project_id=ar.project_id`, e.S("operationId")) +} + +func acceptResidualEffectFact(t *transaction, row, e, incoming Object) bool { + key := "effect:" + e.S("id") + if e.S("externalId") != "" && incoming.S("externalId") != "" && e.S("externalId") != incoming.S("externalId") { + return appendResidualConflict(t, row, key, + Object{"kind": "effect", "state": e.S("state"), "externalId": e.S("externalId"), "result": e.O("result")}, incoming) + } + for _, entry := range residualObjects(row["reclaimEvidence"]) { + if entry.S("key") == key && jsonText(entry.O("fact")) == jsonText(incoming) && e.S("state") != incoming.S("state") { + // An earlier observation can replay without downgrading the canonical terminal. + return false + } + } + if e.S("state") == "succeeded" || e.S("state") == "failed" { + original := Object{ + "kind": "effect", "effectKind": e.S("kind"), "state": e.S("state"), + "externalId": e.S("externalId"), "result": e.O("result"), + } + if !appendResidualFact(t, row.S("runId"), key, original) { + return false + } + } + return appendResidualFact(t, row.S("runId"), key, incoming) +} + +func acceptResidualReceipt(t *transaction, e Object, sequence int64, event string) bool { + row := residualForWorkspace(t, e.S("workspaceId")) + if row == nil { + return true + } + old := t.one("SELECT event FROM node_event_receipts WHERE execution_id=$1 AND sequence=$2", e.S("executionId"), sequence) + if old != nil && old.S("event") != event { + key := "receipt:" + e.S("executionId") + ":" + fmt.Sprint(sequence) + return appendResidualConflict(t, row, key, Object{"kind": "receipt", "event": old.S("event")}, Object{"kind": "receipt", "event": event}) + } + return true +} + +func sameResidualFact(a, b Object) bool { + // Legacy facts also contain receipt timestamps and provisional history. Neither changes the + // terminal's semantic identity when importing an existing receipt into the append-only column. + for _, key := range []string{"kind", "state", "externalId", "detail"} { + if a.S(key) != b.S(key) { + return false + } + } + return jsonText(a.O("result")) == jsonText(b.O("result")) +} + +func residualFactTerminal(fact Object) bool { + switch fact.S("state") { + case "succeeded", "failed", "ready": + return true + } + return false +} + +// requireNoResidualConflict fences future mutations even after a responsibility was cleared. +func requireNoResidualConflict(t *transaction, workspaceID string) { + row := residualForWorkspace(t, workspaceID) + require(row == nil || row["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") +} + +// residualExecutionIdentity verifies the original work and resource binding without requiring a +// live Node, open admission, an unfinished Run, or an undeleted Workspace. +func residualExecutionIdentity(t *transaction, e Object) Object { + row := residualForWorkspace(t, e.S("workspaceId")) + if row == nil { + return nil + } + w := t.one("SELECT * FROM workspaces WHERE id=$1", row.S("workspaceId")) + require(w != nil && w.S("tenantId") == row.S("tenantId") && w.S("projectId") == row.S("projectId") && w.S("issueRunId") == row.S("runId"), 409, "residual_identity_conflict") + if e.S("kind") == "agent_session" || e.S("kind") == "deliver_revision" { + require(e.S("operationId") == row.S("runId"), 409, "residual_identity_conflict") + work := t.one("SELECT * FROM execution_work WHERE id=$1 AND run_id=$2", e.S("workId"), row.S("runId")) + require(work != nil && work.S("executionId") == e.S("executionId") && work.S("kind") == e.S("kind"), 409, "residual_identity_conflict") + target := work.O("target") + require(target.S("workspaceId") == w.S("id") && target.S("nodeId") == e.S("nodeId"), 409, "residual_identity_conflict") + sandbox := t.one("SELECT * FROM sandbox_instances WHERE id=$1 AND workspace_id=$2", target.S("sandboxInstanceId"), w.S("id")) + require(sandbox != nil && sandbox.N("generation") <= row.N("runtimeGeneration"), 409, "residual_identity_conflict") + } else { + require(t.one("SELECT id FROM operations WHERE id=$1 AND workspace_id=$2 AND tenant_id=$3 AND project_id=$4", e.S("operationId"), w.S("id"), row.S("tenantId"), row.S("projectId")) != nil, 409, "residual_identity_conflict") + } + return row +} + +func residualExecutionFact(e, result Object) (string, Object) { + kind, prefix := e.S("kind"), "execution:" + switch kind { + case "install_plugins", "remove_plugins": + kind, prefix = "plugin", "plugin:" + case "": + kind, prefix = "clone", "clone:" + } + state := executionState(Object{"result": result}) + return prefix + e.S("executionId"), Object{"kind": kind, "state": state, "result": result} +} + +func acceptResidualExecutionFact(t *transaction, e, result Object) bool { + row := residualExecutionIdentity(t, e) + if row == nil { + return true + } + key, incoming := residualExecutionFact(e, result) + if previous := e.O("result"); len(previous) > 0 { + _, original := residualExecutionFact(e, previous) + if !appendResidualFact(t, row.S("runId"), key, original) { + return false + } + } + return appendResidualFact(t, row.S("runId"), key, incoming) +} diff --git a/internal/core/clone.go b/internal/core/clone.go index c0eb547e..8b5f3e67 100644 --- a/internal/core/clone.go +++ b/internal/core/clone.go @@ -187,6 +187,9 @@ func recordedSubmission(t *transaction, r *ControlRequest, hash string, apply fu return old.O("response") } out := apply() + if t.residualFactConflict { + out["residualFactConflict"] = true + } t.exec("INSERT INTO control_submissions(submission_id,holder_id,request_hash,response) VALUES($1,$2,$3,$4)", r.SubmissionID, r.Service.Subject, hash, jsonText(out)) return out } @@ -265,6 +268,9 @@ func cloneResult(t *transaction, r *ControlRequest, withReceipt bool) Object { require(e != nil, 404, "not_found") outcome := result.S("outcome") require((outcome == "clone_ready" || outcome == "clone_failed") && result.O("node").S("nodeId") == e.S("nodeId"), 409, "result_conflict") + if !acceptResidualExecutionFact(t, e, result) { + return e + } if previous := e.O("result"); len(previous) > 0 { require(jsonText(previous) == jsonText(result), 409, "result_conflict") } else { diff --git a/internal/core/control.go b/internal/core/control.go index c0dd49a6..74a3e107 100644 --- a/internal/core/control.go +++ b/internal/core/control.go @@ -32,7 +32,12 @@ func (s *Store) Control(ctx context.Context, r *ControlRequest) (Object, error) copyRequest.verification = prepared r = ©Request } - out, err := s.transact(ctx, func(t *transaction) Object { + out, err := s.transact(ctx, func(t *transaction) (response Object) { + defer func() { + if t.residualFactConflict && response != nil { + response["residualFactConflict"] = true + } + }() if r.Action == "access" || r.Action == "admit" { require(r.Service.Role == "controller", 403, "service_forbidden") if r.Action == "admit" || r.Body.S("action") == "execute" { @@ -54,6 +59,10 @@ func (s *Store) Control(ctx context.Context, r *ControlRequest) (Object, error) return cloneCommand(t, r) } leaseValid(t, r) + t.residualFactSource = Object{ + "kind": "controller", "holderId": r.Service.Subject, + "epoch": r.Body.N("epoch"), "submissionId": r.SubmissionID, + } refreshRuntimeControls(t) if r.Action == "effect_permit" { return effectPermit(t, r) @@ -87,6 +96,9 @@ func (s *Store) Control(ctx context.Context, r *ControlRequest) (Object, error) // return the recorded response, not fail the version check it already passed. return submitted(t, r, func() Object { return operationCommand(t, r) }) }) + if err == nil && out.B("residualFactConflict") { + return nil, &Fault{Status: 409, Code: "residual_fact_conflict", Params: Object{}} + } return out, err } @@ -285,22 +297,19 @@ func effectResult(t *transaction, r *ControlRequest, o Object) Object { e := t.one("SELECT * FROM external_effects WHERE id=$1 AND operation_id=$2", r.EffectID, o.S("id")) require(e != nil, 404, "not_found") require(e.S("kind") != "plugin_ensure" && e.S("kind") != "plugin_delete", 409, "invalid_step") + residual := residualEffectRow(t, e) state := r.Body.S("state") require(state == "running" || state == "succeeded" || state == "failed" || state == "absent", 400, "invalid_effect_state") result := r.Body.O("result") external := r.Body.S("externalId") if state == "absent" { - require(e.S("state") == "planned", 409, "external_state_conflict") state = "planned" } else { require(external != "" && len(external) <= 200, 400, "external_id_required") - if e.S("externalId") != "" { + if e.S("externalId") != "" && residual == nil { require(external == e.S("externalId"), 409, "external_binding_conflict") } } - if e.S("state") == "succeeded" { - require(state == "succeeded" && jsonText(result) == jsonText(e.O("result")), 409, "external_state_conflict") - } if state == "succeeded" { switch e.S("kind") { case "sandbox_terminate": @@ -319,6 +328,21 @@ func effectResult(t *transaction, r *ControlRequest, o Object) Object { require(result.B("removed"), 400, "invalid_cleanup_evidence") } } + if residual != nil { + fact := Object{ + "kind": "effect", "effectKind": e.S("kind"), "state": state, + "externalId": external, "result": result, + } + if !acceptResidualEffectFact(t, residual, e, fact) { + return Object{"effect": e, "operation": t.one("SELECT * FROM operations WHERE id=$1", o.S("id"))} + } + } + if r.Body.S("state") == "absent" { + require(e.S("state") == "planned", 409, "external_state_conflict") + } + if e.S("state") == "succeeded" { + require(state == "succeeded" && jsonText(result) == jsonText(e.O("result")), 409, "external_state_conflict") + } if state == "failed" && (e.S("kind") == "plugin_ensure" || e.S("kind") == "plugin_delete") { // A failed install/remove surfaces on the space row immediately so the // UI can render the failure while the operation waits for retry. diff --git a/internal/core/node_execution.go b/internal/core/node_execution.go index ccfbd781..5b688b8b 100644 --- a/internal/core/node_execution.go +++ b/internal/core/node_execution.go @@ -115,6 +115,13 @@ func nodeExecutionResult(t *transaction, r *ControlRequest, withReceipt bool) Ob e := t.one("SELECT * FROM node_executions WHERE execution_id=$1 AND operation_id=$2", execution, operation) require(e != nil, 404, "not_found") require(result.O("node").S("nodeId") == e.S("nodeId"), 409, "result_conflict") + validateNodeResult(e, result) + if withReceipt && !acceptResidualReceipt(t, e, r.Body.N("sequence"), r.Body.S("event")) { + return e + } + if !acceptResidualExecutionFact(t, e, result) { + return e + } fresh := len(e.O("result")) == 0 if !fresh { require(jsonText(e.O("result")) == jsonText(result), 409, "result_conflict") diff --git a/internal/core/revision.go b/internal/core/revision.go index d02d1059..131a9fe1 100644 --- a/internal/core/revision.go +++ b/internal/core/revision.go @@ -28,6 +28,10 @@ func (s *Store) prepareRevision(ctx context.Context, r *ControlRequest) (*revisi e, err := s.transact(ctx, func(t *transaction) Object { require(r.Service != nil && r.Service.Role == "controller", 403, "service_forbidden") leaseValid(t, r) + t.residualFactSource = Object{ + "kind": "controller", "holderId": r.Service.Subject, + "epoch": r.Body.N("epoch"), "submissionId": r.SubmissionID, + } require(len(r.SubmissionID) <= 200, 400, "invalid_submission") if old := t.one("SELECT * FROM control_submissions WHERE submission_id=$1", r.SubmissionID); old != nil { require(old.S("requestHash") == requestHash(r.Action, r.Service.Subject, r.Body), 409, "submission_conflict") @@ -36,6 +40,11 @@ func (s *Store) prepareRevision(ctx context.Context, r *ControlRequest) (*revisi e := t.one("SELECT * FROM node_executions WHERE execution_id=$1 AND operation_id=$2 AND kind='deliver_revision'", r.Body.S("executionId"), r.Body.S("operationId")) require(e != nil, 404, "not_found") if len(e.O("result")) > 0 { + require(r.Body.O("result").O("node").S("nodeId") == e.S("nodeId"), 409, "result_conflict") + validateNodeResult(e, r.Body.O("result")) + if !acceptResidualExecutionFact(t, e, r.Body.O("result")) { + return Object{"residualFactConflict": true} + } require(jsonText(e.O("result")) == jsonText(r.Body.O("result")), 409, "result_conflict") return nil } @@ -47,6 +56,9 @@ func (s *Store) prepareRevision(ctx context.Context, r *ControlRequest) (*revisi if err != nil || e == nil { return nil, err } + if e.B("residualFactConflict") { + return nil, &Fault{Status: 409, Code: "residual_fact_conflict", Params: Object{}} + } result := r.Body.O("result") verified := &revisionVerification{input: jsonText(e.O("input")), valid: true} objects := []Object{result.O("history")} @@ -89,6 +101,10 @@ func validStoredObject(object Object) bool { // liveDelivery fences cancellation, deletion, replaced Nodes and closed runtime bindings again // after external verification. A stale result must not release a different Workspace generation. func liveDelivery(t *transaction, e Object) { + if residualExecutionIdentity(t, e) != nil { + // This is evidence about the original execution, not permission to execute it again. + return + } require(e["terminatedByForceStopId"] == nil, 409, "dispatch_conflict") w := t.one("SELECT * FROM workspaces WHERE id=$1 AND deleted_at IS NULL", e.S("workspaceId")) require(w != nil && w.S("issueRunId") == e.S("operationId") && w.S("observedState") == "ready" && w.B("admissionOpen"), 409, "dispatch_conflict") diff --git a/internal/core/runtime_control.go b/internal/core/runtime_control.go index 21307023..0cc0473c 100644 --- a/internal/core/runtime_control.go +++ b/internal/core/runtime_control.go @@ -34,7 +34,7 @@ func reconcileRuntimeControls(t *transaction) { rows := t.list(`SELECT c.* FROM runtime_controls c JOIN workspaces w ON w.id=c.workspace_id WHERE c.state IN ('draining','reconciling') AND NOT EXISTS(SELECT 1 FROM runtime_force_stops f WHERE f.workspace_id=w.id AND f.state<>'succeeded') - AND NOT EXISTS(SELECT 1 FROM agent_run_residuals ar WHERE ar.workspace_id=w.id AND ar.cleared_at IS NULL) + AND NOT EXISTS(SELECT 1 FROM agent_run_residuals ar WHERE ar.workspace_id=w.id AND (ar.cleared_at IS NULL OR ar.reclaim_conflicted_at IS NOT NULL)) AND (c.input_closed OR (w.observed_state IN ('stopped','deleted') AND NOT EXISTS(SELECT 1 FROM sandbox_instances s WHERE s.workspace_id=w.id AND s.terminated_at IS NULL))) AND NOT EXISTS(SELECT 1 FROM execution_tickets a WHERE a.workspace_id=w.id AND a.state='active' AND a.terminated_by_force_stop_id IS NULL) diff --git a/internal/core/runtime_control_delivery.go b/internal/core/runtime_control_delivery.go index d9e0ff6c..e3880c32 100644 --- a/internal/core/runtime_control_delivery.go +++ b/internal/core/runtime_control_delivery.go @@ -1,5 +1,7 @@ package core +import "fmt" + // runtimeControlCommand is a finite, lease-fenced Controller contract. The Controller may // acknowledge Node evidence, but cannot assign a holder, epoch or target chosen by a user. func runtimeControlCommand(t *transaction, r *ControlRequest) Object { @@ -16,6 +18,15 @@ func runtimeControlCommand(t *transaction, r *ControlRequest) Object { v, ok := id.(string) require(ok && v != "" && len(v) <= 256, 400, "invalid_runtime_evidence") } + if row := residualForWorkspace(t, c.S("workspaceId")); row != nil { + sandbox := t.one("SELECT generation FROM sandbox_instances WHERE id=$1", n.S("sandboxInstanceId")) + appendResidualFact(t, row.S("runId"), "runtime:"+n.S("id")+":"+fmt.Sprint(c.N("controlEpoch"))+":"+fmt.Sprint(c.N("version")), Object{ + "kind": "runtime_closed", "state": "observed", "workspaceId": c.S("workspaceId"), + "sandboxId": n.S("sandboxInstanceId"), "generation": sandbox.N("generation"), + "controlEpoch": c.N("controlEpoch"), "controlVersion": c.N("version"), + "inputClosed": closed, "unfinishedExecutionIds": unfinished, + }) + } if closed && (c.S("state") == "held" || c.S("state") == "acquiring") { // Node expiry is conservative. Observed closed input cannot be reopened by a late renew, // even if the PostgreSQL deadline has not elapsed yet. diff --git a/internal/core/store.go b/internal/core/store.go index 24a281c4..c6b50063 100644 --- a/internal/core/store.go +++ b/internal/core/store.go @@ -250,6 +250,8 @@ type transaction struct { workSignals []string commandSignals []string events []SpaceEvent + residualFactConflict bool + residualFactSource Object } func (t *transaction) exec(q string, args ...any) { From d13d68c36a495d12a7d1381a708166859f9e0fae Mon Sep 17 00:00:00 2001 From: creegon2 <315603738+creegon2@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:51:11 -0700 Subject: [PATCH 03/11] feat(core): reclaim residuals through one durable owned delete Union original targets and require verified delivery plus real execution or per-target termination evidence before system authorization. Bind exactly one cleanup operation, recheck plan/permit/completion under the shared transaction lock, and atomically clear only after all targets and data removal close. Preserve public and project guards and retain the same retry_wait operation after quiesce refusal. Reserve controlled arbitration without exposing an operator command. --- internal/core/agent_run_residual.go | 317 ++++++++++++++++++++ internal/core/agent_run_residual_reclaim.go | 207 +++++++++++++ internal/core/control.go | 11 +- internal/core/node.go | 15 +- internal/core/public.go | 2 + internal/core/run_workspace.go | 3 +- internal/core/runtime_force_stop.go | 3 + 7 files changed, 554 insertions(+), 4 deletions(-) create mode 100644 internal/core/agent_run_residual.go create mode 100644 internal/core/agent_run_residual_reclaim.go diff --git a/internal/core/agent_run_residual.go b/internal/core/agent_run_residual.go new file mode 100644 index 00000000..cab7db3f --- /dev/null +++ b/internal/core/agent_run_residual.go @@ -0,0 +1,317 @@ +package core + +import "sort" + +// residualIdentity re-reads the original ownership in the caller's serialized transaction. +// Neither a request's reclaim ID nor a UUID foreign key alone establishes this scope. +func residualIdentity(t *transaction, row Object) (run, workspace, project Object) { + require(row != nil, 404, "residual_not_found") + run = t.one("SELECT * FROM issue_runs WHERE id=$1", row.S("runId")) + workspace = t.one("SELECT * FROM workspaces WHERE id=$1", row.S("workspaceId")) + project = t.one("SELECT * FROM projects WHERE id=$1", row.S("projectId")) + require(run != nil && workspace != nil && project != nil && run.S("executorType") == "agent" && + run.S("tenantId") == row.S("tenantId") && run.S("workspaceId") == workspace.S("id") && + workspace.S("issueRunId") == run.S("id") && workspace.S("tenantId") == row.S("tenantId") && + workspace.S("projectId") == project.S("id") && project.S("tenantId") == row.S("tenantId") && + workspace.S("kind") == "isolated" && workspace.N("runtimeGeneration") == row.N("runtimeGeneration"), 409, "residual_identity_conflict") + return run, workspace, project +} + +// collectResidualTargets is a union, never a replacement snapshot. It includes original NULL-scope +// effects by their original operation, and retains missing historical references so absence cannot +// silently turn an unknown target into permission to delete data. The bound delete is not a new +// provisioning responsibility. Each target has kind/id plus its original associations. +func collectResidualTargets(t *transaction, row Object) []Object { + _, w, _ := residualIdentity(t, row) + byKey := map[string]Object{} + add := func(target Object) { + if target.S("id") == "" { + return + } + key := target.S("kind") + ":" + target.S("id") + if previous := byKey[key]; previous != nil { + for k, v := range target { + if _, exists := previous[k]; !exists { + previous[k] = v + } + } + return + } + byKey[key] = target + } + for _, target := range residualObjects(row["reclaimTargets"]) { + add(target) + } + ref := row.O("responsibility") + for _, spec := range []struct{ list, kind, key string }{ + {"operations", "operation", "id"}, + {"effects", "effect", "id"}, + {"sandboxes", "sandbox", "id"}, + {"nodes", "node", "id"}, + {"executions", "execution", "executionId"}, + {"clones", "clone", "executionId"}, + } { + for _, original := range residualObjects(ref[spec.list]) { + target := Object{"kind": spec.kind, "id": original.S(spec.key), "workspaceId": w.S("id")} + for _, key := range []string{"operationId", "sandboxInstanceId", "generation", "nodeId", "workId"} { + if v, exists := original[key]; exists { + target[key] = v + } + } + add(target) + } + } + for _, op := range t.list("SELECT * FROM operations WHERE workspace_id=$1 AND id IS DISTINCT FROM $2::uuid", w.S("id"), nullable(row.S("reclaimOperationId"))) { + add(Object{"kind": "operation", "id": op.S("id"), "workspaceId": w.S("id")}) + } + // Resolve all original operations, including a Project-scoped operation named in responsibility. + operations := []string{} + for _, target := range byKey { + if target.S("kind") == "operation" { + operations = append(operations, target.S("id")) + } + } + for _, id := range operations { + for _, e := range t.list("SELECT * FROM external_effects WHERE operation_id=$1 AND (workspace_id=$2 OR workspace_id IS NULL)", id, w.S("id")) { + add(Object{"kind": "effect", "id": e.S("id"), "operationId": id, "workspaceId": w.S("id")}) + } + } + for _, e := range t.list("SELECT * FROM external_effects WHERE workspace_id=$1 AND operation_id IS DISTINCT FROM $2::uuid", w.S("id"), nullable(row.S("reclaimOperationId"))) { + add(Object{"kind": "effect", "id": e.S("id"), "operationId": e.S("operationId"), "workspaceId": w.S("id")}) + } + // Every planned ensure is itself a stable sandbox identity, even before a sandbox row arrives. + effects := []string{} + for _, target := range byKey { + if target.S("kind") == "effect" { + effects = append(effects, target.S("id")) + } + } + for _, id := range effects { + e := t.one("SELECT * FROM external_effects WHERE id=$1", id) + if e != nil && e.S("kind") == "sandbox_ensure" { + add(Object{"kind": "sandbox", "id": id, "ensureEffectId": id, "operationId": e.S("operationId"), "workspaceId": w.S("id")}) + } + } + for _, s := range t.list("SELECT * FROM sandbox_instances WHERE workspace_id=$1", w.S("id")) { + add(Object{"kind": "sandbox", "id": s.S("id"), "generation": s.N("generation"), "workspaceId": w.S("id")}) + } + for _, n := range t.list("SELECT * FROM node_instances WHERE workspace_id=$1", w.S("id")) { + add(Object{"kind": "node", "id": n.S("id"), "sandboxInstanceId": n.S("sandboxInstanceId"), "nodeId": n.S("nodeId"), "workspaceId": w.S("id")}) + } + for _, e := range t.list("SELECT * FROM node_executions WHERE workspace_id=$1 OR operation_id=$2", w.S("id"), row.S("runId")) { + add(Object{"kind": "execution", "id": e.S("executionId"), "operationId": e.S("operationId"), "nodeId": e.S("nodeId"), "workId": e.S("workId"), "workspaceId": w.S("id")}) + } + for _, e := range t.list("SELECT * FROM clone_executions WHERE workspace_id=$1", w.S("id")) { + add(Object{"kind": "clone", "id": e.S("executionId"), "operationId": e.S("operationId"), "nodeId": e.S("nodeId"), "workspaceId": w.S("id")}) + } + for _, work := range t.list("SELECT * FROM execution_work WHERE run_id=$1", row.S("runId")) { + add(Object{"kind": "work", "id": work.S("id"), "workspaceId": w.S("id")}) + if sid := work.O("target").S("sandboxInstanceId"); sid != "" { + add(Object{"kind": "sandbox", "id": sid, "workspaceId": w.S("id"), "generation": work.O("target").N("runtimeGeneration")}) + } + } + add(Object{"kind": "data", "id": w.S("id"), "workspaceId": w.S("id")}) + keys := make([]string, 0, len(byKey)) + for key := range byKey { + keys = append(keys, key) + } + sort.Strings(keys) + targets := make([]Object, 0, len(keys)) + for _, key := range keys { + targets = append(targets, byKey[key]) + } + return targets +} + +func refreshResidualTargets(t *transaction, row Object) Object { + targets := collectResidualTargets(t, row) + if jsonText(targets) != jsonText(row["reclaimTargets"]) { + t.exec("UPDATE agent_run_residuals SET reclaim_targets=$2,reclaim_version=reclaim_version+1 WHERE run_id=$1", row.S("runId"), jsonText(targets)) + } + return t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", row.S("runId")) +} + +// residualSandboxTerminated requires the two distinct physical assertions for the exact stable +// sandbox ID. terminated_at, failed ensures, elapsed time and a missing sandbox row prove neither. +func residualSandboxTerminated(t *transaction, row Object, sid string) bool { + for _, e := range t.list(`SELECT * FROM external_effects WHERE project_id=$1 AND workspace_id=$2 + AND kind='sandbox_terminate' AND request->>'sandboxInstanceId'=$3 AND state='succeeded'`, row.S("projectId"), row.S("workspaceId"), sid) { + if e.O("result").B("terminated") && e.O("result").B("lateEnsureFenced") { + return true + } + } + return false +} + +func residualAllSandboxesTerminated(t *transaction, row Object) bool { + count := 0 + for _, target := range residualObjects(row["reclaimTargets"]) { + if target.S("kind") == "sandbox" { + count++ + if !residualSandboxTerminated(t, row, target.S("id")) { + return false + } + } + } + return count > 0 +} + +// residualTargetsLocated checks references against the real tables. Unknown historical references +// cannot be normalized to the newest generation or dropped from the responsibility. +func residualTargetsLocated(t *transaction, row Object) bool { + for _, target := range residualObjects(row["reclaimTargets"]) { + id, wid := target.S("id"), row.S("workspaceId") + switch target.S("kind") { + case "sandbox": + s := t.one("SELECT * FROM sandbox_instances WHERE id=$1 AND workspace_id=$2", id, wid) + if s == nil { + e := t.one(`SELECT e.id FROM external_effects e JOIN operations o ON o.id=e.operation_id + WHERE e.id=$1 AND e.kind='sandbox_ensure' AND e.project_id=$2 AND o.tenant_id=$3 + AND (e.workspace_id=$4 OR (e.workspace_id IS NULL AND o.workspace_id=$4))`, id, row.S("projectId"), row.S("tenantId"), wid) + if e == nil { + return false + } + } else if target.N("generation") > 0 && target.N("generation") != s.N("generation") { + return false + } + case "operation": + if t.one("SELECT id FROM operations WHERE id=$1 AND tenant_id=$2 AND project_id=$3 AND (workspace_id=$4 OR workspace_id IS NULL)", id, row.S("tenantId"), row.S("projectId"), wid) == nil { + return false + } + case "effect": + if t.one(`SELECT e.id FROM external_effects e JOIN operations o ON o.id=e.operation_id + WHERE e.id=$1 AND o.tenant_id=$2 AND e.project_id=$3 AND (e.workspace_id=$4 OR (e.workspace_id IS NULL AND o.workspace_id=$4))`, id, row.S("tenantId"), row.S("projectId"), wid) == nil { + return false + } + case "node": + if t.one("SELECT id FROM node_instances WHERE id=$1 AND workspace_id=$2 AND sandbox_instance_id=$3", id, wid, target.S("sandboxInstanceId")) == nil { + return false + } + case "execution": + if t.one("SELECT execution_id FROM node_executions WHERE execution_id=$1 AND workspace_id=$2", id, wid) == nil { + return false + } + case "clone": + if t.one("SELECT execution_id FROM clone_executions WHERE execution_id=$1 AND workspace_id=$2", id, wid) == nil { + return false + } + case "work": + if t.one("SELECT id FROM execution_work WHERE id=$1 AND run_id=$2 AND target->>'workspaceId'=$3", id, row.S("runId"), wid) == nil { + return false + } + case "data": + if id != wid { + return false + } + default: + return false + } + } + return true +} + +func residualExecutionsTerminal(t *transaction, row Object) bool { + for _, target := range residualObjects(row["reclaimTargets"]) { + var e Object + switch target.S("kind") { + case "execution": + e = t.one("SELECT * FROM node_executions WHERE execution_id=$1", target.S("id")) + case "clone": + e = t.one("SELECT * FROM clone_executions WHERE execution_id=$1", target.S("id")) + default: + continue + } + if e == nil || e.O("result").S("outcome") == "" { + return false + } + } + return true +} + +func residualInputClosed(t *transaction, row Object) bool { + c := t.one("SELECT * FROM runtime_controls WHERE workspace_id=$1", row.S("workspaceId")) + if c == nil || !c.B("inputClosed") { + return false + } + for _, receipt := range residualObjects(row["reclaimEvidence"]) { + fact := receipt.O("fact") + if fact.S("kind") != "runtime_closed" || !fact.B("inputClosed") || fact.S("workspaceId") != row.S("workspaceId") || + fact.N("controlEpoch") != c.N("controlEpoch") || fact.S("sandboxId") != c.S("boundSandboxId") { + continue + } + // The full list is retained by the verified runtime_ack path; an omitted list is unknown. + ids, exists := fact["unfinishedExecutionIds"] + if !exists { + continue + } + switch values := ids.(type) { + case []any: + if len(values) == 0 { + return true + } + case []string: + if len(values) == 0 { + return true + } + } + } + return false +} + +func residualDeliveryComplete(t *transaction, row Object) bool { + // There is presently no durable explicit delivery-abandonment record type. Startup/creation + // abandoned, failed/skipped, timeout and revision_delivery_skips are not such a record. Until + // an existing explicit source is identified, only the already verified Revision path qualifies. + verified := t.one(`SELECT r.id FROM revisions r JOIN revision_verifications v ON v.execution_id=r.execution_id + JOIN node_executions e ON e.execution_id=r.execution_id JOIN execution_work ew ON ew.id=e.work_id + WHERE r.run_id=$1 AND r.tenant_id=$2 AND r.project_id=$3 AND r.workspace_id=$4 + AND e.operation_id=r.run_id AND e.workspace_id=r.workspace_id AND e.kind='deliver_revision' + AND ew.run_id=r.run_id AND ew.kind='deliver_revision' AND ew.execution_id=e.execution_id + AND v.outcome IN ('delivered','unchanged') AND e.result->>'outcome' IN ('revision_delivered','revision_unchanged')`, row.S("runId"), row.S("tenantId"), row.S("projectId"), row.S("workspaceId")) + if verified == nil { + return false + } + // Every actual delivery must have its own successful verification; a first saved result cannot + // cover a different late delivery that still needs rescue. + return t.one(`SELECT e.execution_id FROM node_executions e WHERE e.workspace_id=$1 AND e.kind='deliver_revision' + AND NOT EXISTS(SELECT 1 FROM revision_verifications v JOIN revisions r ON r.execution_id=v.execution_id + WHERE v.execution_id=e.execution_id AND v.outcome IN ('delivered','unchanged') AND r.run_id=$2 + AND r.workspace_id=$1 AND r.tenant_id=$3 AND r.project_id=$4)`, row.S("workspaceId"), row.S("runId"), row.S("tenantId"), row.S("projectId")) == nil +} + +func residualNeverPlanned(t *transaction, row Object) bool { + for _, target := range residualObjects(row["reclaimTargets"]) { + switch target.S("kind") { + case "sandbox", "node", "execution", "clone", "work": + return false + case "effect": + e := t.one("SELECT kind FROM external_effects WHERE id=$1", target.S("id")) + if e == nil || e.S("kind") == "sandbox_ensure" { + return false + } + } + } + return true +} + +// residualEligibility is deliberately conservative about additional historical IDs: only the one +// actual current sandbox may be passed to owned delete. Every other ID needs prior dual confirmation. +func residualEligibility(t *transaction, row Object) bool { + _, w, p := residualIdentity(t, row) + if row["reclaimConflictedAt"] != nil || w.B("admissionOpen") || p.S("lifecycle") != "active" || !residualTargetsLocated(t, row) || !residualDeliveryComplete(t, row) { + return false + } + allGone := residualAllSandboxesTerminated(t, row) + if !allGone && !residualNeverPlanned(t, row) && (!residualInputClosed(t, row) || !residualExecutionsTerminal(t, row)) { + return false + } + for _, target := range residualObjects(row["reclaimTargets"]) { + if target.S("kind") != "sandbox" || residualSandboxTerminated(t, row, target.S("id")) { + continue + } + if t.one("SELECT id FROM sandbox_instances WHERE id=$1 AND workspace_id=$2 AND generation=$3 AND terminated_at IS NULL", target.S("id"), w.S("id"), row.N("runtimeGeneration")) == nil { + return false + } + } + return t.one("SELECT id FROM execution_tickets WHERE workspace_id=$1 AND state='active' AND terminated_by_force_stop_id IS NULL", w.S("id")) == nil +} diff --git a/internal/core/agent_run_residual_reclaim.go b/internal/core/agent_run_residual_reclaim.go new file mode 100644 index 00000000..3aad6640 --- /dev/null +++ b/internal/core/agent_run_residual_reclaim.go @@ -0,0 +1,207 @@ +package core + +// claimResidualReclamation is the single serialized arbitration point for automatic cleanup and +// a future already-authorized controlled intent. It is internal, not a caller-selectable public +// delete flag. Controlled callers must have authenticated/persisted their intent before entry; +// this step implements only its reservation, never a controlled action or effect. +func claimResidualReclamation(t *transaction, runID, mode string, authorization Object) Object { + require(mode == "owned_delete" || mode == "controlled", 400, "invalid_reclaim_mode") + row := t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", runID) + _, w, p := residualIdentity(t, row) + require(row["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") + if row.S("reclaimState") != "reconciling" { + require(row.S("reclaimMode") == mode, 409, "residual_mode_conflict") + return Object{"residual": row, "operation": residualBoundOperation(t, row), "claimed": false} + } + row = refreshResidualTargets(t, row) + require(!w.B("admissionOpen") && w["deletedAt"] == nil && p.S("lifecycle") == "active", 409, "resource_unavailable") + if t.one("SELECT id FROM operations WHERE project_id=$1 AND state IN ('queued','running','retry_wait','blocked')", p.S("id")) != nil { + return Object{"busy": true, "residual": row} + } + requireNoForceStop(t, w.S("id")) + if mode == "owned_delete" && !residualEligibility(t, row) { + return Object{"eligible": false, "residual": row} + } + if mode == "controlled" { + // No boolean or arbitrary nonempty object is an authority. The future authenticated entry + // passes the stable identity of its already accepted operator intent. This reservation by + // itself does not authorize a terminate, data-delete, new operation or final disposition. + require(authorization.S("kind") == "controlled_intent" && validID(authorization.S("intentId")) && authorization.S("source") != "", 409, "controlled_intent_required") + } + id := newID() + auth := Object{"kind": "residual_owned_delete", "runId": runID, "workspaceId": w.S("id"), "tenantId": row.S("tenantId"), "projectId": p.S("id"), "runtimeGeneration": row.N("runtimeGeneration"), "evidenceVersion": row.N("reclaimVersion")} + if mode == "controlled" { + auth["kind"], auth["intentId"], auth["source"] = "controlled_intent", authorization.S("intentId"), authorization.S("source") + } + auth["claimedAt"] = t.one("SELECT clock_timestamp()::text AS at").S("at") + var op Object + if mode == "owned_delete" { + snapshot := Object{} + for key, value := range w { + if key != "issueRunId" { + snapshot[key] = value + } + } + closeAdmission(t, w, "deleted") + request := Object{"previous": Object{w.S("id"): snapshot}, "reclaimId": id} + op = newOperation(t, &PublicRequest{TenantID: row.S("tenantId"), Key: "residual-" + id}, w.S("creatorUserId"), p.S("id"), w.S("id"), "delete_workspace", "quiesce", requestHash("residual-owned-delete", runID, Object{"reclaimId": id}), request) + reserveRuntimeMaintenance(t, w.S("id"), op.S("id")) + } + t.exec(`UPDATE agent_run_residuals SET reclaim_state='reclaiming',reclaim_id=$2,reclaim_mode=$3, + reclaim_operation_id=$4,reclaim_authorization=$5,reclaim_version=reclaim_version+1 WHERE run_id=$1 AND reclaim_state='reconciling'`, runID, id, mode, nullable(op.S("id")), jsonText(auth)) + return Object{"claimed": true, "residual": t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", runID), "operation": op} +} + +func authorizeResidualOwnedDelete(t *transaction, runID string) Object { + return claimResidualReclamation(t, runID, "owned_delete", nil) +} + +func residualBoundOperation(t *transaction, row Object) Object { + if row.S("reclaimOperationId") == "" { + return nil + } + return t.one("SELECT * FROM operations WHERE id=$1", row.S("reclaimOperationId")) +} + +// residualOperation resolves a persisted binding, including a cleared row for idempotent fact +// reception. A request's reclaimId without the database binding can never obtain this authority. +func residualOperation(t *transaction, op Object) Object { + if op == nil { + return nil + } + return t.one("SELECT * FROM agent_run_residuals WHERE reclaim_operation_id=$1", op.S("id")) +} + +// checkResidualOperation is called before plan, permit and advance, even when returning an existing +// effect. Ordinary operations return nil only when their Workspace has no residual responsibility. +// A conflicting cleared record remains a durable block, rather than reopening ordinary cleanup. +func checkResidualOperation(t *transaction, op Object) Object { + row := residualOperation(t, op) + if row == nil { + if op.S("workspaceId") != "" { + other := residualForWorkspace(t, op.S("workspaceId")) + require(other == nil || (other.S("reclaimState") == "cleared" && other["reclaimConflictedAt"] == nil), 409, "residual_operation_required") + } + require(op.O("request").S("reclaimId") == "", 409, "residual_operation_required") + return nil + } + _, w, _ := residualIdentity(t, row) + require(row.S("reclaimState") == "reclaiming" && row.S("reclaimMode") == "owned_delete" && + row["reclaimConflictedAt"] == nil && row["clearedAt"] == nil && op.S("kind") == "delete_workspace" && + op.S("workspaceId") == w.S("id") && op.S("projectId") == row.S("projectId") && op.S("tenantId") == row.S("tenantId") && + op.O("request").S("reclaimId") == row.S("reclaimId") && !w.B("admissionOpen"), 409, "residual_operation_conflict") + row = refreshResidualTargets(t, row) + require(residualTargetsLocated(t, row), 409, "residual_target_unknown") + return row +} + +// checkResidualCleanupActivities exempts only this binding's own residual guard. All other activity +// guards remain. Unknown executions are physically closed only when every writable original +// sandbox/ensure identity is dual-confirmed, never by result=NULL or a ticket timeout. +func checkResidualCleanupActivities(t *transaction, w, op Object) { + row := checkResidualOperation(t, op) + if row == nil { + checkActivities(t, w) + return + } + require(w.S("id") == row.S("workspaceId") && w.S("kind") == "isolated", 409, "residual_identity_conflict") + require(t.one("SELECT run_id FROM agent_run_residuals WHERE workspace_id=$1 AND run_id<>$2 AND cleared_at IS NULL", w.S("id"), row.S("runId")) == nil, 409, "resource_in_use") + require(t.one("SELECT id FROM execution_tickets WHERE workspace_id=$1 AND state='active' AND terminated_by_force_stop_id IS NULL", w.S("id")) == nil, 409, "resource_in_use") + if !residualAllSandboxesTerminated(t, row) { + require(residualExecutionsTerminal(t, row), 409, "resource_in_use") + } + require(residualDeliveryComplete(t, row), 409, "residual_delivery_unknown") +} + +// residualDeletionReady is the additional per-effect/permit gate. Planning the one current +// sandbox's termination is allowed; cleanup waits for every original ID, including historical +// ensures which never produced a sandbox row. Additional unconfirmed IDs get no delete effects. +func residualDeletionReady(t *transaction, row Object) bool { + if row == nil || row["reclaimConflictedAt"] != nil || !residualTargetsLocated(t, row) || !residualDeliveryComplete(t, row) { + return false + } + for _, target := range residualObjects(row["reclaimTargets"]) { + if target.S("kind") == "sandbox" && !residualSandboxTerminated(t, row, target.S("id")) { + return false + } + } + return true +} + +// checkResidualEffect authorizes only the owned-delete topology and the original current sandbox. +// Use this in both planEffect and effectPermit; pass the actual persisted effect for a permit. +func checkResidualEffect(t *transaction, op Object, kind, workspaceID string, effect Object) { + row := checkResidualOperation(t, op) + if row == nil { + return + } + require(workspaceID == row.S("workspaceId"), 409, "residual_identity_conflict") + w := t.one("SELECT * FROM workspaces WHERE id=$1", workspaceID) + checkResidualCleanupActivities(t, w, op) + require(kind == "sandbox_terminate" || kind == "workspace_data_delete", 409, "residual_effect_forbidden") + if kind == "workspace_data_delete" { + require(residualDeletionReady(t, row), 409, "termination_unconfirmed") + return + } + sid := "" + if effect != nil { + sid = effect.O("request").S("sandboxInstanceId") + } + current := t.one("SELECT * FROM sandbox_instances WHERE workspace_id=$1 AND terminated_at IS NULL", workspaceID) + if sid == "" { + sid = current.S("id") + } + require(sid != "" && (current.S("id") == sid || residualSandboxTerminated(t, row, sid)), 409, "residual_target_unknown") + found := false + for _, target := range residualObjects(row["reclaimTargets"]) { + if target.S("kind") != "sandbox" { + continue + } + if target.S("id") == sid { + found = true + continue + } + require(residualSandboxTerminated(t, row, target.S("id")), 409, "residual_target_unknown") + } + require(found, 409, "residual_target_unknown") +} + +// completeResidualReclamation is called from cleanup->done in the SAME transaction that marks the +// operation succeeded and soft-deletes the Workspace. Failure rolls those writes back together. +func completeResidualReclamation(t *transaction, op Object) bool { + row := residualOperation(t, op) + if row == nil { + return true + } + if row.S("reclaimState") == "cleared" { + require(row["reclaimConflictedAt"] == nil && row.S("disposition") == "released", 409, "residual_fact_conflict") + return true + } + row = checkResidualOperation(t, op) + require(residualDeletionReady(t, row), 409, "residual_cleanup_incomplete") + effect := t.one(`SELECT * FROM external_effects WHERE operation_id=$1 AND workspace_id=$2 AND kind='workspace_data_delete' + AND state='succeeded' AND reconciled_epoch=$3`, op.S("id"), row.S("workspaceId"), op.N("controllerEpoch")) + require(effect != nil && effect.O("result").B("removed"), 409, "residual_cleanup_incomplete") + if !appendResidualFact(t, row.S("runId"), "reclaim:"+row.S("reclaimId")+":complete", Object{"kind": "reclaim_complete", "state": "succeeded", "reclaimId": row.S("reclaimId"), "operationId": op.S("id"), "effectId": effect.S("id"), "disposition": "released"}) { + return false + } + t.exec(`UPDATE agent_run_residuals SET reclaim_state='cleared',disposition='released',cleared_at=clock_timestamp(), + reclaim_version=reclaim_version+1 WHERE run_id=$1 AND reclaim_state='reclaiming' AND reclaim_conflicted_at IS NULL`, row.S("runId")) + return true +} + +// resumeResidualReclamation keeps the original authority and operation across process restarts. +// retry_wait is already claimable by the normal Controller path. A historically failed operation +// is never revived here, and neither a failed query nor quiesce refusal creates another operation. +func resumeResidualReclamation(t *transaction, row Object) Object { + if row.S("reclaimMode") != "owned_delete" || row["reclaimConflictedAt"] != nil { + return Object{"residual": row} + } + op := residualBoundOperation(t, row) + require(op != nil, 409, "residual_operation_missing") + checkResidualOperation(t, op) + if op.S("state") == "queued" || op.S("state") == "retry_wait" || op.S("state") == "blocked" || op.S("state") == "running" { + t.queued = append(t.queued, op.S("id")) + } + return Object{"residual": row, "operation": op} +} diff --git a/internal/core/control.go b/internal/core/control.go index 74a3e107..029109f6 100644 --- a/internal/core/control.go +++ b/internal/core/control.go @@ -249,6 +249,8 @@ func planEffect(t *transaction, r *ControlRequest, o Object) Object { reconciled(t, o) kind, wid := r.Body.S("kind"), r.Body.S("workspaceId") require(validID(wid), 400, "invalid_effect_scope") + requireNoResidualConflict(t, wid) + checkResidualEffect(t, o, kind, wid, nil) found := false for _, w := range operationWorkspaces(t, o) { if w.S("id") == wid { @@ -377,6 +379,7 @@ func completedEffect(t *transaction, o Object, kind, wid string) Object { func advance(t *transaction, r *ControlRequest, o Object) Object { reconciled(t, o) + checkResidualOperation(t, o) next := "" wid := o.S("workspaceId") switch o.S("step") { @@ -407,7 +410,7 @@ func advance(t *transaction, r *ControlRequest, o Object) Object { require(!repositoryOperationsUnfinished(t, o.S("projectId")), 409, "repository_operation_unfinished") } for _, w := range operationWorkspaces(t, o) { - checkActivities(t, w) + checkResidualCleanupActivities(t, w, o) live := t.one("SELECT id FROM sandbox_instances WHERE workspace_id=$1 AND terminated_at IS NULL", w.S("id")) if live != nil { require(t.one("SELECT id FROM node_instances WHERE sandbox_instance_id=$1 AND ended_at IS NULL AND idle_admission_epoch=$2 AND connection_state='connected' AND last_seen_at>clock_timestamp()-interval '30 seconds'", live.S("id"), w.N("admissionEpoch")) != nil, 409, "idle_unconfirmed") @@ -419,6 +422,9 @@ func advance(t *transaction, r *ControlRequest, o Object) Object { live := t.one("SELECT id FROM sandbox_instances WHERE workspace_id=$1 AND terminated_at IS NULL", w.S("id")) if live != nil { completedEffect(t, o, "sandbox_terminate", w.S("id")) + if row := residualOperation(t, o); row != nil { + require(residualSandboxTerminated(t, row, live.S("id")), 409, "termination_unconfirmed") + } t.exec("UPDATE node_instances SET ended_at=now(),connection_state='ended',version=version+1 WHERE sandbox_instance_id=$1 AND ended_at IS NULL", live.S("id")) t.exec("UPDATE sandbox_instances SET observed_state='terminated',terminated_at=now() WHERE id=$1", live.S("id")) } @@ -435,6 +441,9 @@ func advance(t *transaction, r *ControlRequest, o Object) Object { next = "cleanup" } case "cleanup": + if !completeResidualReclamation(t, o) { + return t.one("SELECT * FROM operations WHERE id=$1", o.S("id")) + } // Each Workspace's data was deleted by its own effect, planned only after its sandbox ended. for _, w := range operationWorkspaces(t, o) { completedEffect(t, o, "workspace_data_delete", w.S("id")) diff --git a/internal/core/node.go b/internal/core/node.go index f2f81f95..1a83fb42 100644 --- a/internal/core/node.go +++ b/internal/core/node.go @@ -115,7 +115,7 @@ func nodeStatus(t *transaction, r *ControlRequest, n, w Object) { // A business-abandoned execution still owns residual resources after it reconnects; // connection evidence must not reopen their admission or imply physical cleanup. t.exec(`UPDATE workspaces SET observed_state='ready',admission_open=NOT EXISTS( - SELECT 1 FROM agent_run_residuals WHERE workspace_id=$1 AND cleared_at IS NULL), + SELECT 1 FROM agent_run_residuals WHERE workspace_id=$1 AND (cleared_at IS NULL OR reclaim_conflicted_at IS NOT NULL)), version=version+1 WHERE id=$1`, w.S("id")) } } @@ -139,7 +139,7 @@ func nodeIdle(t *transaction, r *ControlRequest, n, w Object) Object { restoreAdmission(t, o) return Object{"accepted": false, "errorCode": "resource_in_use"} } - checkActivities(t, w) + checkResidualCleanupActivities(t, w, o) c := runtimeControl(t, w.S("id")) require(c.S("state") == "maintenance" && c.S("maintenanceOperationId") == o.S("id") && c.B("bindingConfirmed") && c.S("boundSandboxId") == n.S("sandboxInstanceId"), 409, "runtime_input_closure_unconfirmed") require(n.B("initialized") && n.S("connectionState") == "connected", 409, "idle_unconfirmed") @@ -148,6 +148,17 @@ func nodeIdle(t *transaction, r *ControlRequest, n, w Object) Object { } func restoreAdmission(t *transaction, o Object) { + if row := residualOperation(t, o); row != nil { + checkResidualOperation(t, o) + // The original authority survives refusal. A new binding acknowledgment and idle report + // are required before this SAME operation can retry; admission never reopens. + t.exec(`UPDATE operations SET state='retry_wait',error_code='resource_in_use',retry_at=clock_timestamp(), + version=version+1,updated_at=clock_timestamp() WHERE id=$1`, o.S("id")) + t.exec(`UPDATE runtime_controls SET binding_confirmed=false,input_closed=false,control_epoch=control_epoch+1, + version=version+1,updated_at=clock_timestamp() WHERE workspace_id=$1`, row.S("workspaceId")) + t.exec("UPDATE node_instances SET idle_admission_epoch=NULL WHERE workspace_id=$1", row.S("workspaceId")) + return + } for wid, v := range o.O("request").O("previous") { raw, ok := v.(map[string]any) require(ok, 500, "internal_error") diff --git a/internal/core/public.go b/internal/core/public.go index ca74f9ca..9c299711 100644 --- a/internal/core/public.go +++ b/internal/core/public.go @@ -207,6 +207,7 @@ func (s *Store) Public(ctx context.Context, r *PublicRequest) (Object, int, erro require(r.Method == "DELETE", 405, "method_not_allowed") requireSpaceRole(m, "admin") version(p, r.Body.N("version")) + require(t.one("SELECT run_id FROM agent_run_residuals WHERE project_id=$1 AND reclaim_conflicted_at IS NOT NULL", p.S("id")) == nil, 409, "residual_fact_conflict") idleProject(t, p.S("id")) require(p.S("lifecycle") == "active", 409, "resource_unavailable") ws := t.list("SELECT * FROM workspaces WHERE project_id=$1 AND deleted_at IS NULL ORDER BY id", p.S("id")) @@ -713,6 +714,7 @@ func newOperation(t *transaction, r *PublicRequest, uid, pid, wid, kind, step, h } func checkActivities(t *transaction, w Object) { + requireNoResidualConflict(t, w.S("id")) require(t.one("SELECT run_id FROM agent_run_residuals WHERE workspace_id=$1 AND cleared_at IS NULL", w.S("id")) == nil, 409, "resource_in_use") require(t.one("SELECT id FROM execution_tickets WHERE workspace_id=$1 AND state='active' AND terminated_by_force_stop_id IS NULL", w.S("id")) == nil, 409, "resource_in_use") require(t.one("SELECT execution_id FROM clone_executions WHERE workspace_id=$1 AND result IS NULL AND terminated_by_force_stop_id IS NULL", w.S("id")) == nil, 409, "resource_in_use") diff --git a/internal/core/run_workspace.go b/internal/core/run_workspace.go index 40e19348..943ed1db 100644 --- a/internal/core/run_workspace.go +++ b/internal/core/run_workspace.go @@ -115,6 +115,8 @@ func deleteWorkflowRunWorkspace(t *transaction, runID string) Object { func deleteRunWorkspace(t *transaction, runID string) Object { w := t.one("SELECT * FROM workspaces WHERE issue_run_id=$1", runID) require(w != nil, 404, "not_found") + requireNoResidualConflict(t, w.S("id")) + require(t.one("SELECT run_id FROM agent_run_residuals WHERE workspace_id=$1 AND cleared_at IS NULL", w.S("id")) == nil, 409, "resource_in_use") if w["deletedAt"] != nil { return Object{"busy": false, "workspace": w} } @@ -124,7 +126,6 @@ func deleteRunWorkspace(t *transaction, runID string) Object { if t.one("SELECT id FROM operations WHERE project_id=$1 AND state IN ('queued','running','retry_wait','blocked')", w.S("projectId")) != nil { return Object{"busy": true} } - require(t.one("SELECT run_id FROM agent_run_residuals WHERE workspace_id=$1 AND cleared_at IS NULL", w.S("id")) == nil, 409, "resource_in_use") closeAdmission(t, w, "deleted") actor := w.S("creatorUserId") // The request carries the pre-declaration Workspace row, exactly as the public delete path does diff --git a/internal/core/runtime_force_stop.go b/internal/core/runtime_force_stop.go index 3c4458a6..7627d765 100644 --- a/internal/core/runtime_force_stop.go +++ b/internal/core/runtime_force_stop.go @@ -102,6 +102,9 @@ func effectPermit(t *transaction, r *ControlRequest) Object { } else { e := t.one("SELECT e.*,o.state AS operation_state,o.controller_epoch,w.tenant_id,w.runtime_generation FROM external_effects e JOIN operations o ON o.id=e.operation_id JOIN workspaces w ON w.id=e.workspace_id WHERE e.id=$1", r.EffectID) require(e != nil && e.S("operationState") == "running" && e.N("controllerEpoch") == r.Body.N("epoch"), 409, "stale_effect") + op := t.one("SELECT * FROM operations WHERE id=$1", e.S("operationId")) + requireNoResidualConflict(t, e.S("workspaceId")) + checkResidualEffect(t, op, e.S("kind"), e.S("workspaceId"), e) wid, tid, generation = e.S("workspaceId"), e.S("tenantId"), e.N("runtimeGeneration") requireNoForceStop(t, wid) c := runtimeControl(t, wid) From 5e624ace98c340c341ecf81b469a4b1f8d679651 Mon Sep 17 00:00:00 2001 From: creegon2 <315603738+creegon2@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:51:14 -0700 Subject: [PATCH 04/11] feat(server): reconcile residual cleanup independently of run settlement Scan reconciling and reclaiming rows fairly in bounded batches, preserve original responsibility on restart, and join the existing server cancellation lifecycle. Add a validated one-minute reconciliation interval and paired Chinese/English documentation. Document missing explicit abandonment and existing evidence-discovery limits; add no RPC, proto regeneration, physical termination mechanism or controlled management endpoint. --- cmd/server/README.en.md | 2 + cmd/server/README.md | 2 + cmd/server/main.go | 9 ++ config.toml.template | 5 ++ docs/README.en.md | 1 + docs/README.md | 1 + docs/agent-runs/issue-runs.en.md | 7 ++ docs/agent-runs/issue-runs.md | 5 ++ docs/agent-runs/residual-reclamation.en.md | 98 ++++++++++++++++++++++ docs/agent-runs/residual-reclamation.md | 83 ++++++++++++++++++ internal/config/README.en.md | 2 + internal/config/README.md | 2 + internal/config/config.go | 11 +++ internal/config/residual_reconcile_test.go | 60 +++++++++++++ internal/core/README.en.md | 8 ++ internal/core/README.md | 6 ++ internal/core/agent_run_residual_loop.go | 55 ++++++++++++ 17 files changed, 357 insertions(+) create mode 100644 docs/agent-runs/residual-reclamation.en.md create mode 100644 docs/agent-runs/residual-reclamation.md create mode 100644 internal/config/residual_reconcile_test.go create mode 100644 internal/core/agent_run_residual_loop.go diff --git a/cmd/server/README.en.md b/cmd/server/README.en.md index 531f6513..57fa4494 100644 --- a/cmd/server/README.en.md +++ b/cmd/server/README.en.md @@ -16,6 +16,8 @@ The production entrypoint composes configuration, databases, HTTP/gRPC adapters `main.go` owns loops with one cancellation context and WaitGroup: optional catalog sync and workflow asset imports, Agent dispatch/startup, blocked windows, Thread end/cancellation, delivery/release recovery and Workflow scheduling. Core methods still decide state; loops do not make process-local state authoritative. +The residual-ledger recovery loop calls `ReconcileAgentRunResidualsOnce` immediately at startup and then at `issue_runs.residual_reconcile_interval` (default `1m`), sharing that cancellation context and WaitGroup. It retains recovery responsibility after the business run ends and consumes existing fact entrypoints without adding a Controller discovery RPC or contract. Missing evidence leaves the ledger open; timeouts, reconnection and forced stops do not establish successful cleanup. + Cleanup differs between shutdown and failure paths: - Signal-exit branch (SIGINT/SIGTERM): cancel background tasks and send the Controller drain signal; stop gRPC gracefully, forcing it to stop at the shared 10-second shutdown deadline, then call HTTP Shutdown with that same deadline, join background loops, close the database and flush logs. diff --git a/cmd/server/README.md b/cmd/server/README.md index a91511f7..ff74c691 100644 --- a/cmd/server/README.md +++ b/cmd/server/README.md @@ -16,6 +16,8 @@ `main.go` 用同一个取消 context 与 WaitGroup 拥有循环:可选市场同步和工作流资产导入、Agent 派发/启动、blocked 窗口、Thread 结束/取消、交付/释放恢复,以及 Workflow 调度。具体状态判定仍由各 Core 方法负责;循环不把进程内状态当作权威。 +残留账本恢复循环调用 `ReconcileAgentRunResidualsOnce`,启动即运行一次,之后按 `issue_runs.residual_reconcile_interval`(缺省 `1m`)重试,并参与同一 context 的取消与 WaitGroup 等待。它在业务运行结束后继续承担恢复责任,只消费既有事实入口,不新增 Controller 发现 RPC 或契约;证据收不到就保留 open,不按超时、重连或强停推定清理成功。 + 停机与失败路径的清理范围不同: - 信号退出分支(SIGINT/SIGTERM):取消后台任务,发送 Controller drain 信号;gRPC 先优雅停止,到达与 HTTP 共用的 10 秒停机 deadline 时强制停止,然后调用 HTTP Shutdown,等待后台循环结束,再关闭数据库和刷新日志。 diff --git a/cmd/server/main.go b/cmd/server/main.go index 35ac1b54..81ae0bcc 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -259,6 +259,15 @@ func run() (runErr error) { pluginmarket.RunSyncLoop(ctx, store.GiveUpStaleWorkspaceReleasesOnce, agentDeliveryInterval, pluginmarket.ContextSleep, log) }() } + // Residual recovery remains independent of a run's business terminal state. Cloud consumes + // existing, identity-bound facts; a missing fact leaves its ledger open rather than treating + // a timer or reconnection as cleanup proof. This loop shares cancellation and shutdown joining + // with the delivery/release passes above and has no disable switch. + syncGroup.Add(1) + go func() { + defer syncGroup.Done() + pluginmarket.RunSyncLoop(ctx, store.ReconcileAgentRunResidualsOnce, cfg.IssueRuns.ResidualReconcileInterval, pluginmarket.ContextSleep, log) + }() // Workflow advance loop: the Cloud-side scheduler (specs#99 D1 条件②/④, landing order step 3). // A bounded rescan of pending and running workflow runs, each advanced in its own short // transaction; the ready set is recomputed from committed state only, so the pass is idempotent diff --git a/config.toml.template b/config.toml.template index 49cec212..8ef9759c 100644 --- a/config.toml.template +++ b/config.toml.template @@ -8,6 +8,11 @@ # `task setup` (cmd/devsetup) only turns the values below into what those services already accept: # CLOUD_*/GATEWAY_* environment overrides and TEST_DATABASE_URL in .local/dev.env, which # Taskfile.yml loads for every task, plus the client secret file the Gateway reads. +# Cloud's residual recovery pass runs every 1m by default. Configure +# issue_runs.residual_reconcile_interval in the Cloud YAML, or set +# CLOUD_ISSUE_RUNS_RESIDUAL_RECONCILE_INTERVAL in the server environment. This TOML setup +# template does not accept an [issue_runs] section. The interval must be positive; it is not +# a cleanup-success deadline and there is no disable switch. [database] # PostgreSQL DSN shared by Cloud, the Gateway, `task migrate` and the test suite. Any PostgreSQL 17 diff --git a/docs/README.en.md b/docs/README.en.md index f6697829..5f6f72e3 100644 --- a/docs/README.en.md +++ b/docs/README.en.md @@ -50,6 +50,7 @@ package live in that directory's `README.md`; the authoritative source for busin - [Agent IssueRuns: an Agent run from start to finish](agent-runs/issue-runs.en.md) - [Threads: the conversation of an Agent run](agent-runs/threads.en.md) - [Bounded startup: waiting limits and settling unknown outcomes](agent-runs/startup-settlement.en.md) +- [Agent Run residual reclamation: Cloud authority and evidence](agent-runs/residual-reclamation.en.md) - [Revisions: delivery, verified registration and resume](agent-runs/revisions.en.md) - [Session reads: session lists, archives and model request statistics](agent-runs/session-reads.en.md) diff --git a/docs/README.md b/docs/README.md index 5d2d4ba6..60d87111 100644 --- a/docs/README.md +++ b/docs/README.md @@ -48,6 +48,7 @@ - [Agent IssueRun:一次 Agent 运行的全过程](agent-runs/issue-runs.md) - [Thread:Agent 运行的对话](agent-runs/threads.md) - [有界启动:等待上限与未知结果的收尾](agent-runs/startup-settlement.md) +- [Agent Run 残留回收:Cloud 责任与证据](agent-runs/residual-reclamation.md) - [Revision:交付、校验登记与续接](agent-runs/revisions.md) - [会话读取:会话列表、归档与模型请求统计](agent-runs/session-reads.md) diff --git a/docs/agent-runs/issue-runs.en.md b/docs/agent-runs/issue-runs.en.md index 6536eaee..a6e22c0d 100644 --- a/docs/agent-runs/issue-runs.en.md +++ b/docs/agent-runs/issue-runs.en.md @@ -20,6 +20,7 @@ This document is the overview; individual parts are covered in: | Creation (@Agent, the Project gate, the input snapshot) | [Issue collaboration](../collaboration/interactions.en.md#task-mode-agent) | | Thread entries, user turns, idleness and ending | [Threads](threads.en.md) | | Startup waiting limits and settling unknown outcomes | [Bounded startup](startup-settlement.en.md) | +| D4/D5 residual ledger, system cleanup responsibility and evidence boundaries | [Residual reclamation](residual-reclamation.en.md) | | Object storage, upload grants, Revision registration and resume | [Revisions](revisions.en.md) | | Session lists, archives and model request statistics | [Session reads](session-reads.en.md) | @@ -67,6 +68,12 @@ and it accepts no other run or interactive session. ## Background loops +After D4/D5 business settlement, independent ledger reconciliation continues and grants one durable system owned-delete responsibility only +when evidence qualifies. Retries keep the same operation; cleanup does not settle the Run again. The current scope is +[Cloud step 1 of residual reclamation](residual-reclamation.en.md). Controlled operator endpoints, retained-data/external dispositions, +Controller/Sandbox Server physical acceptance, joint acceptance and D8 historical takeover remain incomplete. Neither `done` nor a completed +reconciliation pass proves that resources were reclaimed. + Besides the synchronous transitions taken while evidence is taken over, the Cloud server process runs a set of loops every 10 seconds for the progress and settlement that have no external event to follow: diff --git a/docs/agent-runs/issue-runs.md b/docs/agent-runs/issue-runs.md index 39995816..cc1b78e9 100644 --- a/docs/agent-runs/issue-runs.md +++ b/docs/agent-runs/issue-runs.md @@ -15,6 +15,7 @@ Cloud 校验并登记之后删除运行 Workspace,释放沙盒和数据。 | 怎么创建(@Agent、Project 关口、输入快照) | [Issue 协作](../collaboration/interactions.md#task-modeagent) | | Thread 条目、用户轮次、空闲与结束 | [Thread](threads.md) | | 启动的等待上限与未知结果收尾 | [有界启动](startup-settlement.md) | +| D4/D5 残留账本、系统清理责任与证据边界 | [残留回收](residual-reclamation.md) | | 对象存储、上传授权、Revision 登记与续接 | [Revision](revisions.md) | | 会话列表、归档与模型请求统计 | [会话读取](session-reads.md) | @@ -49,6 +50,10 @@ Cloud 在接管上一阶段证据的事务里写入。 ## 后台循环 +D4/D5 业务收尾后的残留由独立账本对账继续处理,证据足够才授予一份持久的系统 owned-delete 责任;重试保持同一 operation, +完成回收不重新结算 Run。当前仅覆盖 [残留回收的 Cloud 第 1 步](residual-reclamation.md):受控运维入口、保留数据/外部确认、 +Controller/Sandbox Server 物理验收、跨仓联合验收和 D8 历史接管仍未完成;`done` 或对账已运行均不证明资源已回收。 + 除了接管证据时的同步转换,Cloud server 进程里有一组每 10 秒执行一次的循环,负责那些没有外部事件可跟随的推进和收尾: | 循环 | 作用 | diff --git a/docs/agent-runs/residual-reclamation.en.md b/docs/agent-runs/residual-reclamation.en.md new file mode 100644 index 00000000..a98fd72a --- /dev/null +++ b/docs/agent-runs/residual-reclamation.en.md @@ -0,0 +1,98 @@ +# Agent Run residual reclamation: Cloud authority and evidence + +English | [中文](residual-reclamation.md) + +D4/D5 [bounded startup settlement](startup-settlement.en.md) can finish the business Run while retaining unresolved resources in +`agent_run_residuals`. Cloud continues collecting evidence on that original ledger and only grants system cleanup authority when the +evidence qualifies. Deletion follows the existing owned-delete path. It neither settles the [IssueRun](issue-runs.en.md) again nor reopens its Workspace. + +This page describes the Cloud step 1 implementation boundary. It is not evidence of physical termination by the Controller or Sandbox Server, +deployment acceptance, or joint fault acceptance. The +[residual reclamation ADR](https://github.com/ora-space/specs/blob/main/decisions/cloud/issue-run/20261018-residual-agent-run-reclamation.md) +remains `approved`; this step does not declare the whole ADR implemented. + +## Ledger states and one durable responsibility + +| `reclaim_state` | Meaning | Authority fields | +|---|---|---| +| `reconciling` | Collect original-identity facts; deletion is not yet authorized | ID, mode and operation are NULL; authorization is `{}` | +| `reclaiming` | One durable responsibility has been granted; continue or await evidence | Fixed `reclaim_id`, mode and nonempty authorization; `owned_delete` binds exactly one operation | +| `cleared` | Disposition under the original responsibility is complete | Keep that responsibility, with both `cleared_at` and disposition | + +Step 1 production authorization uses only `owned_delete` and completes only as `released`. `controlled` reserves the shared internal arbitration +point; it is neither a controlled management endpoint nor operator authority. The schema's `terminated_data_retained` and `externally_confirmed` +dispositions are reserved for later controlled paths. + +Automatic responsibility stays bound to the same `delete_workspace` operation. Restarts, lease changes, retries and quiesce refusals do not +create another responsibility or operation, and do not revive a failed historical operation. Ordinary deletion, dispatch and permit paths retain +residual guards; an internal system responsibility is not a public deletion flag. The transaction rechecks the ledger, Run, Workspace, Project and +operation tenant, original bindings and generation; a UUID foreign key alone does not establish that scope. + +The [migration](../../internal/core/migrations/README.en.md) only adds columns. Defaults initialize existing D4/D5 rows as `reconciling`, without +inferring completion, backfilling D8, or changing old `reason`, `responsibility` or business settlement. A legacy row with non-null `cleared_at` +causes an explicit error and rollback pending compatibility handling (A2). + +## Original-identity evidence and deletion eligibility + +`reclaim_targets` is a union of original plans and fact targets: original operations/effects, preallocated sandbox ensure IDs, sandboxes, Nodes, +executions, clones, work and data. Late facts may add original targets; they cannot replace an old target with the current generation or drop an +unknown target because a query did not find it. + +New facts append to `reclaim_evidence` with logical event keys, facts, sources and database receipt times, instead of changing old +`responsibility.lateFacts`. An identical replay reuses its receipt; new evidence, additional targets or authority transitions increment +`reclaim_version`. Evidence intake, authorization, permits and completion use Cloud's existing serialized transaction boundary. Completion must +therefore reread the full target set and latest facts in its own transaction. + +System authorization requires located original identities, closed admission, no conflict, no blocking Project operation or other activity, and +qualifying delivery and execution evidence. Execution evidence includes input closure and an explicit unfinished-execution list for the same +runtime control identity, together with original execution terminals. Per-target termination evidence can also establish physical closure. +A no-target proof applies only when original plans and execution relationships truly do not exist; missing rows, NULL results and elapsed time +do not establish that proof. + +Step 1 accepts delivery eligibility only through a genuinely registered and verified Revision for the original `deliver_revision` execution; +every other actual delivery execution needs its own completed verification. There is currently no unified existing durable source for explicit +delivery abandonment. `failed`, `skipped`, business timeouts, absent object storage and missing results do not imply explicit abandonment (A1). +Entries with insufficient evidence remain open. + +Authorized cleanup follows existing quiesce → terminate → cleanup. A quiesce refusal leaves the original operation in `retry_wait`, retaining +the original authority and guards for evidence collection and retry (A3). It neither restores new execution eligibility, escalates to force-stop, +nor creates another operation. + +Every original sandbox/ensure ID requires a corresponding successful termination result containing both `terminated` and `lateEnsureFenced`. +`terminated_at`, a failed ensure, a 404 or elapsed time alone is insufficient. The current safe target can reuse owned delete; an additional +historical ID without both confirmations remains open, without replacing its identity or dispatching another termination effect (A5). +Data-deletion planning, permits and completion recheck target coverage. One transaction confirms successful data deletion, succeeds the operation, +soft-deletes the Workspace and writes `cleared/released`, without changing the Run's phase/status/result or repeating business settlement activity. + +## Existing fact intake and conflicts + +Cloud uses existing `runtime_ack`, creation-fact, execution-result and Revision intake. This step adds no RPC, proto or Controller discovery +mechanism. After checking the original control epoch/version, Node, sandbox and generation, `runtime_ack` records input closure and the full +`unfinishedExecutionIds` list on the ledger. Session terminals and delivery results are accepted under their original execution/work/resource +identity. A done Run or cleared ledger is not a reason to reject a consistent late fact. Output rescue uses existing Revision verification; +it does not open admission or start a new session or execution. + +Consistent replay preserves the original conclusion. A contradictory terminal first appends a conflict audit and sets `reclaim_conflicted_at`; +the audit and block commit before the stable `409 residual_fact_conflict` is returned (A4). Canonical facts, existing `cleared_at` and disposition +remain intact. Further deletion or reuse is blocked; this step provides no conflict-resolution endpoint. + +## Reconciliation, recovery and remaining scope + +`ReconcileAgentRunResidualsOnce` scans `reconciling/reclaiming` ledger rows independently of whether the business Run is still `releasing`. +Bounded passes rotate by `reclaim_checked_at NULLS FIRST, created_at, run_id`; a checked time is scheduling metadata, not completion evidence. +Each entry rereads targets, eligibility and original bindings. An existing responsibility resumes its original operation, subject to the current +Controller lease and effect permits. Query failures, insufficient evidence and temporary refusals neither clear the ledger nor expand authority. + +If a failed create's original effects have no existing active-discovery path, they continue waiting for verifiable original-identity facts (A6). +Cloud's evidence intake and responsibility scheduling do not establish new Controller discovery capability, or production validation of real +containers, late-ensure fencing or physical data deletion. + +| Later scope | Step 1 boundary | +|---|---| +| Step 2: controlled operator endpoints, retained data, external confirmation | Not implemented; internal `controlled` arbitration does not supply authentication, intent or commands | +| Step 3: Controller / Sandbox Server physical contracts and per-target evidence | This page states Cloud evidence requirements, not completed physical acceptance | +| Step 4: cross-repository joint fault and recovery acceptance | Incomplete; single-repository implementation or database tests are not a substitute | +| Taking over D8 historical deletion abandonment and residuals without ledger rows | Not implemented; no ledger backfill, automatic retry or historical business-result rewrite | + +See the [core code map](../../internal/core/README.en.md) for implementation entry points. Test results require their execution artifacts; +this page does not claim migration, concurrency or end-to-end tests have passed. diff --git a/docs/agent-runs/residual-reclamation.md b/docs/agent-runs/residual-reclamation.md new file mode 100644 index 00000000..a695ca4b --- /dev/null +++ b/docs/agent-runs/residual-reclamation.md @@ -0,0 +1,83 @@ +# Agent Run 残留回收:Cloud 责任与证据 + +[English](residual-reclamation.en.md) | 中文 + +[有界启动](startup-settlement.md)的 D4/D5 收尾可以先结束业务运行,把尚未查明的资源留在 +`agent_run_residuals`。Cloud 的残留回收在原账本上继续取证,证据足够后才取得系统清理责任,沿已有 +owned delete 执行删除。它不重新结算 [IssueRun](issue-runs.md),也不重新开放运行 Workspace。 + +本文描述 Cloud 第 1 步的实现边界,不作为 Controller、Sandbox Server 的物理终止、部署或联合故障验收证据。 +[残留回收 ADR](https://github.com/ora-space/specs/blob/main/decisions/cloud/issue-run/20261018-residual-agent-run-reclamation.md) +仍为 `approved`;本步不是整份 ADR 已实施完成的声明。 + +## 账本状态和单一责任 + +| `reclaim_state` | 含义 | 取得责任的字段 | +|---|---|---| +| `reconciling` | 收集原身份事实,尚未授予删除资格 | ID、mode、operation 为空,authorization 为 `{}` | +| `reclaiming` | 已取得一份持久责任,继续推进或等待证据 | 固定 `reclaim_id`、mode 和非空 authorization;`owned_delete` 必须绑定唯一 operation | +| `cleared` | 原责任下的处置已完成 | 保留原责任,同时有 `cleared_at` 和 disposition | + +第 1 步的生产授权只使用 `owned_delete`,完成处置只写 `released`。`controlled` 只预留共用内部仲裁位置, +不构成受控管理端点或运维权限。数据库接受的 `terminated_data_retained`、`externally_confirmed` 留给后续受控路径。 + +自动责任始终绑定同一个 `delete_workspace` operation。重启、换 lease、重试和 quiesce 拒绝不会产生第二个责任或 operation, +也不会把失败的历史 operation 恢复为运行态。普通删除、派发和许可入口继续检查残留保护;内部系统责任不能被当成公开删除开关。 +账本、Run、Workspace、Project、operation 的租户、原绑定和 generation 在同一事务里重新比对;UUID 外键本身不证明这些范围一致。 + +[迁移](../../internal/core/migrations/README.md)只增列。存量 D4/D5 行通过默认值进入 `reconciling`,不推断完成、不回填 D8, +不修改旧 `reason`、`responsibility` 或业务结算。旧行已有非空 `cleared_at` 时明确报错回滚,等待显式兼容处理(A2)。 + +## 原身份取证与删除资格 + +`reclaim_targets` 是原计划和事实目标的并集,包括原 operation/effect、预先分配的 sandbox ensure ID、sandbox、Node、 +execution、clone、work 和数据目标。迟到事实可以补充原目标,不能将旧目标替换成当前 generation,不能用“查不到”删去未知目标。 + +新事实追加到 `reclaim_evidence`,包含逻辑事件键、事实、来源和数据库接收时间,不再写入旧 `responsibility.lateFacts`。 +相同事实重放复用原回执;新证据、目标补充或状态推进增加 `reclaim_version`。证据接管与授权、许可、完成都使用 Cloud 的既有 +串行事务边界,因此完成必须重新看本事务内的完整目标和最新事实。 + +系统授权要求原身份可定位、admission 已关闭、无冲突、无阻挡中的 Project 操作或其他活动,并满足交付和执行证据条件。 +执行侧需要同一 runtime control 身份的关闭输入确认及明确的未完成执行列表,并检查原执行终态;逐目标的终止证据也可证明其 +物理执行已封闭。仅在原计划和执行关系确实不存在时才能采用无目标证明,缺行、NULL result 或等待时间均不构成这种证明。 + +第 1 步只接受原 `deliver_revision` 执行真实登记、核验完成的 Revision 作为交付资格;其他实际交付执行也必须各自核验完成。 +当前没有可统一引用的既有“明确放弃交付”记录来源。`failed`、`skipped`、业务超时、没有对象存储或无结果都不自动等于明确放弃(A1), +证据不足的条目继续保持未完成。 + +获授权后沿已有 quiesce → terminate → cleanup 推进。quiesce 拒绝使原 operation 留在 `retry_wait`,保留原责任及保护,稍后重新取证再试(A3)。 +它不恢复新执行资格,不升级为强停,也不重新创建 operation。 + +每个原计划 sandbox/ensure ID 都需要对应成功终止结果同时包含 `terminated` 和 `lateEnsureFenced`。 +仅有 `terminated_at`、ensure 失败、404 或经过一段时间都不够。当前安全目标可以复用已有 owned delete;额外历史 ID 缺少双确认时保持未完成, +不为它改写目标或另发终止 effect(A5)。数据删除 effect 的计划、许可和完成再次检查目标覆盖;同一事务确认数据删除成功、operation 成功、 +Workspace 软删除并写 `cleared/released`,不改 Run 的 phase/status/result 或重复追加业务结算活动。 + +## 既有事实入口与冲突 + +Cloud 使用现有 `runtime_ack`、创建事实、执行结果和 Revision intake;没有新增 RPC、proto 或 Controller 取证机制。 +`runtime_ack` 通过原 control epoch/version、Node、sandbox 和 generation 检查后,将关闭输入和 `unfinishedExecutionIds` 原样接入账本。 +会话终态和交付结果按原 execution/work/资源身份接管;Run 已 done 或账本已 cleared 也不能成为拒收一致迟到事实的理由。 +成果救援沿既有 Revision 核验路径,不开放 admission,不启动新会话或执行。 + +一致重放不改原结论。矛盾终态先追加冲突审计并设置 `reclaim_conflicted_at`,提交这些事实及阻挡后才返回稳定的 +`409 residual_fact_conflict`(A4)。规范事实、已有 `cleared_at` 和 disposition 保留,后续删除或复用受阻;本步没有解除冲突的入口。 + +## 对账、恢复与尚未覆盖的范围 + +`ReconcileAgentRunResidualsOnce` 独立扫描账本的 `reconciling/reclaiming`,不依赖业务 Run 仍处于 `releasing`。 +扫描按 `reclaim_checked_at NULLS FIRST, created_at, run_id` 有界轮转;检查时间只是调度记录,不是完成证据。 +每个条目重新读取目标、资格和原绑定;已有责任恢复原 operation,继续服从当前 Controller lease 和 effect 许可。 +查询失败、证据不足和暂时拒绝均不清账、不扩大授权。 + +failed create 的原 effects 如果没有现成可主动发现的路径,就继续等待可验证的原身份事实(A6)。Cloud 接收证据和调度责任不证明 +Controller 已新增发现能力,更不证明真实容器、迟到 ensure 围栏或物理数据删除已经通过生产验证。 + +| 后续范围 | 本步边界 | +|---|---| +| 第 2 步:受控运维入口、保留数据、外部确认 | 未实现;内部 `controlled` 仲裁预留不替代认证、意图或命令 | +| 第 3 步:Controller / Sandbox Server 物理契约与逐目标证据 | 本文仅描述 Cloud 的证据要求,不声明物理验收完成 | +| 第 4 步:跨仓联合故障与恢复验收 | 未完成,不以单仓实现或数据库测试代替 | +| D8 历史删除放弃、无账本残留的接管 | 未实现;不补账、不自动重试、不改历史业务结果 | + +核心代码入口见 [core 代码地图](../../internal/core/README.md)。测试结果应以对应执行产物为准,本文不声明迁移、并发或端到端测试已经通过。 diff --git a/internal/config/README.en.md b/internal/config/README.en.md index 7ce46eb2..52c9aa5a 100644 --- a/internal/config/README.en.md +++ b/internal/config/README.en.md @@ -16,4 +16,6 @@ Without an explicit path it searches `./configs`, `../configs` and `.` for `conf Configuration stores deployment references; raw private keys and secrets load from separate files and never enter configuration bodies, logs or databases. This package validates startup configuration; Core owns durable window freezing, database clocks and recovery decisions. Enabled object storage does not establish online health. +`issue_runs.residual_reconcile_interval` sets the cadence of Cloud's residual-ledger recovery pass and defaults to `1m`; `CLOUD_ISSUE_RUNS_RESIDUAL_RECONCILE_INTERVAL` overrides it. Explicit zero, negative and malformed durations reject startup, and there is no disable switch. The interval only schedules scans; it is not a timeout proving successful resource cleanup. Cloud consumes real facts through existing entrypoints without adding a Controller discovery contract; missing evidence leaves the ledger open. + Tests cover defaults, environment overrides, invalid durations, TLS, storage and repository configuration. See [server](../../cmd/server/README.en.md), [Bounded startup](../../docs/agent-runs/startup-settlement.en.md) and [Deployment egress](../../docs/operations/deployment-egress.en.md). diff --git a/internal/config/README.md b/internal/config/README.md index ceefe47b..8b61f71b 100644 --- a/internal/config/README.md +++ b/internal/config/README.md @@ -16,4 +16,6 @@ 配置文件保存部署引用,原始私钥和机密通过独立文件加载,不写进配置正文、日志或数据库。包负责启动期配置有效性;业务窗口的冻结、数据库时钟与恢复判定在 Core。对象存储开启不等于在线健康探测。 +`issue_runs.residual_reconcile_interval` 控制 Cloud 残留账本恢复扫描的间隔,缺省为 `1m`;环境变量 `CLOUD_ISSUE_RUNS_RESIDUAL_RECONCILE_INTERVAL` 可覆盖它。显式零、负数和非法时长均拒绝启动,没有关闭开关。该间隔只安排扫描,不是资源清理成功的超时判据。Cloud 只接收既有入口提供的真实事实,不新增 Controller 发现契约;证据未到时账本保持 open。 + 测试覆盖默认值、环境覆盖、非法时长、TLS、对象存储与仓库配置。参见 [server](../../cmd/server/README.md)、[有界启动](../../docs/agent-runs/startup-settlement.md) 与 [部署出网配置](../../docs/operations/deployment-egress.md)。 diff --git a/internal/config/config.go b/internal/config/config.go index c5b6f571..a58edbb5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -48,6 +48,10 @@ type IssueRunsConfig struct { // rejects startup rather than silently selecting a zero-length window that would abandon every // blocked creation the moment it blocked. AgentWorkspaceBlockedTimeout time.Duration `mapstructure:"agent_workspace_blocked_timeout"` + // ResidualReconcileInterval is the cadence for Cloud's residual-ledger recovery pass. Omission + // selects one minute; an explicit non-positive or malformed duration rejects startup. This + // schedules reconciliation and is never a deadline proving external cleanup succeeded. + ResidualReconcileInterval time.Duration `mapstructure:"residual_reconcile_interval"` // ThreadIdleTimeout is Thread D4's idle window: how long a Thread may sit `idle` before Cloud // asks the session to end. It is a duration configured per deployment, never a per-run column, // and the comparison that judges it runs on database time — no Cloud, Controller or Node process @@ -171,6 +175,7 @@ func Load(configPath string) (*Config, error) { v.SetDefault("issue_runs.agent_restore_timeout", DefaultAgentRestoreTimeout) v.SetDefault("issue_runs.agent_end_confirm_timeout", DefaultAgentEndConfirmTimeout) v.SetDefault("issue_runs.agent_workspace_blocked_timeout", DefaultAgentWorkspaceBlockedTimeout) + v.SetDefault("issue_runs.residual_reconcile_interval", DefaultResidualReconcileInterval) v.SetDefault("plugins.asset_import_interval", DefaultAssetImportInterval) var cfg Config if err := v.Unmarshal(&cfg); err != nil { @@ -194,6 +199,9 @@ func Load(configPath string) (*Config, error) { if cfg.IssueRuns.AgentStartTimeout <= 0 || cfg.IssueRuns.AgentRestoreTimeout <= 0 || cfg.IssueRuns.AgentEndConfirmTimeout <= 0 || cfg.IssueRuns.AgentWorkspaceBlockedTimeout <= 0 { return nil, fmt.Errorf("issue_runs startup, restore, end confirmation and workspace blocked timeouts must be positive durations") } + if cfg.IssueRuns.ResidualReconcileInterval <= 0 { + return nil, fmt.Errorf("issue_runs.residual_reconcile_interval must be a positive duration") + } // IssueRun D5's two approved defaults, applied for the same reason: an operator who omits the keys // gets the limits the decision fixes, not a zero-length window that would abandon every delivery // immediately. @@ -224,6 +232,9 @@ const DefaultAgentEndConfirmTimeout = 5 * time.Minute // issue_runs.agent_workspace_blocked_timeout is absent or not positive. const DefaultAgentWorkspaceBlockedTimeout = 30 * time.Minute +// DefaultResidualReconcileInterval is the residual-ledger recovery cadence when omitted. +const DefaultResidualReconcileInterval = time.Minute + // DefaultDeliveryGiveUpAfter and DefaultDeliveryUnreachableAfter are IssueRun D5's first-version // give-up limits, applied when the corresponding key is absent or not positive. const ( diff --git a/internal/config/residual_reconcile_test.go b/internal/config/residual_reconcile_test.go new file mode 100644 index 00000000..1b5209ea --- /dev/null +++ b/internal/config/residual_reconcile_test.go @@ -0,0 +1,60 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestResidualReconcileIntervalDefaultsAndOverrides(t *testing.T) { + for _, tc := range []struct { + name, yaml, environment string + want time.Duration + }{ + {"omitted", "", "", time.Minute}, + {"file", "issue_runs: {residual_reconcile_interval: 45s}\n", "", 45 * time.Second}, + {"environment with omitted key", "", "2m", 2 * time.Minute}, + {"environment overrides file", "issue_runs: {residual_reconcile_interval: 45s}\n", "3m", 3 * time.Minute}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("CLOUD_ISSUE_RUNS_RESIDUAL_RECONCILE_INTERVAL", tc.environment) + cfg, err := Load(writeResidualReconcileConfig(t, tc.yaml)) + if err != nil { + t.Fatal(err) + } + if cfg.IssueRuns.ResidualReconcileInterval != tc.want { + t.Fatalf("residual reconcile interval = %s, want %s", cfg.IssueRuns.ResidualReconcileInterval, tc.want) + } + }) + } +} + +func TestResidualReconcileIntervalRejectsInvalidConfiguration(t *testing.T) { + for _, source := range []string{"file", "environment"} { + for _, value := range []string{"0s", "-1m", "infinite", "999999999999999999h"} { + t.Run(source+"/"+value, func(t *testing.T) { + yaml, environment := "issue_runs: {residual_reconcile_interval: "+value+"}\n", "" + if source == "environment" { + yaml, environment = "", value + } + t.Setenv("CLOUD_ISSUE_RUNS_RESIDUAL_RECONCILE_INTERVAL", environment) + _, err := Load(writeResidualReconcileConfig(t, yaml)) + if err == nil || !strings.Contains(err.Error(), "residual_reconcile_interval") { + t.Fatalf("invalid residual reconcile interval %q from %s must reject configuration, got %v", value, source, err) + } + }) + } + } +} + +func writeResidualReconcileConfig(t *testing.T, issueRuns string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config.yaml") + content := "server: {read_timeout: 10s, write_timeout: 15s}\ndatabase: {conn_max_lifetime: 1h}\ncontrol: {grpc_addr: '127.0.0.1:8082'}\n" + issueRuns + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return path +} diff --git a/internal/core/README.en.md b/internal/core/README.en.md index 87887e72..a4675f79 100644 --- a/internal/core/README.en.md +++ b/internal/core/README.en.md @@ -14,6 +14,8 @@ This package owns business state, authorization, transactions and recovery decis | `repository_*.go`, `git_*.go` | Bindings, sync/publish Operations, GitGrant and credential metadata | [External Git line](../../docs/repositories/external-git.en.md) | | `issue*.go`, `collaboration*.go`, `form_descriptor.go` | Issue aggregates, timeline, interactions and consuming collaboration ports | [Issue collaboration](../../docs/collaboration/interactions.en.md) | | `agent_run_*.go`, `execution*.go`, `thread*.go`, `business_hooks.go` | Run lifecycle, execution registry, Thread and transaction-scoped settlement hooks | [IssueRun](../../docs/agent-runs/issue-runs.en.md), [Controller](../../docs/runtime/controller-integration.en.md) | +| `agent_run_residual.go`, `agent_run_residual_reclaim.go` | Original identity/target coverage and evidence eligibility, one reclamation responsibility and one owned-delete operation, effect/permit and completion checks | [Residual reclamation](../../docs/agent-runs/residual-reclamation.en.md) | +| `agent_run_residual_fact.go`, `agent_run_residual_loop.go` | Appended facts without rewriting old responsibility, replay and durable conflict blocking; ledger scans independent of Run phase and original-operation recovery | [Residual reclamation](../../docs/agent-runs/residual-reclamation.en.md) | | `task_conversation_*.go`, `thread_preview.go` | Ordered task messages, serial same-task Agent handoff and pause barriers, follow-up prompts frozen from Node visible text, configuration-gap hints and a bounded in-process live preview; initial Agent prompts read only the frozen run input | [Thread](../../docs/agent-runs/threads.en.md), [IssueRun](../../docs/agent-runs/issue-runs.en.md) | | `revision*.go`, `session_*.go` | Revision registration, archives, session listing and statistics | [Revision](../../docs/agent-runs/revisions.en.md), [Session reads](../../docs/agent-runs/session-reads.en.md) | | `model_*.go` | Personal configuration, frozen bindings, grant digests and request observations; no API Key decryption | [Model access](../../docs/models/model-access.en.md) | @@ -30,3 +32,9 @@ This package owns business state, authorization, transactions and recovery decis - Commands wire Store ports and policy durations. See [collab](../collab/README.en.md) for development ports; simulated results do not replace physical execution evidence. See [migrations](migrations/README.en.md) for complete migration filenames. Package `*_db_test.go` tests need isolated real PostgreSQL schemas; [integration](../../integration/README.en.md) covers HTTP, Git and storage boundaries. + +Residual facts retain their original intake: `agent_run_provisioning_fact.go` handles creation facts, `node_execution.go` and `revision.go` handle +original execution terminals and delivery, and `runtime_control_delivery.go` accepts full runtime evidence through existing `runtime_ack`. +`run_workspace.go`, `control.go` and effect/permit paths enforce the original responsibility binding. No new RPC/proto or public system-delete +flag is introduced. This is Cloud step 1 only; see the dedicated page for conservative A1–A6 eligibility and remaining scope, without implying +physical or joint acceptance. diff --git a/internal/core/README.md b/internal/core/README.md index 0a3ebee9..19231274 100644 --- a/internal/core/README.md +++ b/internal/core/README.md @@ -14,6 +14,8 @@ | `repository_*.go`、`git_*.go` | Binding、同步/发布 Operation、GitGrant 与凭据元数据 | [外部 Git 线](../../docs/repositories/external-git.md) | | `issue*.go`、`collaboration*.go`、`form_descriptor.go` | Issue 聚合、时间线、交互和消费方协作端口 | [Issue 协作](../../docs/collaboration/interactions.md) | | `agent_run_*.go`、`execution*.go`、`thread*.go`、`business_hooks.go` | 运行生命周期、执行登记、Thread 与同事务结算钩子 | [IssueRun](../../docs/agent-runs/issue-runs.md)、[Controller](../../docs/runtime/controller-integration.md) | +| `agent_run_residual.go`、`agent_run_residual_reclaim.go` | 原身份/目标覆盖与证据资格、单一回收责任和唯一 owned-delete operation、effect/许可及完成检查 | [残留回收](../../docs/agent-runs/residual-reclamation.md) | +| `agent_run_residual_fact.go`、`agent_run_residual_loop.go` | 不改旧责任的追加事实、重放与持久冲突阻挡;独立于 Run phase 的账本扫描和原 operation 恢复 | [残留回收](../../docs/agent-runs/residual-reclamation.md) | | `task_conversation_*.go`、`thread_preview.go` | 任务级有序消息、同任务 Agent 串行交接与暂停屏障、从 Node 可见正文冻结的后续提示、配置缺口提示和有界进程内实时预览;Agent 初始提示只读冻结的运行输入 | [Thread](../../docs/agent-runs/threads.md)、[IssueRun](../../docs/agent-runs/issue-runs.md) | | `revision*.go`、`session_*.go` | Revision 登记、归档读取、会话列表与统计 | [Revision](../../docs/agent-runs/revisions.md)、[会话读取](../../docs/agent-runs/session-reads.md) | | `model_*.go` | 个人配置、冻结绑定、授权摘要与请求观察;不解密 API Key | [模型访问](../../docs/models/model-access.md) | @@ -30,3 +32,7 @@ - `Store` 的端口与策略时长由命令装配。开发协作端口见 [collab](../collab/README.md);物理执行证据不由模拟结果代替。 迁移目录与完整文件名见 [migrations](migrations/README.md)。包内 `*_db_test.go` 需独立真实 PostgreSQL schema;跨 HTTP、Git 与存储边界的覆盖在 [integration](../../integration/README.md)。 + +残留事实继续由原入口接管:`agent_run_provisioning_fact.go` 处理创建侧,`node_execution.go` 与 `revision.go` 处理原执行终态和交付, +`runtime_control_delivery.go` 接收已有 `runtime_ack` 的完整运行时证据。`run_workspace.go`、`control.go` 及 effect/许可路径检查原责任绑定; +不新增 RPC/proto 或公开的系统删除开关。当前仅为 Cloud 第 1 步实现,A1–A6 的保守资格和后续未实现范围见上述专页,不代表物理或联合验收完成。 diff --git a/internal/core/agent_run_residual_loop.go b/internal/core/agent_run_residual_loop.go new file mode 100644 index 00000000..342c8749 --- /dev/null +++ b/internal/core/agent_run_residual_loop.go @@ -0,0 +1,55 @@ +package core + +import "context" + +// ReconcileAgentRunResidualsOnce scans the ledger, independently of the business Run's phase. +// Scheduling time has no evidentiary meaning. Old unresolved rows rotate behind rows not yet +// checked, so one bounded batch of unknowns cannot starve the remainder of the ledger. +func (s *Store) ReconcileAgentRunResidualsOnce(ctx context.Context) error { + rows, err := s.Pool.QueryContext(ctx, `SELECT run_id FROM agent_run_residuals + WHERE reclaim_state IN ('reconciling','reclaiming') + ORDER BY reclaim_checked_at NULLS FIRST,created_at,run_id LIMIT $1`, agentDeliveryBatchSize) + if err != nil { + return err + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return err + } + ids = append(ids, id) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + for _, id := range ids { + if ctx.Err() != nil { + return ctx.Err() + } + // Claim the scheduling position separately: even an invalid historical ownership row + // whose reconciliation rolls back must yield its scan slot to the next bounded pass. + _, markErr := s.transact(ctx, func(t *transaction) Object { + t.exec("UPDATE agent_run_residuals SET reclaim_checked_at=clock_timestamp() WHERE run_id=$1 AND reclaim_state IN ('reconciling','reclaiming')", id) + return nil + }) + if markErr != nil { + continue + } + _, _ = s.transact(ctx, func(t *transaction) Object { + row := t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1 AND reclaim_state IN ('reconciling','reclaiming')", id) + if row == nil || row["reclaimConflictedAt"] != nil { + return nil + } + row = refreshResidualTargets(t, row) + if row.S("reclaimState") == "reconciling" { + return authorizeResidualOwnedDelete(t, id) + } + return resumeResidualReclamation(t, row) + }) + } + return nil +} From 4d07ade6626449f44906896402ffe35d0867ea11 Mon Sep 17 00:00:00 2001 From: creegon2 <315603738+creegon2@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:53:41 -0700 Subject: [PATCH 05/11] test(core): cover residual cleanup obligations and transaction races Exercise the real Cloud owned-delete topology, negative evidence, target union, dual termination, atomic rollback and restart without changing business settlement. Add controlled two-connection PostgreSQL interleavings for both commit orders across authorization, mode arbitration, completion, new uncovered executions, restart and cleared replays. The six D2 test functions passed with -race -count=3; keep physical and operator acceptance out of scope. --- .../agent_run_residual_reclaim_db_test.go | 322 ++++++++++++++ ...agent_run_residual_reclaim_race_db_test.go | 392 ++++++++++++++++++ 2 files changed, 714 insertions(+) create mode 100644 internal/core/agent_run_residual_reclaim_db_test.go create mode 100644 internal/core/agent_run_residual_reclaim_race_db_test.go diff --git a/internal/core/agent_run_residual_reclaim_db_test.go b/internal/core/agent_run_residual_reclaim_db_test.go new file mode 100644 index 00000000..1c13ddd1 --- /dev/null +++ b/internal/core/agent_run_residual_reclaim_db_test.go @@ -0,0 +1,322 @@ +package core + +import ( + "context" + "errors" + "strings" + "testing" +) + +type residualReclaimScene struct { + store *Store + abandonedCreation + node, delivery, work string + deliveryInput, deliveryResult Object +} + +func seedResidualExec(t *testing.T, s *Store, query string, args ...any) { + t.Helper() + if _, err := s.Pool.ExecContext(t.Context(), query, args...); err != nil { + t.Fatal(err) + } +} + +// The object-verification outcome is an explicit fixture. The actual delivery applier, receipt, +// Revision insert and residual fact write run together below; this does not test object storage. +func seedResidualScene(t *testing.T, ready bool) residualReclaimScene { + t.Helper() + s := commandStore(t) + scene := residualReclaimScene{store: s, abandonedCreation: seedAbandonedCreation(t, s, true), node: newID(), delivery: "residual-delivery-" + newID(), work: newID()} + seedResidualExec(t, s, `INSERT INTO node_instances(id,workspace_id,sandbox_instance_id,service_subject,connection_state,protocol_version,initialized,node_id,node_incarnation_id) + VALUES($1,$2,$3,'residual-node','connected',1,true,'residual-node','residual-incarnation')`, scene.node, scene.workspace, scene.effect) + seedResidualExec(t, s, `UPDATE sandbox_instances SET observed_state='running' WHERE id=$1`, scene.effect) + if ready { + _, err := submitFact(t, s, scene.abandonedCreation, "residual-ensure", Object{"epoch": 1, "kind": "effect", "effectId": scene.effect, "state": "succeeded", "externalId": "residual-substrate", "result": Object{"sandboxInstanceId": scene.effect, "nodeId": "residual-node"}}) + if err != nil { + t.Fatal(err) + } + _, err = submitFact(t, s, scene.abandonedCreation, "residual-creation", Object{"epoch": 1, "kind": "creation", "outcome": "ready"}) + if err != nil { + t.Fatal(err) + } + } + scene.deliveryInput = Object{"kind": "deliver_revision", "baseCommit": baseCommit, "revisionRef": "refs/ora/revisions/" + scene.run, "historyKey": "revisions/" + scene.run + "/history"} + scene.deliveryResult = Object{"outcome": "revision_unchanged", "node": Object{"nodeId": "residual-node", "nodeIncarnationId": "residual-incarnation"}, "baseCommit": baseCommit, "finalCommit": baseCommit, "revisionRef": scene.deliveryInput.S("revisionRef"), "history": Object{"key": scene.deliveryInput.S("historyKey"), "size": 0, "sha256": strings.Repeat("a", 64)}} + seedResidualExec(t, s, `INSERT INTO execution_work(id,run_id,kind,input,target,execution_id) VALUES($1,$2,'deliver_revision',$3,$4,$5)`, scene.work, scene.run, jsonText(scene.deliveryInput), jsonText(Object{"workspaceId": scene.workspace, "sandboxInstanceId": scene.effect, "runtimeGeneration": 1, "nodeId": "residual-node"}), scene.delivery) + seedResidualExec(t, s, `INSERT INTO node_executions(execution_id,kind,operation_id,work_id,workspace_id,node_id,node_operation_id,input,dispatched_epoch) + VALUES($1,'deliver_revision',$2,$3,$4,'residual-node',$1,$5,1)`, scene.delivery, scene.run, scene.work, scene.workspace, jsonText(scene.deliveryInput)) + if ready { + _, err := s.transact(t.Context(), func(tx *transaction) Object { return seedResidualDelivery(tx, scene, scene.deliveryResult) }) + if err != nil { + t.Fatal(err) + } + seedResidualRuntimeAck(t, scene, true) + } + return scene +} + +func seedResidualDelivery(tx *transaction, scene residualReclaimScene, result Object) Object { + return nodeExecutionResult(tx, &ControlRequest{Action: "clone_takeover", Body: Object{"operationId": scene.run, "executionId": scene.delivery, "sequence": 1, "event": "residual-delivery-terminal", "result": result}, verification: &revisionVerification{input: jsonText(scene.deliveryInput), valid: true}}, true) +} + +func seedResidualRuntimeAck(t *testing.T, scene residualReclaimScene, closed bool) { + t.Helper() + _, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + c := runtimeControl(tx, scene.workspace) + return runtimeControlCommand(tx, &ControlRequest{Action: "runtime_ack", Body: Object{"workspaceId": scene.workspace, "nodeInstanceId": scene.node, "controlEpoch": c.N("controlEpoch"), "controlVersion": c.N("version"), "inputClosed": closed, "unfinishedExecutionIds": []any{}}}) + }) + if err != nil { + t.Fatal(err) + } +} + +func seedResidualClaim(t *testing.T, scene residualReclaimScene) Object { + t.Helper() + if err := scene.store.ReconcileAgentRunResidualsOnce(t.Context()); err != nil { + t.Fatal(err) + } + out, err := scene.store.Control(t.Context(), &ControlRequest{Action: "claim", Body: Object{"epoch": 1}, Service: scene.claims}) + if err != nil { + t.Fatal(err) + } + op := out.O("operation") + if op.S("kind") != "delete_workspace" || op.O("request").S("reclaimId") == "" { + t.Fatalf("no owned residual delete claimed: %v", out) + } + return op +} + +func seedResidualAction(t *testing.T, scene residualReclaimScene, op Object, action string, body Object, effect string) Object { + t.Helper() + if body == nil { + body = Object{} + } + body["epoch"], body["version"] = int64(1), op.N("version") + out, err := scene.store.Control(t.Context(), &ControlRequest{Action: action, OperationID: op.S("id"), EffectID: effect, Body: body, Service: scene.claims}) + if err != nil { + t.Fatalf("residual %s at %s: %v", action, op.S("step"), err) + } + if out.S("id") != "" { + return Object{"operation": out} + } + return out +} + +// Drive the shipped quiesce -> terminate -> cleanup topology, leaving cleanup -> done to callers. +func seedResidualCleanup(t *testing.T, scene residualReclaimScene) Object { + t.Helper() + op := seedResidualClaim(t, scene) + seedResidualRuntimeAck(t, scene, false) + _, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + n := tx.one("SELECT * FROM node_instances WHERE id=$1", scene.node) + w := tx.one("SELECT * FROM workspaces WHERE id=$1", scene.workspace) + return nodeIdle(tx, &ControlRequest{Body: Object{"version": n.N("version"), "operationId": op.S("id"), "admissionEpoch": w.N("admissionEpoch"), "idle": true}}, n, w) + }) + if err != nil { + t.Fatal(err) + } + op = seedResidualAction(t, scene, op, "advance", nil, "").O("operation") + planned := seedResidualAction(t, scene, op, "plan", Object{"kind": "sandbox_terminate", "workspaceId": scene.workspace}, "") + op = planned.O("operation") + out := seedResidualAction(t, scene, op, "effect_result", Object{"state": "succeeded", "externalId": "residual-termination", "result": Object{"terminated": true, "lateEnsureFenced": true}}, planned.O("effect").S("id")) + op = out.O("operation") + op = seedResidualAction(t, scene, op, "advance", nil, "").O("operation") + planned = seedResidualAction(t, scene, op, "plan", Object{"kind": "workspace_data_delete", "workspaceId": scene.workspace}, "") + op = planned.O("operation") + return seedResidualAction(t, scene, op, "effect_result", Object{"state": "succeeded", "externalId": "residual-cleanup", "result": Object{"removed": true}}, planned.O("effect").S("id")).O("operation") +} + +func seedResidualAssert(t *testing.T, scene residualReclaimScene, query string, args ...any) { + t.Helper() + var ok bool + if err := scene.store.Pool.QueryRowContext(t.Context(), query, args...).Scan(&ok); err != nil { + t.Fatal(err) + } + if !ok { + t.Fatalf("residual invariant failed: %s", query) + } +} + +func TestResidualOwnedDeleteRunsCloudTopologyAndPreservesBusinessTerminal(t *testing.T) { + scene := seedResidualScene(t, true) + op := seedResidualCleanup(t, scene) + out := seedResidualAction(t, scene, op, "advance", nil, "") + if out.O("operation").S("state") != "succeeded" { + t.Fatalf("cleanup did not finish: %v", out) + } + seedResidualAssert(t, scene, `SELECT r.reclaim_state='cleared' AND r.disposition='released' AND r.cleared_at IS NOT NULL + AND r.responsibility=$2::jsonb AND r.reason='workspace_unavailable' AND w.deleted_at IS NOT NULL AND NOT w.admission_open + AND i.phase='done' AND i.status='failed' AND i.failure_reason='workspace_unavailable' + AND (SELECT count(*) FROM operations WHERE workspace_id=w.id AND kind='delete_workspace')=1 + AND (SELECT count(*) FROM external_effects WHERE operation_id=r.reclaim_operation_id AND kind IN ('sandbox_terminate','workspace_data_delete') AND state='succeeded')=2 + FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id JOIN issue_runs i ON i.id=r.run_id WHERE r.run_id=$1`, scene.run, scene.responsibility) +} + +func TestResidualUnknownAndDeliveryFailureNeverAuthorizeDataDeletion(t *testing.T) { + for _, delivery := range []string{"unknown", "failed", "skipped", "timeout"} { + t.Run(delivery, func(t *testing.T) { + scene := seedResidualScene(t, false) + if delivery != "unknown" { + _, err := submitFact(t, scene.store, scene.abandonedCreation, "negative-ready-ensure", Object{"epoch": 1, "kind": "effect", "effectId": scene.effect, "state": "succeeded", "externalId": "negative-substrate", "result": Object{"sandboxInstanceId": scene.effect, "nodeId": "residual-node"}}) + if err != nil { + t.Fatal(err) + } + _, err = scene.store.transact(t.Context(), func(tx *transaction) Object { + return seedResidualDelivery(tx, scene, Object{"outcome": "revision_failed", "reason": "upload_failed", "node": Object{"nodeId": "residual-node", "nodeIncarnationId": "residual-incarnation"}}) + }) + if err != nil { + t.Fatal(err) + } + seedResidualRuntimeAck(t, scene, true) + seedResidualExec(t, scene.store, `UPDATE issue_runs SET result=jsonb_set(COALESCE(result,'{}'),'{deliveryState}',to_jsonb($2::text)) WHERE id=$1`, scene.run, map[string]string{"failed": "failed", "skipped": "skipped", "timeout": "failed"}[delivery]) + } + if delivery == "timeout" { + seedResidualExec(t, scene.store, `UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '10 days' WHERE id=$1`, scene.run) + } + if err := scene.store.ReconcileAgentRunResidualsOnce(t.Context()); err != nil { + t.Fatal(err) + } + seedResidualAssert(t, scene, `SELECT reclaim_state='reconciling' AND reclaim_id IS NULL AND cleared_at IS NULL + AND responsibility=$2::jsonb AND NOT EXISTS(SELECT 1 FROM operations WHERE workspace_id=$3 AND kind='delete_workspace') + AND NOT EXISTS(SELECT 1 FROM external_effects WHERE workspace_id=$3 AND kind IN ('sandbox_terminate','workspace_data_delete')) + FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.responsibility, scene.workspace) + }) + } +} + +func TestResidualPartialAndNullScopeTargetsRemainAUnion(t *testing.T) { + scene := seedResidualScene(t, true) + extra, missing := newID(), newID() + seedResidualExec(t, scene.store, `INSERT INTO external_effects(id,operation_id,project_id,workspace_id,kind,state,request,reconciled_epoch) + VALUES($1,$2,$3,NULL,'sandbox_ensure','failed','{}',1)`, extra, scene.operation, scene.project) + seedResidualExec(t, scene.store, `UPDATE agent_run_residuals SET reclaim_targets=$2 WHERE run_id=$1`, scene.run, jsonText([]Object{{"kind": "sandbox", "id": missing, "workspaceId": scene.workspace}})) + if err := scene.store.ReconcileAgentRunResidualsOnce(t.Context()); err != nil { + t.Fatal(err) + } + seedResidualAssert(t, scene, `SELECT reclaim_state='reconciling' AND reclaim_id IS NULL + AND reclaim_targets @> jsonb_build_array(jsonb_build_object('kind','sandbox','id',$2::text)) + AND reclaim_targets @> jsonb_build_array(jsonb_build_object('kind','effect','id',$3::text)) + AND reclaim_targets @> jsonb_build_array(jsonb_build_object('kind','sandbox','id',$3::text)) + AND responsibility=$4::jsonb FROM agent_run_residuals WHERE run_id=$1`, scene.run, missing, extra, scene.responsibility) +} + +func TestResidualCleanupRequiresEveryDualTerminationAndRemovedProof(t *testing.T) { + for _, proof := range []string{"terminated_only", "wrong_sandbox", "not_removed", "historical_target"} { + t.Run(proof, func(t *testing.T) { + scene := seedResidualScene(t, true) + op := seedResidualCleanup(t, scene) + switch proof { + case "terminated_only": + seedResidualExec(t, scene.store, `UPDATE external_effects SET result='{"terminated":true}' WHERE operation_id=$1 AND kind='sandbox_terminate'`, op.S("id")) + case "wrong_sandbox": + seedResidualExec(t, scene.store, `UPDATE external_effects SET request=jsonb_set(request,'{sandboxInstanceId}',to_jsonb($2::text)) WHERE operation_id=$1 AND kind='sandbox_terminate'`, op.S("id"), newID()) + case "not_removed": + seedResidualExec(t, scene.store, `UPDATE external_effects SET result='{"removed":false}' WHERE operation_id=$1 AND kind='workspace_data_delete'`, op.S("id")) + case "historical_target": + seedResidualExec(t, scene.store, `UPDATE agent_run_residuals SET reclaim_targets=reclaim_targets||$2::jsonb WHERE run_id=$1`, scene.run, jsonText([]Object{{"kind": "sandbox", "id": newID(), "workspaceId": scene.workspace}})) + } + _, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + return Object{"complete": completeResidualReclamation(tx, tx.one("SELECT * FROM operations WHERE id=$1", op.S("id")))} + }) + if err == nil { + t.Fatal("incomplete cleanup proof cleared the residual") + } + seedResidualAssert(t, scene, `SELECT r.cleared_at IS NULL AND r.reclaim_state='reclaiming' AND w.deleted_at IS NULL FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id WHERE r.run_id=$1`, scene.run) + }) + } +} + +func TestResidualClaimRollbackAndRestartKeepOneBoundOperation(t *testing.T) { + scene := seedResidualScene(t, true) + _, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + authorizeResidualOwnedDelete(tx, scene.run) + reject(409, "fixture_rollback") + return nil + }) + var fault *Fault + if !errors.As(err, &fault) || fault.Code != "fixture_rollback" { + t.Fatalf("rollback boundary: %v", err) + } + seedResidualAssert(t, scene, `SELECT reclaim_state='reconciling' AND reclaim_id IS NULL AND NOT EXISTS(SELECT 1 FROM operations WHERE workspace_id=$2 AND kind='delete_workspace') FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.workspace) + op := seedResidualClaim(t, scene) + seedResidualExec(t, scene.store, `UPDATE operations SET state='retry_wait',retry_at=clock_timestamp()-interval '1 second',error_code='node_unavailable' WHERE id=$1`, op.S("id")) + var authority, targets string + if err := scene.store.Pool.QueryRow(`SELECT reclaim_authorization::text,reclaim_targets::text FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&authority, &targets); err != nil { + t.Fatal(err) + } + // A new process-owned Store value shares the persisted database, not any in-memory claim state. + restarted := seedResidualRebuildStore(scene.store) + if err := restarted.ReconcileAgentRunResidualsOnce(context.Background()); err != nil { + t.Fatal(err) + } + _, err = restarted.transact(t.Context(), func(tx *transaction) Object { return authorizeResidualOwnedDelete(tx, scene.run) }) + if err != nil { + t.Fatal(err) + } + // Run Workspaces are intentionally hidden from public retryOperation. The server resumes + // their due retry_wait operation through the Controller's existing claim path instead. + reclaimed, err := restarted.Control(t.Context(), &ControlRequest{Action: "claim", Body: Object{"epoch": 1}, Service: scene.claims}) + if err != nil { + t.Fatal(err) + } + if retried := reclaimed.O("operation"); retried.S("id") != op.S("id") || retried.S("state") != "running" { + t.Fatalf("restart did not retry the original bound operation: %v", reclaimed) + } + seedResidualAssert(t, scene, `SELECT reclaim_operation_id=$2 AND reclaim_authorization=$3::jsonb AND reclaim_targets=$4::jsonb + AND (SELECT count(*) FROM operations WHERE workspace_id=$5 AND kind='delete_workspace')=1 FROM agent_run_residuals WHERE run_id=$1`, scene.run, op.S("id"), authority, targets, scene.workspace) +} + +func seedResidualRebuildStore(original *Store) *Store { + restarted := &Store{Pool: original.Pool, Signals: NewControlHub()} + BindBusinessHooks(restarted) + return restarted +} + +func TestResidualCompletionRollbackKeepsLedgerAndWorkspaceAtomic(t *testing.T) { + scene := seedResidualScene(t, true) + op := seedResidualCleanup(t, scene) + _, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + advance(tx, &ControlRequest{}, tx.one("SELECT * FROM operations WHERE id=$1", op.S("id"))) + reject(409, "fixture_completion_rollback") + return nil + }) + var fault *Fault + if !errors.As(err, &fault) || fault.Code != "fixture_completion_rollback" { + t.Fatalf("completion rollback boundary: %v", err) + } + seedResidualAssert(t, scene, `SELECT r.reclaim_state='reclaiming' AND r.cleared_at IS NULL AND r.disposition IS NULL + AND w.deleted_at IS NULL AND o.state='running' AND o.step='cleanup' + AND NOT EXISTS(SELECT 1 FROM jsonb_array_elements(r.reclaim_evidence) e WHERE e->'fact'->>'kind'='reclaim_complete') + FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id JOIN operations o ON o.id=r.reclaim_operation_id WHERE r.run_id=$1`, scene.run) +} + +func TestResidualOrdinaryLifecycleGuardsAndUnrelatedD8StayProtected(t *testing.T) { + scene := seedResidualScene(t, true) + if _, err := scene.store.DeleteRunWorkspace(t.Context(), scene.run); err == nil { + t.Fatal("ordinary run delete bypassed the residual") + } + for _, kind := range []string{"stop", "delete_workspace", "delete_project"} { + _, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + w := tx.one("SELECT * FROM workspaces WHERE id=$1", scene.workspace) + if kind == "delete_project" { + for _, item := range tx.list("SELECT * FROM workspaces WHERE project_id=$1", scene.project) { + checkActivities(tx, item) + } + } else { + checkActivities(tx, w) + } + return nil + }) + var fault *Fault + if !errors.As(err, &fault) || fault.Code != "resource_in_use" { + t.Fatalf("%s guard bypass: %v", kind, err) + } + } + // D8 remains on its existing releasing-only scan: this done residual never enters that path. + if err := scene.store.GiveUpStaleWorkspaceReleasesOnce(t.Context()); err != nil { + t.Fatal(err) + } + seedResidualAssert(t, scene, `SELECT i.phase='done' AND i.status='failed' AND i.failure_reason='workspace_unavailable' + AND r.reason='workspace_unavailable' AND r.responsibility=$2::jsonb AND r.cleared_at IS NULL + FROM issue_runs i JOIN agent_run_residuals r ON r.run_id=i.id WHERE i.id=$1`, scene.run, scene.responsibility) +} diff --git a/internal/core/agent_run_residual_reclaim_race_db_test.go b/internal/core/agent_run_residual_reclaim_race_db_test.go new file mode 100644 index 00000000..ea9e1094 --- /dev/null +++ b/internal/core/agent_run_residual_reclaim_race_db_test.go @@ -0,0 +1,392 @@ +package core + +import ( + "context" + "database/sql" + "fmt" + "testing" + "time" +) + +type residualRaceResult struct { + out, observed Object + err error +} + +// Wrap the two real, dedicated-connection transactions with the production transaction context. +// This deliberately mirrors Store.transact's hooks: the callbacks below call the same production +// state functions while the advisory lock is held, never substitutes built only for the race. +func seedResidualTransaction(ctx context.Context, s *Store, raw *sql.Tx) *transaction { + tx := &transaction{tx: raw, ctx: ctx, store: s, pluginExecution: s.PluginExecution, legacyCloneFixture: s.legacyCloneFixture, directory: s.Directory, contextBuilder: s.Context, forms: s.Forms, assist: s.Assist, simulator: s.Simulator, objectStore: s.ObjectStore, onThreadEvents: s.OnThreadEvents, onSessionEnded: s.OnSessionEnded, onRunWorkspaceSettled: s.OnRunWorkspaceSettled, onRunWorkspaceDeleted: s.OnRunWorkspaceDeleted, onDeliverySettled: s.OnDeliverySettled} + tx.ctx = context.WithValue(ctx, businessTransactionKey{}, tx) + return tx +} + +func seedResidualRaceApply(tx *transaction, run string, fn func(*transaction) Object) (result residualRaceResult) { + defer func() { + if recovered := recover(); recovered != nil { + switch value := recovered.(type) { + case *Fault: + result.err = value + case databaseFailure: + result.err = value.err + default: + panic(recovered) + } + } + }() + // This statement happens after lock acquisition. A default READ COMMITTED transaction must + // observe A's commit here, even though B began and tried the lock before that commit. + result.observed = tx.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", run) + result.out = fn(tx) + return result +} + +// A executes while holding the production lock and stays uncommitted. B proves contention with +// pg_try_advisory_xact_lock=false on its SAME transaction, signals the channel, then blocks on the +// lock. Only then may A commit/roll back; B subsequently re-reads and calls its production function. +// No sleep, external probe connection or elapsed-time inference substitutes for that ordering. +func seedResidualRace(t *testing.T, scene residualReclaimScene, first, second func(*transaction) Object) (residualRaceResult, residualRaceResult) { + return seedResidualRaceStores(t, scene, scene.store, scene.store, first, second) +} + +func seedResidualRaceStores(t *testing.T, scene residualReclaimScene, storeA, storeB *Store, first, second func(*transaction) Object) (residualRaceResult, residualRaceResult) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + a, err := scene.store.Pool.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer a.Close() + b, err := scene.store.Pool.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer b.Close() + rawA, err := a.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer rawA.Rollback() + rawB, err := b.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer rawB.Rollback() + if _, err = rawA.ExecContext(ctx, "SELECT pg_advisory_xact_lock(67420911)"); err != nil { + t.Fatal(err) + } + txA, txB := seedResidualTransaction(ctx, storeA, rawA), seedResidualTransaction(ctx, storeB, rawB) + resultA := seedResidualRaceApply(txA, scene.run, first) + ready, finished := make(chan error, 1), make(chan residualRaceResult, 1) + go func() { + var acquired bool + if err := rawB.QueryRowContext(ctx, "SELECT pg_try_advisory_xact_lock(67420911)").Scan(&acquired); err != nil { + ready <- err + return + } + if acquired { + ready <- fmt.Errorf("B acquired A's still-uncommitted lock") + return + } + ready <- nil + if _, err := rawB.ExecContext(ctx, "SELECT pg_advisory_xact_lock(67420911)"); err != nil { + finished <- residualRaceResult{err: err} + return + } + result := seedResidualRaceApply(txB, scene.run, second) + if result.err != nil { + _ = rawB.Rollback() + } else { + result.err = rawB.Commit() + if result.err == nil && txB.residualFactConflict { + result.err = &Fault{Status: 409, Code: "residual_fact_conflict"} + } + } + finished <- result + }() + select { + case err := <-ready: + if err != nil { + t.Fatal(err) + } + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + if resultA.err != nil { + _ = rawA.Rollback() + } else { + resultA.err = rawA.Commit() + if resultA.err == nil && txA.residualFactConflict { + resultA.err = &Fault{Status: 409, Code: "residual_fact_conflict"} + } + } + var resultB residualRaceResult + select { + case resultB = <-finished: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + return resultA, resultB +} + +// A delayed dispatch record is a staged precondition, not a new runtime invocation. Its real +// terminal is taken over by nodeExecutionResult. Session completion still leaves its independent +// deliver_revision target without verified storage evidence, which must block data deletion. +func seedResidualLateExecution(tx *transaction, scene residualReclaimScene, kind string) Object { + work, execution := newID(), "late-"+kind+"-"+newID() + target := Object{"workspaceId": scene.workspace, "sandboxInstanceId": scene.effect, "runtimeGeneration": 1, "nodeId": "residual-node"} + tx.exec(`INSERT INTO execution_work(id,run_id,kind,input,target,execution_id) VALUES($1,$2,$3,'{}',$4,$5)`, work, scene.run, kind, jsonText(target), execution) + tx.exec(`INSERT INTO node_executions(execution_id,kind,operation_id,work_id,workspace_id,node_id,node_operation_id,input,dispatched_epoch) + VALUES($1,$2,$3,$4,$5,'residual-node',$1,'{}',1)`, execution, kind, scene.run, work, scene.workspace) + result := Object{"outcome": "revision_failed", "reason": "upload_failed", "node": Object{"nodeId": "residual-node", "nodeIncarnationId": "residual-incarnation"}} + if kind == "agent_session" { + result["outcome"], result["reason"] = "agent_session_ended", "agent_failed" + out := nodeExecutionResult(tx, &ControlRequest{Body: Object{"operationId": scene.run, "executionId": execution, "sequence": 1, "event": "late-original-terminal", "result": result}}, true) + pendingWork, pendingExecution := newID(), "pending-rescue-"+newID() + tx.exec(`INSERT INTO execution_work(id,run_id,kind,input,target,execution_id) VALUES($1,$2,'deliver_revision','{}',$3,$4)`, pendingWork, scene.run, jsonText(target), pendingExecution) + tx.exec(`INSERT INTO node_executions(execution_id,kind,operation_id,work_id,workspace_id,node_id,node_operation_id,input,dispatched_epoch) + VALUES($1,'deliver_revision',$2,$3,$4,'residual-node',$1,'{}',1)`, pendingExecution, scene.run, pendingWork, scene.workspace) + return out + } + return nodeExecutionResult(tx, &ControlRequest{Body: Object{"operationId": scene.run, "executionId": execution, "sequence": 1, "event": "late-original-terminal", "result": result}}, true) +} + +func TestResidualRaceCompletionVersusNewUnclosedExecutionTargets(t *testing.T) { + for _, kind := range []string{"agent_session", "deliver_revision"} { + for _, factFirst := range []bool{false, true} { + t.Run(fmt.Sprint(kind, "/fact_first=", factFirst), func(t *testing.T) { + scene := seedResidualScene(t, true) + op := seedResidualCleanup(t, scene) + complete := func(tx *transaction) Object { + return advance(tx, &ControlRequest{}, tx.one("SELECT * FROM operations WHERE id=$1", op.S("id"))) + } + fact := func(tx *transaction) Object { return seedResidualLateExecution(tx, scene, kind) } + first, second := complete, fact + if factFirst { + first, second = fact, complete + } + a, b := seedResidualRace(t, scene, first, second) + if factFirst { + if a.err != nil { + t.Fatal(a.err) + } + if b.err == nil { + t.Fatal("completion ignored the newly committed unclosed delivery target") + } + if b.observed.S("reclaimState") != "reclaiming" { + t.Fatal("completion did not reread the retained responsibility") + } + } else if a.err != nil || b.err != nil { + t.Fatalf("completed responsibility did not retain late original facts: %v / %v", a.err, b.err) + } + seedResidualAssert(t, scene, `SELECT (r.cleared_at IS NOT NULL)=$2 AND (w.deleted_at IS NOT NULL)=$2 + AND r.responsibility=$3::jsonb AND i.phase='done' AND i.status='failed' + AND (SELECT count(*) FROM node_executions WHERE operation_id=r.run_id AND kind=$4 AND result IS NOT NULL)>=1 + AND (SELECT count(*) FROM revisions WHERE run_id=r.run_id)=1 + AND (SELECT count(*) FROM node_executions e WHERE e.workspace_id=r.workspace_id AND e.kind='deliver_revision' + AND NOT EXISTS(SELECT 1 FROM revision_verifications v WHERE v.execution_id=e.execution_id AND v.outcome IN ('delivered','unchanged')))>=1 + AND (SELECT count(*) FROM operations WHERE workspace_id=r.workspace_id AND kind='delete_workspace')=1 + FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id JOIN issue_runs i ON i.id=r.run_id WHERE r.run_id=$1`, scene.run, !factFirst, scene.responsibility, kind) + }) + } + } +} + +func TestResidualRaceRestartRebuildsTwoStoresWithoutNewResponsibility(t *testing.T) { + for _, resumeFirst := range []bool{false, true} { + t.Run(fmt.Sprint("resume_first=", resumeFirst), func(t *testing.T) { + scene := seedResidualScene(t, true) + op := seedResidualCleanup(t, scene) + var reclaim, authority, targets string + if err := scene.store.Pool.QueryRow(`SELECT reclaim_id::text,reclaim_authorization::text,reclaim_targets::text FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&reclaim, &authority, &targets); err != nil { + t.Fatal(err) + } + storeA, storeB := seedResidualRebuildStore(scene.store), seedResidualRebuildStore(scene.store) + resume := func(tx *transaction) Object { + return resumeResidualReclamation(tx, tx.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", scene.run)) + } + reclaimAgain := func(tx *transaction) Object { return authorizeResidualOwnedDelete(tx, scene.run) } + first, second := reclaimAgain, resume + if resumeFirst { + first, second = resume, reclaimAgain + } + a, b := seedResidualRaceStores(t, scene, storeA, storeB, first, second) + if a.err != nil || b.err != nil { + t.Fatalf("restart contenders failed: %v / %v", a.err, b.err) + } + if a.out.O("operation").S("id") != op.S("id") || b.out.O("operation").S("id") != op.S("id") { + t.Fatal("a rebuilt Store substituted a second operation") + } + seedResidualAssert(t, scene, `SELECT reclaim_id=$2 AND reclaim_operation_id=$3 AND reclaim_authorization=$4::jsonb + AND reclaim_targets=$5::jsonb AND responsibility=$6::jsonb AND reclaim_state='reclaiming' AND cleared_at IS NULL + AND (SELECT count(*) FROM operations WHERE workspace_id=$7 AND kind='delete_workspace')=1 + AND (SELECT count(*) FROM external_effects WHERE operation_id=$3)=2 + FROM agent_run_residuals WHERE run_id=$1`, scene.run, reclaim, op.S("id"), authority, targets, scene.responsibility, scene.workspace) + }) + } +} + +func TestResidualRaceAuthorizationVersusLateFactsBothCommitOrders(t *testing.T) { + for _, factFirst := range []bool{false, true} { + t.Run(fmt.Sprint("fact_first=", factFirst), func(t *testing.T) { + scene := seedResidualScene(t, true) + authorize := func(tx *transaction) Object { return authorizeResidualOwnedDelete(tx, scene.run) } + fact := func(tx *transaction) Object { + return provisioningCreationFact(tx, &ControlRequest{OperationID: scene.operation, Body: Object{"outcome": "failed"}}, scene.run) + } + first, second := authorize, fact + if factFirst { + first, second = fact, authorize + } + a, b := seedResidualRace(t, scene, first, second) + if factFirst { + assertResidualFactConflict(t, a.err) + assertResidualFactConflict(t, b.err) + if b.observed["reclaimConflictedAt"] == nil { + t.Fatal("authorization did not reread the committed conflict") + } + } else { + if a.err != nil { + t.Fatal(a.err) + } + assertResidualFactConflict(t, b.err) + if b.observed.S("reclaimMode") != "owned_delete" { + t.Fatal("late fact did not observe the committed authorization") + } + } + seedResidualAssert(t, scene, `SELECT reclaim_conflicted_at IS NOT NULL AND responsibility=$2::jsonb AND cleared_at IS NULL + AND (SELECT count(*) FROM operations WHERE workspace_id=$3 AND kind='delete_workspace')=$4 + AND NOT EXISTS(SELECT 1 FROM external_effects WHERE workspace_id=$3 AND kind='workspace_data_delete') + FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.responsibility, scene.workspace, map[bool]int{false: 1, true: 0}[factFirst]) + }) + } +} + +func TestResidualRaceAutomaticVersusControlledClaimBothCommitOrders(t *testing.T) { + for _, controlledFirst := range []bool{false, true} { + t.Run(fmt.Sprint("controlled_first=", controlledFirst), func(t *testing.T) { + scene := seedResidualScene(t, true) + auto := func(tx *transaction) Object { return authorizeResidualOwnedDelete(tx, scene.run) } + intent := newID() + controlled := func(tx *transaction) Object { + return claimResidualReclamation(tx, scene.run, "controlled", Object{"kind": "controlled_intent", "intentId": intent, "source": "authorized-test-intent"}) + } + first, second := auto, controlled + mode := "owned_delete" + count := 1 + if controlledFirst { + first, second = controlled, auto + mode = "controlled" + count = 0 + } + a, b := seedResidualRace(t, scene, first, second) + if a.err != nil { + t.Fatal(a.err) + } + fault, ok := b.err.(*Fault) + if !ok || fault.Status != 409 || fault.Code != "residual_mode_conflict" { + t.Fatalf("losing claimant did not respect the committed mode: %v", b.err) + } + if b.observed.S("reclaimId") != a.out.O("residual").S("reclaimId") { + t.Fatal("B did not reread A's durable responsibility") + } + seedResidualAssert(t, scene, `SELECT reclaim_state='reclaiming' AND reclaim_mode=$2 AND reclaim_id IS NOT NULL + AND responsibility=$3::jsonb AND cleared_at IS NULL + AND (SELECT count(*) FROM operations WHERE workspace_id=$4 AND kind='delete_workspace')=$5 + AND NOT EXISTS(SELECT 1 FROM external_effects WHERE workspace_id=$4 AND kind IN ('sandbox_terminate','workspace_data_delete')) + FROM agent_run_residuals WHERE run_id=$1`, scene.run, mode, scene.responsibility, scene.workspace, count) + }) + } +} + +func TestResidualRaceCompletionVersusLateDeliveryBothCommitOrders(t *testing.T) { + for _, factFirst := range []bool{false, true} { + t.Run(fmt.Sprint("fact_first=", factFirst), func(t *testing.T) { + scene := seedResidualScene(t, true) + op := seedResidualCleanup(t, scene) + complete := func(tx *transaction) Object { + return advance(tx, &ControlRequest{}, tx.one("SELECT * FROM operations WHERE id=$1", op.S("id"))) + } + late := func(tx *transaction) Object { + return seedResidualDelivery(tx, scene, Object{"outcome": "revision_failed", "reason": "upload_failed", "node": Object{"nodeId": "residual-node", "nodeIncarnationId": "residual-incarnation"}}) + } + first, second := complete, late + if factFirst { + first, second = late, complete + } + a, b := seedResidualRace(t, scene, first, second) + if factFirst { + assertResidualFactConflict(t, a.err) + if b.err == nil || b.observed["reclaimConflictedAt"] == nil { + t.Fatalf("completion ignored committed delivery conflict: %v", b.err) + } + } else { + if a.err != nil { + t.Fatal(a.err) + } + assertResidualFactConflict(t, b.err) + if b.observed.S("reclaimState") != "cleared" { + t.Fatal("late delivery did not reread cleared state") + } + } + seedResidualAssert(t, scene, `SELECT r.reclaim_conflicted_at IS NOT NULL AND r.responsibility=$2::jsonb + AND (r.cleared_at IS NOT NULL)=$3 AND (w.deleted_at IS NOT NULL)=$3 + AND e.result=$4::jsonb AND (SELECT count(*) FROM revisions WHERE run_id=r.run_id)=1 + AND (SELECT count(*) FROM node_event_receipts WHERE execution_id=$5)=1 + FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id JOIN node_executions e ON e.execution_id=$5 WHERE r.run_id=$1`, scene.run, scene.responsibility, !factFirst, jsonText(scene.deliveryResult), scene.delivery) + }) + } +} + +func TestResidualRaceClearedReplayVersusLateFactBothCommitOrders(t *testing.T) { + for _, conflict := range []bool{false, true} { + for _, factFirst := range []bool{false, true} { + t.Run(fmt.Sprint("conflict=", conflict, "/fact_first=", factFirst), func(t *testing.T) { + scene := seedResidualScene(t, true) + op := seedResidualCleanup(t, scene) + seedResidualAction(t, scene, op, "advance", nil, "") + var before int + if err := scene.store.Pool.QueryRow(`SELECT jsonb_array_length(reclaim_evidence) FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&before); err != nil { + t.Fatal(err) + } + replay := func(tx *transaction) Object { + return Object{"complete": completeResidualReclamation(tx, tx.one("SELECT * FROM operations WHERE id=$1", op.S("id")))} + } + fact := func(tx *transaction) Object { + outcome := "ready" + if conflict { + outcome = "failed" + } + return provisioningCreationFact(tx, &ControlRequest{OperationID: scene.operation, Body: Object{"outcome": outcome}}, scene.run) + } + first, second := replay, fact + if factFirst { + first, second = fact, replay + } + a, b := seedResidualRace(t, scene, first, second) + if !conflict { + if a.err != nil || b.err != nil { + t.Fatalf("cleared replay failed: %v / %v", a.err, b.err) + } + } else if factFirst { + assertResidualFactConflict(t, a.err) + assertResidualFactConflict(t, b.err) + } else { + if a.err != nil { + t.Fatal(a.err) + } + assertResidualFactConflict(t, b.err) + } + if b.observed.S("reclaimState") != "cleared" { + t.Fatal("second transaction lost the cleared identity") + } + seedResidualAssert(t, scene, `SELECT reclaim_state='cleared' AND disposition='released' AND cleared_at IS NOT NULL + AND responsibility=$2::jsonb AND jsonb_array_length(reclaim_evidence)=$3 AND (reclaim_conflicted_at IS NOT NULL)=$4 + AND (SELECT count(*) FROM operations WHERE workspace_id=$5 AND kind='delete_workspace')=1 + FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.responsibility, before+map[bool]int{false: 0, true: 1}[conflict], conflict, scene.workspace) + }) + } + } +} From dad1fef1ec955e6669806b14beb5a9f144a6e579 Mon Sep 17 00:00:00 2001 From: creegon2 <315603738+creegon2@users.noreply.github.com> Date: Tue, 20 Oct 2026 05:20:10 -0700 Subject: [PATCH 06/11] fix(core): preserve retryable cleanup facts and nullable scopes Treat only failed effects bound to the original residual cleanup operation as retryable observations, preserving both failure and later success. Keep successful conclusions and historical provisioning failures immutable. Accept completed cleanup receipts through the existing effect-result intake without restoring command authority or changing operation versions. Avoid UUID queries for absent Workspace scopes. Add both cleanup retry/epoch tests and real NULL-scope intake coverage; focused clone regressions, lint, formatting and the D2 race suite pass. --- docs/agent-runs/residual-reclamation.en.md | 6 + docs/agent-runs/residual-reclamation.md | 4 + integration/agent_run_residual_facts_test.go | 6 +- internal/core/agent_run_residual_fact.go | 24 ++- .../agent_run_residual_reclaim_db_test.go | 173 +++++++++++++++++- internal/core/control.go | 16 ++ 6 files changed, 219 insertions(+), 10 deletions(-) diff --git a/docs/agent-runs/residual-reclamation.en.md b/docs/agent-runs/residual-reclamation.en.md index a98fd72a..dd4962ff 100644 --- a/docs/agent-runs/residual-reclamation.en.md +++ b/docs/agent-runs/residual-reclamation.en.md @@ -58,6 +58,12 @@ Authorized cleanup follows existing quiesce → terminate → cleanup. A quiesce the original authority and guards for evidence collection and retry (A3). It neither restores new execution eligibility, escalates to force-stop, nor creates another operation. +A failed `sandbox_terminate` / `workspace_data_delete` effect bound to this responsibility is a retryable observation, not a disposal conclusion. +After fresh evidence, the same effect may advance to success while retaining both observations. Historical provisioning/execution failures, +external identity bindings and successful conclusions remain protected against contradictions. +After completion, the existing effect-result intake only preserves original bound facts or audits conflicts; it cannot advance the operation, +change its version or grant execution authority again. + Every original sandbox/ensure ID requires a corresponding successful termination result containing both `terminated` and `lateEnsureFenced`. `terminated_at`, a failed ensure, a 404 or elapsed time alone is insufficient. The current safe target can reuse owned delete; an additional historical ID without both confirmations remains open, without replacing its identity or dispatching another termination effect (A5). diff --git a/docs/agent-runs/residual-reclamation.md b/docs/agent-runs/residual-reclamation.md index a695ca4b..cf69a1f0 100644 --- a/docs/agent-runs/residual-reclamation.md +++ b/docs/agent-runs/residual-reclamation.md @@ -48,6 +48,10 @@ execution、clone、work 和数据目标。迟到事实可以补充原目标, 获授权后沿已有 quiesce → terminate → cleanup 推进。quiesce 拒绝使原 operation 留在 `retry_wait`,保留原责任及保护,稍后重新取证再试(A3)。 它不恢复新执行资格,不升级为强停,也不重新创建 operation。 +该责任绑定的 `sandbox_terminate` / `workspace_data_delete` effect 失败是可重试观察,不是已处置结论。 +重新取证后同一 effect 可推进到成功,失败与成功证据均保留;历史创建/执行失败、外部身份和已成功结论仍受冲突保护。 +责任完成后,既有 effect 结果入口只接管原绑定事实或提交冲突审计,不推进 operation、改变其版本或重新授予执行资格。 + 每个原计划 sandbox/ensure ID 都需要对应成功终止结果同时包含 `terminated` 和 `lateEnsureFenced`。 仅有 `terminated_at`、ensure 失败、404 或经过一段时间都不够。当前安全目标可以复用已有 owned delete;额外历史 ID 缺少双确认时保持未完成, 不为它改写目标或另发终止 effect(A5)。数据删除 effect 的计划、许可和完成再次检查目标覆盖;同一事务确认数据删除成功、operation 成功、 diff --git a/integration/agent_run_residual_facts_test.go b/integration/agent_run_residual_facts_test.go index 5a1b375b..a46f5615 100644 --- a/integration/agent_run_residual_facts_test.go +++ b/integration/agent_run_residual_facts_test.go @@ -34,6 +34,10 @@ func TestResidualProvisioningFactsAcrossReclaimStatesOverGRPC(t *testing.T) { planned, err := c.operations.PlanEffect(c.ctx, &controlpb.PlanEffectRequest{SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, Version: snapshot.GetOperation().GetVersion(), Kind: controlpb.EffectKind_EFFECT_KIND_SANDBOX_ENSURE, WorkspaceId: workspace}) must(t, err) effect := planned.GetEffect().GetId() + // Older ensure plans may carry no Workspace on the effect itself. Its original + // operation still binds the run Workspace, so intake must use that durable identity. + _, err = f.store.Pool.Exec(`UPDATE external_effects SET workspace_id=NULL WHERE id=$1 AND operation_id=$2`, effect, operation) + must(t, err) clone, plugin := "late-clone-"+state, "late-plugin-"+state _, err = f.store.Pool.Exec(`INSERT INTO clone_executions(execution_id,operation_id,workspace_id,node_id,input,dispatched_epoch,node_operation_id) VALUES($1,$2,$3,'node-late','{"kind":"clone"}',1,$1::text)`, clone, operation, workspace) @@ -115,7 +119,7 @@ func TestResidualProvisioningFactsAcrossReclaimStatesOverGRPC(t *testing.T) { AND (SELECT count(DISTINCT e->>'key') FROM jsonb_array_elements(r.reclaim_evidence) e WHERE e->>'key'=ANY($7::text[]))=4 AND (SELECT result=$5::jsonb FROM clone_executions WHERE execution_id=$4) AND (SELECT result=$6::jsonb FROM node_executions WHERE execution_id=$8) - AND (SELECT state='succeeded' AND external_id='substrate-late' + AND (SELECT workspace_id IS NULL AND state='succeeded' AND external_id='substrate-late' AND result=jsonb_build_object('sandboxInstanceId',$9::text,'nodeId','node-late') FROM external_effects WHERE id=$9::uuid) AND (SELECT phase='done' AND status='failed' FROM issue_runs WHERE id=$1) AND (SELECT NOT admission_open FROM workspaces WHERE id=$10) diff --git a/internal/core/agent_run_residual_fact.go b/internal/core/agent_run_residual_fact.go index fcf1257b..1f7d1a90 100644 --- a/internal/core/agent_run_residual_fact.go +++ b/internal/core/agent_run_residual_fact.go @@ -1,10 +1,16 @@ package core -import "fmt" +import ( + "fmt" + "strings" +) // residualForWorkspace also finds cleared responsibilities: cleanup never closes the original // identity's evidence intake. Soft-deleted Workspace rows remain the identity authority. func residualForWorkspace(t *transaction, workspaceID string) Object { + if workspaceID == "" { + return nil + } return t.one("SELECT * FROM agent_run_residuals WHERE workspace_id=$1", workspaceID) } @@ -56,13 +62,27 @@ func appendResidualFact(t *transaction, run, key string, fact Object) bool { if previous == nil { previous = row.O("responsibility").O("lateFacts").O(key) } - if len(previous) > 0 && residualFactTerminal(previous) && !sameResidualFact(previous, fact) { + if len(previous) > 0 && residualFactTerminal(previous) && !residualCleanupFailureRetryable(t, row, key, previous) && !sameResidualFact(previous, fact) { return appendResidualConflict(t, row, key, previous, fact) } appendResidualEntry(t, row, key, fact, false) return true } +// A3 retries the same bound cleanup operation and effect. Its failed observation does not prove +// disposal and can advance after fresh evidence; keep that observation instead of treating the +// later success as an A4 contradiction. Historical provisioning failures and every successful +// conclusion remain immutable, and external binding conflicts are checked before this path. +func residualCleanupFailureRetryable(t *transaction, row Object, key string, fact Object) bool { + if fact.S("state") != "failed" || fact.S("kind") != "effect" || row.S("reclaimMode") != "owned_delete" || + row.S("reclaimOperationId") == "" || !strings.HasPrefix(key, "effect:") { + return false + } + id := strings.TrimPrefix(key, "effect:") + return validID(id) && t.one(`SELECT id FROM external_effects WHERE id=$1 AND operation_id=$2 + AND kind IN ('sandbox_terminate','workspace_data_delete')`, id, row.S("reclaimOperationId")) != nil +} + func appendResidualConflict(t *transaction, row Object, key string, previous, incoming Object) bool { t.residualFactConflict = true for _, entry := range residualObjects(row["reclaimEvidence"]) { diff --git a/internal/core/agent_run_residual_reclaim_db_test.go b/internal/core/agent_run_residual_reclaim_db_test.go index 1c13ddd1..b2e8edff 100644 --- a/internal/core/agent_run_residual_reclaim_db_test.go +++ b/internal/core/agent_run_residual_reclaim_db_test.go @@ -104,6 +104,21 @@ func seedResidualAction(t *testing.T, scene residualReclaimScene, op Object, act // Drive the shipped quiesce -> terminate -> cleanup topology, leaving cleanup -> done to callers. func seedResidualCleanup(t *testing.T, scene residualReclaimScene) Object { + t.Helper() + op, effect := seedResidualCleanupPlan(t, scene) + return seedResidualAction(t, scene, op, "effect_result", Object{"state": "succeeded", "externalId": "residual-cleanup", "result": Object{"removed": true}}, effect.S("id")).O("operation") +} + +func seedResidualCleanupPlan(t *testing.T, scene residualReclaimScene) (Object, Object) { + t.Helper() + op, termination := seedResidualTerminationPlan(t, scene) + op = seedResidualAction(t, scene, op, "effect_result", Object{"state": "succeeded", "externalId": "residual-termination", "result": Object{"terminated": true, "lateEnsureFenced": true}}, termination.S("id")).O("operation") + op = seedResidualAction(t, scene, op, "advance", nil, "").O("operation") + planned := seedResidualAction(t, scene, op, "plan", Object{"kind": "workspace_data_delete", "workspaceId": scene.workspace}, "") + return planned.O("operation"), planned.O("effect") +} + +func seedResidualTerminationPlan(t *testing.T, scene residualReclaimScene) (Object, Object) { t.Helper() op := seedResidualClaim(t, scene) seedResidualRuntimeAck(t, scene, false) @@ -117,13 +132,157 @@ func seedResidualCleanup(t *testing.T, scene residualReclaimScene) Object { } op = seedResidualAction(t, scene, op, "advance", nil, "").O("operation") planned := seedResidualAction(t, scene, op, "plan", Object{"kind": "sandbox_terminate", "workspaceId": scene.workspace}, "") - op = planned.O("operation") - out := seedResidualAction(t, scene, op, "effect_result", Object{"state": "succeeded", "externalId": "residual-termination", "result": Object{"terminated": true, "lateEnsureFenced": true}}, planned.O("effect").S("id")) - op = out.O("operation") - op = seedResidualAction(t, scene, op, "advance", nil, "").O("operation") - planned = seedResidualAction(t, scene, op, "plan", Object{"kind": "workspace_data_delete", "workspaceId": scene.workspace}, "") - op = planned.O("operation") - return seedResidualAction(t, scene, op, "effect_result", Object{"state": "succeeded", "externalId": "residual-cleanup", "result": Object{"removed": true}}, planned.O("effect").S("id")).O("operation") + return planned.O("operation"), planned.O("effect") +} + +func TestResidualOwnedCleanupRetriesSameFailedEffectAcrossControllerEpoch(t *testing.T) { + for _, kind := range []string{"sandbox_terminate", "workspace_data_delete"} { + t.Run(kind, func(t *testing.T) { seedResidualRetryCleanupEffect(t, kind) }) + } +} + +func seedResidualRetryCleanupEffect(t *testing.T, kind string) { + t.Helper() + scene := seedResidualScene(t, true) + var op, cleanup Object + success := Object{"removed": true} + if kind == "sandbox_terminate" { + op, cleanup = seedResidualTerminationPlan(t, scene) + success = Object{"terminated": true, "lateEnsureFenced": true} + } else { + op, cleanup = seedResidualCleanupPlan(t, scene) + } + cleanupID := cleanup.S("id") + var reclaimID, terminationID string + if err := scene.store.Pool.QueryRow(`SELECT r.reclaim_id::text,e.id::text FROM agent_run_residuals r JOIN external_effects e ON e.operation_id=r.reclaim_operation_id AND e.kind='sandbox_terminate' WHERE r.run_id=$1`, scene.run).Scan(&reclaimID, &terminationID); err != nil { + t.Fatal(err) + } + failed := Object{"state": "failed", "externalId": "residual-cleanup-retry", "result": Object{"error": "temporary_storage_unavailable"}} + op = seedResidualAction(t, scene, op, "effect_result", failed, cleanupID).O("operation") + op = seedResidualAction(t, scene, op, "defer", Object{"state": "retry_wait", "errorCode": "node_unavailable", "retrySeconds": 1}, "").O("operation") + seedResidualExec(t, scene.store, `UPDATE operations SET retry_at=clock_timestamp()-interval '1 second' WHERE id=$1`, op.S("id")) + _, err := scene.store.Control(t.Context(), &ControlRequest{Action: "lease_release", Body: Object{"epoch": 1}, Service: scene.claims}) + if err != nil { + t.Fatal(err) + } + restarted := seedResidualRebuildStore(scene.store) + lease, err := restarted.Control(t.Context(), &ControlRequest{Action: "lease_acquire", Body: Object{}, Service: scene.claims}) + if err != nil { + t.Fatal(err) + } + epoch := lease.N("epoch") + if epoch != 2 { + t.Fatalf("replacement Controller epoch=%d, want 2", epoch) + } + if err := restarted.ReconcileAgentRunResidualsOnce(t.Context()); err != nil { + t.Fatal(err) + } + claimed, err := restarted.Control(t.Context(), &ControlRequest{Action: "claim", Body: Object{"epoch": epoch}, Service: scene.claims}) + if err != nil { + t.Fatal(err) + } + if current := claimed.O("operation"); current.S("id") != op.S("id") || current.N("controllerEpoch") != epoch { + t.Fatalf("retry replaced its owned operation: %v", claimed) + } else { + op = current + } + control := func(action, effect string, body Object) (Object, error) { + body["epoch"], body["version"] = epoch, op.N("version") + out, err := restarted.Control(t.Context(), &ControlRequest{Action: action, OperationID: op.S("id"), EffectID: effect, Body: body, Service: scene.claims}) + if err == nil { + if current := out.O("operation"); len(current) > 0 { + op = current + } else if out.S("id") != "" { + op = out + } + } + return out, err + } + // Reconcile each original effect under the replacement epoch before reusing its plan. These + // are fresh accepted observations, not mutation of the old effect or invention of a new one. + if kind == "workspace_data_delete" { + if _, err := control("effect_result", terminationID, Object{"state": "succeeded", "externalId": "residual-termination", "result": Object{"terminated": true, "lateEnsureFenced": true}}); err != nil { + t.Fatal(err) + } + } + if _, err := control("effect_result", cleanupID, Object{"state": "failed", "externalId": "residual-cleanup-retry", "result": Object{"error": "temporary_storage_unavailable"}}); err != nil { + t.Fatal(err) + } + planned, err := control("plan", "", Object{"kind": kind, "workspaceId": scene.workspace}) + if err != nil { + t.Fatal(err) + } + if planned.O("effect").S("id") != cleanupID { + t.Fatalf("retry substituted a new cleanup effect: %v", planned) + } + if _, err := control("effect_result", cleanupID, Object{"state": "succeeded", "externalId": "residual-cleanup-retry", "result": success}); err != nil { + t.Fatalf("same-effect failed-to-success retry was rejected: %v", err) + } + if kind == "sandbox_terminate" { + if _, err := control("advance", "", Object{}); err != nil { + t.Fatal(err) + } + dataPlan, err := control("plan", "", Object{"kind": "workspace_data_delete", "workspaceId": scene.workspace}) + if err != nil { + t.Fatal(err) + } + if _, err := control("effect_result", dataPlan.O("effect").S("id"), Object{"state": "succeeded", "externalId": "post-termination-cleanup", "result": Object{"removed": true}}); err != nil { + t.Fatal(err) + } + } + finished, err := control("advance", "", Object{}) + if err != nil { + t.Fatal(err) + } + if finished.S("state") != "succeeded" { + t.Fatalf("retry did not finish cleanup: %v", finished) + } + completedVersion := finished.N("version") + if completedVersion <= 1 { + t.Fatalf("cleanup did not advance its operation version: %v", finished) + } + lateResult := func(submission, message string) (Object, error) { + // The succeeded operation is no longer executable. Its original binding still accepts + // facts through Control with the current lease and a deliberately stale operation version. + return restarted.Control(t.Context(), &ControlRequest{ + Action: "effect_result", OperationID: op.S("id"), EffectID: cleanupID, SubmissionID: submission, + Body: Object{"epoch": epoch, "version": 1, "state": "failed", "externalId": "residual-cleanup-retry", "result": Object{"error": message}}, Service: scene.claims, + }) + } + // A previously accepted failed observation can replay after success, even after clearing. It + // must neither downgrade the effect nor manufacture a conflict or another evidence entry. + for attempt := 0; attempt < 2; attempt++ { + replayed, err := lateResult("cleared-prior-failure-"+kind, "temporary_storage_unavailable") + if err != nil { + t.Fatalf("consistent historical failed observation did not replay through Control: %v", err) + } + if replayed.O("operation").N("version") != completedVersion || replayed.O("effect").S("state") != "succeeded" { + t.Fatalf("cleared fact replay changed canonical state/version: %v", replayed) + } + } + seedResidualAssert(t, scene, `SELECT r.reclaim_id=$2::uuid AND r.reclaim_operation_id=$3::uuid AND r.reclaim_state='cleared' + AND r.disposition='released' AND r.reclaim_conflicted_at IS NULL AND r.responsibility=$5::jsonb + AND e.state='succeeded' AND e.reconciled_epoch=$6 AND e.result=$7::jsonb + AND (SELECT version=$8 AND state='succeeded' FROM operations WHERE id=r.reclaim_operation_id) + AND (SELECT count(*) FROM external_effects WHERE operation_id=r.reclaim_operation_id)=2 + AND (SELECT count(*) FROM operations WHERE workspace_id=r.workspace_id AND kind='delete_workspace')=1 + AND (SELECT string_agg(entry->'fact'->>'state',',' ORDER BY ordinal) FROM jsonb_array_elements(r.reclaim_evidence) WITH ORDINALITY facts(entry,ordinal) + WHERE entry->>'key'='effect:'||$4::text)='failed,succeeded' + FROM agent_run_residuals r JOIN external_effects e ON e.id=$4::uuid WHERE r.run_id=$1`, scene.run, reclaimID, op.S("id"), cleanupID, scene.responsibility, epoch, jsonText(success), completedVersion) + // A changed post-success conclusion remains an A4 contradiction, durably audited even after + // the business/physical cleanup completed. It cannot rewrite the accepted success. + for attempt := 0; attempt < 2; attempt++ { + _, err = lateResult("cleared-conflicting-failure-"+kind, "different_post_success_failure") + assertResidualFactConflict(t, err) + } + seedResidualAssert(t, scene, `SELECT r.reclaim_state='cleared' AND r.reclaim_conflicted_at IS NOT NULL + AND r.disposition='released' AND r.cleared_at IS NOT NULL + AND (SELECT version=$4 AND state='succeeded' FROM operations WHERE id=r.reclaim_operation_id) + AND e.state='succeeded' AND e.result=$3::jsonb + AND (SELECT count(*) FROM jsonb_array_elements(r.reclaim_evidence) entry WHERE entry->>'key'='effect:'||$2::text + AND entry->'fact'->>'kind'='conflict' AND entry->'fact'->'original'->>'state'='succeeded' + AND entry->'fact'->'incoming'->'result'->>'error'='different_post_success_failure')=1 + FROM agent_run_residuals r JOIN external_effects e ON e.id=$2::uuid WHERE r.run_id=$1`, scene.run, cleanupID, jsonText(success), completedVersion) } func seedResidualAssert(t *testing.T, scene residualReclaimScene, query string, args ...any) { diff --git a/internal/core/control.go b/internal/core/control.go index 029109f6..4ad3414c 100644 --- a/internal/core/control.go +++ b/internal/core/control.go @@ -104,6 +104,17 @@ func (s *Store) Control(ctx context.Context, r *ControlRequest) (Object, error) // operationCommand runs one Effect-level action on the operation the caller claimed. func operationCommand(t *transaction, r *ControlRequest) Object { + if r.Action == "effect_result" && validID(r.OperationID) { + // A completed residual operation has no execution authority left, but its original + // effect receipts still arrive through this existing intake. Do not require a running + // operation or its old version merely to preserve an identical fact or audit a conflict. + completed := t.one(`SELECT o.* FROM operations o JOIN agent_run_residuals ar ON ar.reclaim_operation_id=o.id + WHERE o.id=$1 AND o.kind='delete_workspace' AND o.state='succeeded' + AND ar.reclaim_state='cleared' AND ar.reclaim_mode='owned_delete'`, r.OperationID) + if completed != nil { + return effectResult(t, r, completed) + } + } o := operation(t, r) switch r.Action { case "snapshot": @@ -338,6 +349,11 @@ func effectResult(t *transaction, r *ControlRequest, o Object) Object { if !acceptResidualEffectFact(t, residual, e, fact) { return Object{"effect": e, "operation": t.one("SELECT * FROM operations WHERE id=$1", o.S("id"))} } + if residual.S("reclaimState") == "cleared" && residual.S("reclaimOperationId") == o.S("id") { + // Receipt only: neither canonical effects, operation versions nor resource rows may + // advance again after atomic completion. Contradictions already committed their block. + return Object{"effect": e, "operation": o} + } } if r.Body.S("state") == "absent" { require(e.S("state") == "planned", 409, "external_state_conflict") From 9558dbcb58ebcac4cc496c71cfc369cd9413b2ad Mon Sep 17 00:00:00 2001 From: creegon2 <315603738+creegon2@users.noreply.github.com> Date: Tue, 20 Oct 2026 05:38:00 -0700 Subject: [PATCH 07/11] fix(core): keep residual cleanup deferrals claimable The existing Controller requests blocked for unconfirmed termination, but hidden run Workspaces cannot use the public retry action. Normalize only active unconflicted owned cleanup deferrals to retry_wait, retaining the same operation, error and delay while plans and permits recheck evidence. Extend the real Controller retry test to request blocked and prove the same operation resumes under the next epoch. --- docs/agent-runs/residual-reclamation.en.md | 2 ++ docs/agent-runs/residual-reclamation.md | 1 + internal/core/agent_run_residual_reclaim_db_test.go | 12 +++++++++++- internal/core/control.go | 6 ++++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/agent-runs/residual-reclamation.en.md b/docs/agent-runs/residual-reclamation.en.md index dd4962ff..155be8d6 100644 --- a/docs/agent-runs/residual-reclamation.en.md +++ b/docs/agent-runs/residual-reclamation.en.md @@ -57,6 +57,8 @@ Entries with insufficient evidence remain open. Authorized cleanup follows existing quiesce → terminate → cleanup. A quiesce refusal leaves the original operation in `retry_wait`, retaining the original authority and guards for evidence collection and retry (A3). It neither restores new execution eligibility, escalates to force-stop, nor creates another operation. +When the existing Controller requests `blocked` for unconfirmed termination, this system responsibility instead persists claimable `retry_wait`; +actual plans and permits still recheck evidence. A failed `sandbox_terminate` / `workspace_data_delete` effect bound to this responsibility is a retryable observation, not a disposal conclusion. After fresh evidence, the same effect may advance to success while retaining both observations. Historical provisioning/execution failures, diff --git a/docs/agent-runs/residual-reclamation.md b/docs/agent-runs/residual-reclamation.md index cf69a1f0..92271906 100644 --- a/docs/agent-runs/residual-reclamation.md +++ b/docs/agent-runs/residual-reclamation.md @@ -47,6 +47,7 @@ execution、clone、work 和数据目标。迟到事实可以补充原目标, 获授权后沿已有 quiesce → terminate → cleanup 推进。quiesce 拒绝使原 operation 留在 `retry_wait`,保留原责任及保护,稍后重新取证再试(A3)。 它不恢复新执行资格,不升级为强停,也不重新创建 operation。 +既有 Controller 因终止未确认而请求 `blocked` 时,此系统责任也保存为可领取的 `retry_wait`;实际计划和许可仍重新检查证据。 该责任绑定的 `sandbox_terminate` / `workspace_data_delete` effect 失败是可重试观察,不是已处置结论。 重新取证后同一 effect 可推进到成功,失败与成功证据均保留;历史创建/执行失败、外部身份和已成功结论仍受冲突保护。 diff --git a/internal/core/agent_run_residual_reclaim_db_test.go b/internal/core/agent_run_residual_reclaim_db_test.go index b2e8edff..5c2c8dc8 100644 --- a/internal/core/agent_run_residual_reclaim_db_test.go +++ b/internal/core/agent_run_residual_reclaim_db_test.go @@ -159,7 +159,17 @@ func seedResidualRetryCleanupEffect(t *testing.T, kind string) { } failed := Object{"state": "failed", "externalId": "residual-cleanup-retry", "result": Object{"error": "temporary_storage_unavailable"}} op = seedResidualAction(t, scene, op, "effect_result", failed, cleanupID).O("operation") - op = seedResidualAction(t, scene, op, "defer", Object{"state": "retry_wait", "errorCode": "node_unavailable", "retrySeconds": 1}, "").O("operation") + deferredState, deferredCode := "retry_wait", "node_unavailable" + if kind == "sandbox_terminate" { + // The existing Controller uses blocked for an unconfirmed termination. A residual + // responsibility must normalize that request to a claimable retry_wait, not strand + // the hidden Workspace behind an unavailable public/operator retry action. + deferredState, deferredCode = "blocked", "termination_unconfirmed" + } + op = seedResidualAction(t, scene, op, "defer", Object{"state": deferredState, "errorCode": deferredCode, "retrySeconds": 1}, "").O("operation") + if op.S("state") != "retry_wait" || op.S("errorCode") != deferredCode { + t.Fatalf("owned cleanup defer is not retryable: %v", op) + } seedResidualExec(t, scene.store, `UPDATE operations SET retry_at=clock_timestamp()-interval '1 second' WHERE id=$1`, op.S("id")) _, err := scene.store.Control(t.Context(), &ControlRequest{Action: "lease_release", Body: Object{"epoch": 1}, Service: scene.claims}) if err != nil { diff --git a/internal/core/control.go b/internal/core/control.go index 4ad3414c..b379f125 100644 --- a/internal/core/control.go +++ b/internal/core/control.go @@ -132,6 +132,12 @@ func operationCommand(t *transaction, r *ControlRequest) Object { require(deferCodes[code], 400, "invalid_error_code") delay := r.Body.N("retrySeconds") require(delay >= 1 && delay <= 3600, 400, "invalid_retry_delay") + if row := residualOperation(t, o); row != nil && row.S("reclaimMode") == "owned_delete" && row.S("reclaimState") == "reclaiming" && row["reclaimConflictedAt"] == nil { + // A3 has no public/operator retry path for this hidden Workspace. Preserve the + // same retryable operation even when the existing Controller requests blocked + // for an unconfirmed termination; plan/permit still require fresh real evidence. + state = "retry_wait" + } if o.S("step") == "clone" && code == "clone_failed" { deferWorkspaceClone(t, o) return t.one("SELECT * FROM operations WHERE id=$1", o.S("id")) From 12e8625e7ea940f639a214e33de1a20edaa3f1da Mon Sep 17 00:00:00 2001 From: creegon2 <315603738+creegon2@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:26:09 -0700 Subject: [PATCH 08/11] fix(core): persist residual clone receipt conflicts Audit conflicting clone event receipts in the residual ledger before returning residual_fact_conflict, preserving the canonical result and receipt. Keep the ordinary receipt_conflict rollback and the node-execution path unchanged. Cover receipt takeover over gRPC in reconciling, reclaiming and cleared states, including committed conflict blocking, replay stability and the no-ledger path. Validation: checkformat, lint, race-enabled receipt regressions, internal/core tests and residual/clone-related integration tests passed with PostgreSQL. --- .../agent_run_residual_clone_receipt_test.go | 195 ++++++++++++++++++ internal/core/agent_run_residual_fact.go | 15 ++ internal/core/clone.go | 3 + 3 files changed, 213 insertions(+) create mode 100644 integration/agent_run_residual_clone_receipt_test.go diff --git a/integration/agent_run_residual_clone_receipt_test.go b/integration/agent_run_residual_clone_receipt_test.go new file mode 100644 index 00000000..2eadceec --- /dev/null +++ b/integration/agent_run_residual_clone_receipt_test.go @@ -0,0 +1,195 @@ +package integration + +import ( + "context" + "encoding/base64" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + "github.com/wanglongan587/cloud/internal/controlpb" +) + +// Each state is a staged ledger precondition, not a claim that recovery reached it. The +// original result and receipt enter through the same real takeover RPC as the contradiction. +func TestResidualCloneReceiptConflictAcrossReclaimStatesOverGRPC(t *testing.T) { + for _, state := range []string{"reconciling", "reclaiming", "cleared"} { + t.Run(state, func(t *testing.T) { + f, c, run, workspace, operation, original := cloneReceiptScene(t) + _, err := c.operations.DeferOperation(c.ctx, &controlpb.DeferOperationRequest{ + SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation.GetId(), Version: operation.GetVersion(), + State: controlpb.DeferState_DEFER_STATE_BLOCKED, Reason: controlpb.DeferReason_DEFER_REASON_NODE_UNAVAILABLE, RetrySeconds: 60, + }) + must(t, err) + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '1 second' WHERE id=$1`, run) + must(t, err) + must(t, f.store.GiveUpBlockedWorkspaceCreationsOnce(context.Background())) + + // A running disposal operation gives PlanEffect a valid operation/version. Its + // rejection must come from the newly committed conflict, not stale-operation fencing. + disposal := uuid.NewString() + _, err = f.store.Pool.Exec(`INSERT INTO operations(id,tenant_id,actor_user_id,project_id,workspace_id, + kind,state,step,request,idempotency_key,request_hash,controller_epoch) + SELECT $2::uuid,w.tenant_id,w.creator_user_id,w.project_id,w.id,'delete_workspace','running','terminate', + '{}','clone-receipt-disposal','clone-receipt-fixture',$3 FROM workspaces w WHERE w.id=$1`, workspace, disposal, c.epoch) + must(t, err) + if state != "reconciling" { + _, err = f.store.Pool.Exec(`UPDATE agent_run_residuals SET reclaim_state=$2,reclaim_id=$3, + reclaim_mode='controlled',reclaim_operation_id=$3,reclaim_authorization='{"source":"test-prestate"}', + cleared_at=CASE WHEN $2='cleared' THEN now() END, + disposition=CASE WHEN $2='cleared' THEN 'externally_confirmed' END WHERE run_id=$1`, run, state, disposal) + must(t, err) + } + before := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()) + var responsibility string + must(t, f.store.Pool.QueryRow(`SELECT responsibility::text FROM agent_run_residuals WHERE run_id=$1`, run).Scan(&responsibility)) + conflict := proto.Clone(original).(*controlpb.TakeOverNodeEventRequest) + conflict.SubmissionId, conflict.Event = c.sub(), []byte("clone event B") + response, err := f.executions.TakeOverNodeEvent(c.ctx, conflict) + expectStatus(t, err, codes.Aborted, controlpb.ErrorCode_ERROR_CODE_CONFLICT) + if response != nil || status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("receipt-only conflict was not durably rejected: response=%v error=%v", response, err) + } + + var retained bool + must(t, f.store.Pool.QueryRow(`SELECT r.reclaim_conflicted_at IS NOT NULL + AND r.reclaim_state=$2 AND (r.cleared_at IS NOT NULL)=($2='cleared') AND r.responsibility=$3::jsonb + AND (SELECT count(*) FROM jsonb_array_elements(r.reclaim_evidence) x + WHERE x->>'key'=$4 AND x->'fact'->>'kind'='conflict' + AND x->'fact'->'original'->>'kind'='receipt' AND x->'fact'->'original'->>'event'=$5 + AND x->'fact'->'incoming'->>'kind'='receipt' AND x->'fact'->'incoming'->>'event'=$6 + AND x->>'receivedAt'<>'' AND x->'source'->>'submissionId'=$7)=1 + AND EXISTS(SELECT 1 FROM control_submissions WHERE submission_id=$7 AND response->>'residualFactConflict'='true') + FROM agent_run_residuals r WHERE r.run_id=$1`, run, state, responsibility, + "receipt:"+original.GetExecutionId()+":1", base64.StdEncoding.EncodeToString(original.GetEvent()), + base64.StdEncoding.EncodeToString(conflict.GetEvent()), conflict.GetSubmissionId()).Scan(&retained)) + if !retained { + t.Fatal("conflict error rolled back receipt evidence, replay response, or original ledger state") + } + if after := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()); after != before { + t.Fatalf("receipt conflict changed original clone result or event receipt:\nbefore=%s\nafter=%s", before, after) + } + + var evidence string + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object('evidence',reclaim_evidence, + 'version',reclaim_version,'conflictedAt',reclaim_conflicted_at)::text FROM agent_run_residuals WHERE run_id=$1`, run).Scan(&evidence)) + for _, submission := range []string{conflict.GetSubmissionId(), c.sub()} { + conflict.SubmissionId = submission + response, err = f.executions.TakeOverNodeEvent(c.ctx, conflict) + expectStatus(t, err, codes.Aborted, controlpb.ErrorCode_ERROR_CODE_CONFLICT) + if response != nil || status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("conflicting receipt replay lost its stable result: response=%v error=%v", response, err) + } + } + var replayStable bool + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object('evidence',reclaim_evidence, + 'version',reclaim_version,'conflictedAt',reclaim_conflicted_at)=$2::jsonb FROM agent_run_residuals WHERE run_id=$1`, run, evidence).Scan(&replayStable)) + if !replayStable { + t.Fatal("receipt conflict replay duplicated evidence or moved conflict/version metadata") + } + + var disposalVersion int64 + var disposalBefore string + must(t, f.store.Pool.QueryRow(`SELECT version,jsonb_build_object('state',state,'step',step, + 'version',version,'updatedAt',updated_at,'effects',(SELECT count(*) FROM external_effects WHERE operation_id=$1))::text + FROM operations WHERE id=$1`, disposal).Scan(&disposalVersion, &disposalBefore)) + planned, err := c.operations.PlanEffect(c.ctx, &controlpb.PlanEffectRequest{ + SubmissionId: c.sub(), Epoch: c.epoch, OperationId: disposal, Version: disposalVersion, + Kind: controlpb.EffectKind_EFFECT_KIND_SANDBOX_TERMINATE, WorkspaceId: workspace, + }) + expectStatus(t, err, codes.Aborted, controlpb.ErrorCode_ERROR_CODE_CONFLICT) + if planned != nil || status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("committed receipt conflict did not block disposal: response=%v error=%v", planned, err) + } + var disposalAfter string + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object('state',state,'step',step, + 'version',version,'updatedAt',updated_at,'effects',(SELECT count(*) FROM external_effects WHERE operation_id=$1))::text + FROM operations WHERE id=$1`, disposal).Scan(&disposalAfter)) + if disposalAfter != disposalBefore { + t.Fatalf("blocked disposal changed its operation or planned an effect:\nbefore=%s\nafter=%s", disposalBefore, disposalAfter) + } + }) + } +} + +func TestCloneReceiptConflictWithoutResidualWritesNothingOverGRPC(t *testing.T) { + f, c, _, _, _, original := cloneReceiptScene(t) + before := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()) + var bookkeepingBefore string + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object( + 'residuals',(SELECT jsonb_agg(to_jsonb(r) ORDER BY run_id) FROM agent_run_residuals r), + 'submissions',(SELECT jsonb_agg(jsonb_build_object('id',submission_id,'holder',holder_id, + 'response',response,'createdAt',created_at) ORDER BY submission_id) FROM control_submissions))::text`).Scan(&bookkeepingBefore)) + conflict := proto.Clone(original).(*controlpb.TakeOverNodeEventRequest) + conflict.SubmissionId, conflict.Event = c.sub(), []byte("clone event B") + response, err := f.executions.TakeOverNodeEvent(c.ctx, conflict) + expectStatus(t, err, codes.Aborted, controlpb.ErrorCode_ERROR_CODE_CONFLICT) + if response != nil || status.Convert(err).Message() != "receipt_conflict" { + t.Fatalf("ordinary receipt conflict changed behavior: response=%v error=%v", response, err) + } + if after := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()); after != before { + t.Fatalf("ordinary receipt conflict changed canonical data:\nbefore=%s\nafter=%s", before, after) + } + var bookkeepingAfter string + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object( + 'residuals',(SELECT jsonb_agg(to_jsonb(r) ORDER BY run_id) FROM agent_run_residuals r), + 'submissions',(SELECT jsonb_agg(jsonb_build_object('id',submission_id,'holder',holder_id, + 'response',response,'createdAt',created_at) ORDER BY submission_id) FROM control_submissions))::text`).Scan(&bookkeepingAfter)) + if bookkeepingAfter != bookkeepingBefore { + t.Fatalf("ordinary receipt conflict wrote residual evidence or a control submission:\nbefore=%s\nafter=%s", bookkeepingBefore, bookkeepingAfter) + } +} + +func cloneReceiptScene(t *testing.T) (*fixture, *grpcController, string, string, *controlpb.Operation, *controlpb.TakeOverNodeEventRequest) { + t.Helper() + f := setup(t) + f.bindBusinessHooks() + created := f.create("clone-receipt-conflict") + f.drain() + run := f.insertAgentRun(t, created.O("resource").S("id")) + provisioned, err := f.store.CreateRunWorkspace(context.Background(), run) + must(t, err) + workspace, operation := provisioned.O("workspace").S("id"), provisioned.O("operation").S("id") + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET phase='provisioning',status='dispatched',workspace_id=$2,version=version+1 WHERE id=$1`, run, workspace) + must(t, err) + c := newGRPCController(t, f) + snapshot := c.claim() + if snapshot.GetOperation().GetId() != operation { + t.Fatalf("claim picked %s, want %s", snapshot.GetOperation().GetId(), operation) + } + planned, err := c.operations.PlanEffect(c.ctx, &controlpb.PlanEffectRequest{ + SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, Version: snapshot.GetOperation().GetVersion(), + Kind: controlpb.EffectKind_EFFECT_KIND_SANDBOX_ENSURE, WorkspaceId: workspace, + }) + must(t, err) + execution := "clone-receipt-execution" + _, err = f.store.Pool.Exec(`INSERT INTO clone_executions(execution_id,operation_id,workspace_id,node_id,input,dispatched_epoch,node_operation_id) + VALUES($1,$2,$3,'node-late','{"kind":"clone"}',$4,$1::text)`, execution, operation, workspace, c.epoch) + must(t, err) + original := &controlpb.TakeOverNodeEventRequest{ + SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, ExecutionId: execution, Sequence: 1, + Result: &controlpb.ExecutionResult{ + Node: &controlpb.NodeIdentity{NodeId: "node-late", NodeIncarnationId: "inc-late"}, + Outcome: &controlpb.ExecutionResult_CloneFailed{CloneFailed: &controlpb.CloneFailed{Reason: controlpb.CloneFailureReason_CLONE_FAILURE_REASON_BRANCH_NOT_FOUND}}, + }, + Event: []byte("clone event A"), + } + taken, err := f.executions.TakeOverNodeEvent(c.ctx, original) + must(t, err) + if taken.GetRecord().GetExecutionId() != execution || !proto.Equal(taken.GetRecord().GetResult(), original.GetResult()) { + t.Fatalf("initial clone takeover did not preserve its original result: %v", taken) + } + return f, c, run, workspace, planned.GetOperation(), original +} + +func cloneReceiptCanonicalSnapshot(t *testing.T, f *fixture, execution string) string { + t.Helper() + var snapshot string + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object('clone',to_jsonb(e), + 'receipts',(SELECT jsonb_agg(to_jsonb(r) ORDER BY sequence) FROM clone_event_receipts r WHERE r.execution_id=e.execution_id))::text + FROM clone_executions e WHERE e.execution_id=$1`, execution).Scan(&snapshot)) + return snapshot +} diff --git a/internal/core/agent_run_residual_fact.go b/internal/core/agent_run_residual_fact.go index 1f7d1a90..2be50213 100644 --- a/internal/core/agent_run_residual_fact.go +++ b/internal/core/agent_run_residual_fact.go @@ -159,6 +159,21 @@ func acceptResidualReceipt(t *transaction, e Object, sequence int64, event strin return true } +// Clone receipts use a separate registry. Audit a conflicting event before cloneResult can +// reject it with a rollback, retaining the original receipt and blocking further disposal. +func acceptResidualCloneReceipt(t *transaction, e Object, sequence int64, event string) bool { + row := residualForWorkspace(t, e.S("workspaceId")) + if row == nil { + return true + } + old := t.one("SELECT event FROM clone_event_receipts WHERE execution_id=$1 AND sequence=$2", e.S("executionId"), sequence) + if old != nil && old.S("event") != event { + key := "receipt:" + e.S("executionId") + ":" + fmt.Sprint(sequence) + return appendResidualConflict(t, row, key, Object{"kind": "receipt", "event": old.S("event")}, Object{"kind": "receipt", "event": event}) + } + return true +} + func sameResidualFact(a, b Object) bool { // Legacy facts also contain receipt timestamps and provisional history. Neither changes the // terminal's semantic identity when importing an existing receipt into the append-only column. diff --git a/internal/core/clone.go b/internal/core/clone.go index 8b5f3e67..09e0bebf 100644 --- a/internal/core/clone.go +++ b/internal/core/clone.go @@ -284,6 +284,9 @@ func cloneResult(t *transaction, r *ControlRequest, withReceipt bool) Object { if withReceipt { sequence, event := r.Body.N("sequence"), r.Body.S("event") require(sequence >= 0 && event != "", 400, "invalid_receipt") + if !acceptResidualCloneReceipt(t, e, sequence, event) { + return e + } if old := t.one("SELECT * FROM clone_event_receipts WHERE execution_id=$1 AND sequence=$2", execution, sequence); old != nil { require(old.S("event") == event, 409, "receipt_conflict") } else { From 01b8ae8bcb32d7c85c57475ea5918b4c2160eaf8 Mon Sep 17 00:00:00 2001 From: creegon2 <315603738+creegon2@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:16:09 -0700 Subject: [PATCH 09/11] fix(core): check clone receipt conflicts before accepting results Check residual clone receipts before importing execution facts so a rejected receipt only persists its conflict audit. Share the node and clone receipt helper, keeping node evidence keys stable and using a separate clone prefix. Add a gRPC regression for results held only in legacy responsibility facts. Verified checkformat, lint, receipt takeover tests with -race, core tests, and the residual/clone integration tests with PostgreSQL required. --- .../agent_run_residual_clone_receipt_test.go | 76 ++++++++++++++++--- internal/core/agent_run_residual_fact.go | 27 +++---- internal/core/clone.go | 6 +- internal/core/node_execution.go | 2 +- 4 files changed, 78 insertions(+), 33 deletions(-) diff --git a/integration/agent_run_residual_clone_receipt_test.go b/integration/agent_run_residual_clone_receipt_test.go index 2eadceec..5fcf8404 100644 --- a/integration/agent_run_residual_clone_receipt_test.go +++ b/integration/agent_run_residual_clone_receipt_test.go @@ -18,20 +18,12 @@ import ( func TestResidualCloneReceiptConflictAcrossReclaimStatesOverGRPC(t *testing.T) { for _, state := range []string{"reconciling", "reclaiming", "cleared"} { t.Run(state, func(t *testing.T) { - f, c, run, workspace, operation, original := cloneReceiptScene(t) - _, err := c.operations.DeferOperation(c.ctx, &controlpb.DeferOperationRequest{ - SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation.GetId(), Version: operation.GetVersion(), - State: controlpb.DeferState_DEFER_STATE_BLOCKED, Reason: controlpb.DeferReason_DEFER_REASON_NODE_UNAVAILABLE, RetrySeconds: 60, - }) - must(t, err) - _, err = f.store.Pool.Exec(`UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '1 second' WHERE id=$1`, run) - must(t, err) - must(t, f.store.GiveUpBlockedWorkspaceCreationsOnce(context.Background())) + f, c, run, workspace, original := cloneReceiptResidualScene(t) // A running disposal operation gives PlanEffect a valid operation/version. Its // rejection must come from the newly committed conflict, not stale-operation fencing. disposal := uuid.NewString() - _, err = f.store.Pool.Exec(`INSERT INTO operations(id,tenant_id,actor_user_id,project_id,workspace_id, + _, err := f.store.Pool.Exec(`INSERT INTO operations(id,tenant_id,actor_user_id,project_id,workspace_id, kind,state,step,request,idempotency_key,request_hash,controller_epoch) SELECT $2::uuid,w.tenant_id,w.creator_user_id,w.project_id,w.id,'delete_workspace','running','terminate', '{}','clone-receipt-disposal','clone-receipt-fixture',$3 FROM workspaces w WHERE w.id=$1`, workspace, disposal, c.epoch) @@ -64,7 +56,7 @@ func TestResidualCloneReceiptConflictAcrossReclaimStatesOverGRPC(t *testing.T) { AND x->>'receivedAt'<>'' AND x->'source'->>'submissionId'=$7)=1 AND EXISTS(SELECT 1 FROM control_submissions WHERE submission_id=$7 AND response->>'residualFactConflict'='true') FROM agent_run_residuals r WHERE r.run_id=$1`, run, state, responsibility, - "receipt:"+original.GetExecutionId()+":1", base64.StdEncoding.EncodeToString(original.GetEvent()), + "clone_receipt:"+original.GetExecutionId()+":1", base64.StdEncoding.EncodeToString(original.GetEvent()), base64.StdEncoding.EncodeToString(conflict.GetEvent()), conflict.GetSubmissionId()).Scan(&retained)) if !retained { t.Fatal("conflict error rolled back receipt evidence, replay response, or original ledger state") @@ -115,6 +107,54 @@ func TestResidualCloneReceiptConflictAcrossReclaimStatesOverGRPC(t *testing.T) { } } +func TestResidualCloneReceiptConflictDoesNotImportLegacyResultOverGRPC(t *testing.T) { + f, c, run, _, original := cloneReceiptResidualScene(t) + // Model a pre-migration terminal fact: its result exists only in lateFacts, + // with no accepted clone fact in the new evidence ledger. + _, err := f.store.Pool.Exec(`UPDATE agent_run_residuals r SET + responsibility=jsonb_set(r.responsibility,'{lateFacts}',jsonb_build_object( + 'clone:'||e.execution_id,jsonb_build_object('kind','clone','state','failed','result',e.result))), + reclaim_version=7 FROM clone_executions e WHERE r.run_id=$1 AND e.execution_id=$2`, run, original.GetExecutionId()) + must(t, err) + var responsibility string + var version int64 + var emptyEvidence bool + must(t, f.store.Pool.QueryRow(`SELECT responsibility::text,reclaim_version, + reclaim_evidence='[]'::jsonb FROM agent_run_residuals WHERE run_id=$1`, run).Scan(&responsibility, &version, &emptyEvidence)) + if !emptyEvidence { + t.Fatal("legacy fixture already has evidence in the reclaim ledger") + } + before := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()) + conflict := proto.Clone(original).(*controlpb.TakeOverNodeEventRequest) + conflict.SubmissionId, conflict.Event = c.sub(), []byte("clone event B") + response, err := f.executions.TakeOverNodeEvent(c.ctx, conflict) + expectStatus(t, err, codes.Aborted, controlpb.ErrorCode_ERROR_CODE_CONFLICT) + if response != nil || status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("legacy receipt conflict was not durably rejected: response=%v error=%v", response, err) + } + + var retained bool + must(t, f.store.Pool.QueryRow(`SELECT r.responsibility=$2::jsonb + AND r.reclaim_version=$3+1 AND r.reclaim_conflicted_at IS NOT NULL + AND jsonb_array_length(r.reclaim_evidence)=1 + AND r.reclaim_evidence->0->>'key'=$4 + AND r.reclaim_evidence->0->'fact'=jsonb_build_object('kind','conflict','state','conflict', + 'original',jsonb_build_object('kind','receipt','event',$5::text), + 'incoming',jsonb_build_object('kind','receipt','event',$6::text)) + AND r.reclaim_evidence->0->>'receivedAt'<>'' + AND r.reclaim_evidence->0->'source'->>'submissionId'=$7 + AND EXISTS(SELECT 1 FROM control_submissions WHERE submission_id=$7 AND response->>'residualFactConflict'='true') + FROM agent_run_residuals r WHERE r.run_id=$1`, run, responsibility, version, + "clone_receipt:"+original.GetExecutionId()+":1", base64.StdEncoding.EncodeToString(original.GetEvent()), + base64.StdEncoding.EncodeToString(conflict.GetEvent()), conflict.GetSubmissionId()).Scan(&retained)) + if !retained { + t.Fatal("rejected receipt must append exactly one conflict and one version, preserving lateFacts without importing an ordinary clone fact") + } + if after := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()); after != before { + t.Fatalf("legacy receipt conflict changed original clone result or event receipt:\nbefore=%s\nafter=%s", before, after) + } +} + func TestCloneReceiptConflictWithoutResidualWritesNothingOverGRPC(t *testing.T) { f, c, _, _, _, original := cloneReceiptScene(t) before := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()) @@ -143,6 +183,20 @@ func TestCloneReceiptConflictWithoutResidualWritesNothingOverGRPC(t *testing.T) } } +func cloneReceiptResidualScene(t *testing.T) (*fixture, *grpcController, string, string, *controlpb.TakeOverNodeEventRequest) { + t.Helper() + f, c, run, workspace, operation, original := cloneReceiptScene(t) + _, err := c.operations.DeferOperation(c.ctx, &controlpb.DeferOperationRequest{ + SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation.GetId(), Version: operation.GetVersion(), + State: controlpb.DeferState_DEFER_STATE_BLOCKED, Reason: controlpb.DeferReason_DEFER_REASON_NODE_UNAVAILABLE, RetrySeconds: 60, + }) + must(t, err) + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '1 second' WHERE id=$1`, run) + must(t, err) + must(t, f.store.GiveUpBlockedWorkspaceCreationsOnce(context.Background())) + return f, c, run, workspace, original +} + func cloneReceiptScene(t *testing.T) (*fixture, *grpcController, string, string, *controlpb.Operation, *controlpb.TakeOverNodeEventRequest) { t.Helper() f := setup(t) diff --git a/internal/core/agent_run_residual_fact.go b/internal/core/agent_run_residual_fact.go index 2be50213..d103825e 100644 --- a/internal/core/agent_run_residual_fact.go +++ b/internal/core/agent_run_residual_fact.go @@ -146,29 +146,20 @@ func acceptResidualEffectFact(t *transaction, row, e, incoming Object) bool { return appendResidualFact(t, row.S("runId"), key, incoming) } -func acceptResidualReceipt(t *transaction, e Object, sequence int64, event string) bool { +// receipts is a constant selected by the node/clone registry call sites, never request input. +func acceptResidualReceipt(t *transaction, e Object, sequence int64, event, receipts string) bool { row := residualForWorkspace(t, e.S("workspaceId")) if row == nil { return true } - old := t.one("SELECT event FROM node_event_receipts WHERE execution_id=$1 AND sequence=$2", e.S("executionId"), sequence) + old := t.one("SELECT event FROM "+receipts+" WHERE execution_id=$1 AND sequence=$2", e.S("executionId"), sequence) if old != nil && old.S("event") != event { - key := "receipt:" + e.S("executionId") + ":" + fmt.Sprint(sequence) - return appendResidualConflict(t, row, key, Object{"kind": "receipt", "event": old.S("event")}, Object{"kind": "receipt", "event": event}) - } - return true -} - -// Clone receipts use a separate registry. Audit a conflicting event before cloneResult can -// reject it with a rollback, retaining the original receipt and blocking further disposal. -func acceptResidualCloneReceipt(t *transaction, e Object, sequence int64, event string) bool { - row := residualForWorkspace(t, e.S("workspaceId")) - if row == nil { - return true - } - old := t.one("SELECT event FROM clone_event_receipts WHERE execution_id=$1 AND sequence=$2", e.S("executionId"), sequence) - if old != nil && old.S("event") != event { - key := "receipt:" + e.S("executionId") + ":" + fmt.Sprint(sequence) + prefix := "receipt:" + // Separate primary keys do not guarantee uniqueness across the two execution registries. + if receipts == "clone_event_receipts" { + prefix = "clone_receipt:" + } + key := prefix + e.S("executionId") + ":" + fmt.Sprint(sequence) return appendResidualConflict(t, row, key, Object{"kind": "receipt", "event": old.S("event")}, Object{"kind": "receipt", "event": event}) } return true diff --git a/internal/core/clone.go b/internal/core/clone.go index 09e0bebf..9b3ecc71 100644 --- a/internal/core/clone.go +++ b/internal/core/clone.go @@ -268,6 +268,9 @@ func cloneResult(t *transaction, r *ControlRequest, withReceipt bool) Object { require(e != nil, 404, "not_found") outcome := result.S("outcome") require((outcome == "clone_ready" || outcome == "clone_failed") && result.O("node").S("nodeId") == e.S("nodeId"), 409, "result_conflict") + if withReceipt && !acceptResidualReceipt(t, e, r.Body.N("sequence"), r.Body.S("event"), "clone_event_receipts") { + return e + } if !acceptResidualExecutionFact(t, e, result) { return e } @@ -284,9 +287,6 @@ func cloneResult(t *transaction, r *ControlRequest, withReceipt bool) Object { if withReceipt { sequence, event := r.Body.N("sequence"), r.Body.S("event") require(sequence >= 0 && event != "", 400, "invalid_receipt") - if !acceptResidualCloneReceipt(t, e, sequence, event) { - return e - } if old := t.one("SELECT * FROM clone_event_receipts WHERE execution_id=$1 AND sequence=$2", execution, sequence); old != nil { require(old.S("event") == event, 409, "receipt_conflict") } else { diff --git a/internal/core/node_execution.go b/internal/core/node_execution.go index 5b688b8b..0c3eb917 100644 --- a/internal/core/node_execution.go +++ b/internal/core/node_execution.go @@ -116,7 +116,7 @@ func nodeExecutionResult(t *transaction, r *ControlRequest, withReceipt bool) Ob require(e != nil, 404, "not_found") require(result.O("node").S("nodeId") == e.S("nodeId"), 409, "result_conflict") validateNodeResult(e, result) - if withReceipt && !acceptResidualReceipt(t, e, r.Body.N("sequence"), r.Body.S("event")) { + if withReceipt && !acceptResidualReceipt(t, e, r.Body.N("sequence"), r.Body.S("event"), "node_event_receipts") { return e } if !acceptResidualExecutionFact(t, e, result) { From bce3ca19fb0b36cc4732473ac6efba4f2da931e2 Mon Sep 17 00:00:00 2001 From: creegon2 <315603738+creegon2@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:06:33 -0700 Subject: [PATCH 10/11] docs(core): clarify residual takeover and retain migration notes Restore main's bilingual 0050 migration explanation and document the intentional clone validation order and RPC fixture boundary. Verification: checkformat and lint (0 issues); core 301 pass; receipt race tests 5 pass; targeted integration 30 pass, with one known Windows git http-backend failure. --- integration/agent_run_residual_facts_test.go | 1 + internal/core/clone.go | 1 + internal/core/migrations/README.en.md | 2 ++ internal/core/migrations/README.md | 2 ++ 4 files changed, 6 insertions(+) diff --git a/integration/agent_run_residual_facts_test.go b/integration/agent_run_residual_facts_test.go index a5996735..b09a1cff 100644 --- a/integration/agent_run_residual_facts_test.go +++ b/integration/agent_run_residual_facts_test.go @@ -39,6 +39,7 @@ func TestResidualProvisioningFactsAcrossReclaimStatesOverGRPC(t *testing.T) { _, err = f.store.Pool.Exec(`UPDATE external_effects SET workspace_id=NULL WHERE id=$1 AND operation_id=$2`, effect, operation) must(t, err) clone, plugin := "late-clone-"+state, "late-plugin-"+state + // Only pre-seed the execution row; the entry under test is the real RPC. _, err = f.store.Pool.Exec(`INSERT INTO clone_executions(execution_id,operation_id,workspace_id,node_id,input,dispatched_epoch,node_operation_id) VALUES($1,$2,$3,'node-late','{"kind":"clone"}',1,$1::text)`, clone, operation, workspace) must(t, err) diff --git a/internal/core/clone.go b/internal/core/clone.go index f23329c5..21465b69 100644 --- a/internal/core/clone.go +++ b/internal/core/clone.go @@ -119,6 +119,7 @@ func cloneResult(t *transaction, r *ControlRequest, withReceipt bool) Object { require((outcome == "clone_ready" || outcome == "clone_failed") && result.O("node").S("nodeId") == e.S("nodeId"), 409, "result_conflict") // A checkout of any other commit than the snapshot it was given is a contradiction, not a // fact to record: the baseline would be wrong for every later delivery and resume (contract D3). + // Validate before receipt auditing intentionally: invalid results return 400 without audit, as on the Node path. require(outcome != "clone_ready" || result.S("commit") == e.O("input").S("baseCommit"), 400, "base_commit_mismatch") if withReceipt && !acceptResidualReceipt(t, e, r.Body.N("sequence"), r.Body.S("event"), "clone_event_receipts") { return e diff --git a/internal/core/migrations/README.en.md b/internal/core/migrations/README.en.md index b871ac34..a9a21119 100644 --- a/internal/core/migrations/README.en.md +++ b/internal/core/migrations/README.en.md @@ -68,6 +68,8 @@ There are currently 57 SQL files. | [0054_git_delivery_refs.sql](0054_git_delivery_refs.sql) | Revisions delivered over Git | | [0055_agent_run_residual_reclamation.sql](0055_agent_run_residual_reclamation.sql) | Residual reclamation responsibility, appended evidence and disposition | +0050 copies no personal connection or key and backfills no historical run. Clearing configuration retains its revision; run options are stored separately from shared run input. + ## Immutability and upgrades - Never edit or rename applied SQL; corrections use new forward migrations. Repeated application skips applied files with matching checksums. diff --git a/internal/core/migrations/README.md b/internal/core/migrations/README.md index 1eb5a71f..48047bf0 100644 --- a/internal/core/migrations/README.md +++ b/internal/core/migrations/README.md @@ -68,6 +68,8 @@ SQL 通过 `core/store.go` 的 `embed.FS` 编入二进制。`Store.Migrate` 在 | [0054_git_delivery_refs.sql](0054_git_delivery_refs.sql) | 经 Git 交付的 Revision | | [0055_agent_run_residual_reclamation.sql](0055_agent_run_residual_reclamation.sql) | 残留回收责任、追加证据和处置状态 | +0050 不复制个人连接或 Key,不回填历史运行;清除配置保留 revision,运行选项独立于共享运行输入保存。 + ## 不可变性与升级 - 已应用 SQL 不改内容、不改文件名;更正用新的前向迁移。重复应用跳过 checksum 相同的已应用文件。 From 022760c47efb2e7660fe47d3850407f2b6ee3219 Mon Sep 17 00:00:00 2001 From: creegon2 <315603738+creegon2@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:15:26 -0700 Subject: [PATCH 11/11] docs(migrations): retain main's original 0050 explanation Remove the duplicate 0050 upgrade bullets introduced by the residual branch, leaving main's original paragraphs and only residual-specific migration additions. SQL and Go code are unchanged from the completed round-two gates. --- internal/core/migrations/README.en.md | 1 - internal/core/migrations/README.md | 1 - 2 files changed, 2 deletions(-) diff --git a/internal/core/migrations/README.en.md b/internal/core/migrations/README.en.md index a9a21119..83895e04 100644 --- a/internal/core/migrations/README.en.md +++ b/internal/core/migrations/README.en.md @@ -80,7 +80,6 @@ There are currently 57 SQL files. - 0045 adds `agent_configuration` (association and byte-exact static copy) and `agent_configuration_issue` (bounded reason) without backfilling existing rows. Only validated declarations are consumable; absence and invalid declarations are distinct, and invalid declarations do not block package installation. See [Plugin marketplace](../../../docs/plugins/marketplace.en.md) and `TestMigration0045AgentConfigurationCatalogAppliesFreshAndUpgrades` for fresh/upgrade/repeat coverage. - 0046–0048 backfill no old messages, Threads, history positions or pause state; `visible_text` is stored separately from the raw `record`, and old tasks are not paused automatically. Fresh, upgrade and repeated application are covered by `TestMigration0046TaskConversationFreshAndUpgrade` and the integration migration tests. - 0049 lets `thread_commands` accept a WorkflowRun owner and requires a graph node on workflow commands; each node holds at most one `end_session`, so the scheduler can replay it safely. Every old row carries `run_id` and is not backfilled. Semantics are in [workflows](../../../docs/workflows/workflows.en.md); coverage is in the `TestWorkflowSessionEndsAfterSuccessfulTurn` DB tests. -- 0050 copies no personal connection or key and backfills no historical run. Clearing configuration retains its revision; run options are stored separately from shared run input. - 0051 adds the cancellation-request instant to `workflow_runs` and backfills no old row; the request records intent only and is settled by the scheduler on later passes. Semantics are in [workflows](../../../docs/workflows/workflows.en.md#scheduling); coverage is in the `TestWorkflowRunCancelBeforeStartNeverStarts` DB tests. - 0052 settles the live `executor_type='workflow'` IssueRuns still `queued`/`dispatched` at upgrade time: they have no executor and no frozen workflow identity, so the ADR's D3 marks them failed with `failure_reason='workflow_execution_unavailable'` instead of migrating them. Other runs and soft-deleted rows are untouched. Rows of that shape stop being produced only once the workflow launch gate binds a manual run to a real WorkflowRun; until that gate (circus#71) lands, `createRun` keeps producing new ones, and this migration runs once at upgrade time and does not settle them. Coverage is `TestMigration0052LegacyWorkflowIssueRunSettlementAppliesFreshAndUpgrades`. - 0053 is the switch to Workspaces cloned from Git Server snapshots and migrates no data: clone executions that belong to no Workspace and their receipts are deleted, `clone_executions.clone_request_id` and the `one_request_execution` index are dropped, `clone_executions.workspace_id` becomes NOT NULL, and `clone_requests` is dropped; Workspace clone executions and their receipts stay, and it is retry-safe. See [Workspaces](../../../docs/runtime/workspaces.en.md); upgrading a database that still holds tenant-level clones is covered by `TestMigration0053RetiresTenantLevelClonesKeepingWorkspaceClones`. diff --git a/internal/core/migrations/README.md b/internal/core/migrations/README.md index 48047bf0..7de2f203 100644 --- a/internal/core/migrations/README.md +++ b/internal/core/migrations/README.md @@ -80,7 +80,6 @@ SQL 通过 `core/store.go` 的 `embed.FS` 编入二进制。`Store.Migrate` 在 - 0045 增加 `agent_configuration`(关联与逐字节静态副本)和 `agent_configuration_issue`(有界原因),不回填既有行;只保存校验通过的可消费声明,区分未声明与声明无效,后者不阻止包安装。语义见 [插件市场](../../../docs/plugins/marketplace.md),空库、升级与重复应用覆盖见 `TestMigration0045AgentConfigurationCatalogAppliesFreshAndUpgrades`。 - 0046–0048 不回填旧消息、旧 Thread、历史位置或暂停状态;`visible_text` 与原始 `record` 分开保存,旧任务不自动暂停。空库、升级与重复应用覆盖见 `TestMigration0046TaskConversationFreshAndUpgrade` 及 integration 的迁移测试。 - 0049 让 `thread_commands` 接受 WorkflowRun 归属,并要求工作流命令带图节点;每个节点至多一条 `end_session`,调度器可安全重放。旧行全部带 `run_id`,不回填。语义见 [工作流](../../../docs/workflows/workflows.md),覆盖见 `TestWorkflowSessionEndsAfterSuccessfulTurn` 等 DB 测试。 -- 0050 不复制个人连接或 Key,不回填历史运行;清除配置保留 revision,运行选项独立于共享运行输入保存。 - 0051 给 `workflow_runs` 增加取消请求时刻,不回填旧行;请求只记录意图,由调度器在后续轮次结算。语义见[工作流](../../../docs/workflows/workflows.md#调度),覆盖见 `TestWorkflowRunCancelBeforeStartNeverStarts` 等 DB 测试。 - 0052 结算升级时仍为 `queued`/`dispatched` 的、存活的 `executor_type='workflow'` IssueRun:它们没有执行器、也没有冻结的工作流身份,按 ADR D3 标记失败并写 `failure_reason='workflow_execution_unavailable'`,不做迁移;其他状态的行与软删除行保持原样。工作流启动门禁把 manual 运行绑定到真实 WorkflowRun 之后才会停止产生这种行;门禁(circus#71)落地前 `createRun` 仍会造出新行,而本迁移只在升级时执行一次,不会回头结算它们。覆盖见 `TestMigration0052LegacyWorkflowIssueRunSettlementAppliesFreshAndUpgrades`。 - 0053 是 Workspace 改为从 Git Server 快照 clone 的切换,不迁移数据:删除不属于 Workspace 的 clone execution 及其收据,去掉 `clone_executions.clone_request_id` 与索引 `one_request_execution`,`clone_executions.workspace_id` 改为 NOT NULL,删除 `clone_requests`;Workspace 的 clone execution 与收据保留,可重复执行。语义见 [Workspace](../../../docs/runtime/workspaces.md),仍有租户级 clone 的库的升级覆盖见 `TestMigration0053RetiresTenantLevelClonesKeepingWorkspaceClones`。