From 22bac95f5679fe7d461ae6607f5969e09d407157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=BE=99=E5=AE=89?= <297888591+wanglongan587@users.noreply.github.com> Date: Sun, 25 Oct 2026 01:28:04 +0800 Subject: [PATCH] feat(cloudctl): explicitly reclaim D8 residuals without a delete operation --- cmd/cloudctl/README.en.md | 5 +- cmd/cloudctl/README.md | 5 +- cmd/cloudctl/residual.go | 6 +- cmd/cloudctl/residual_test.go | 7 + docs/agent-runs/residual-reclamation.en.md | 9 +- docs/agent-runs/residual-reclamation.md | 7 +- ...t_run_controlled_missing_delete_db_test.go | 188 ++++++++++++++++++ internal/core/agent_run_controlled_reclaim.go | 36 +++- .../agent_run_controlled_reclaim_delete.go | 4 +- .../agent_run_controlled_reclaim_query.go | 10 +- .../0070_d8_missing_delete_operation.sql | 12 ++ internal/core/migrations/README.en.md | 5 +- internal/core/migrations/README.md | 5 +- 13 files changed, 277 insertions(+), 22 deletions(-) create mode 100644 internal/core/agent_run_controlled_missing_delete_db_test.go create mode 100644 internal/core/migrations/0070_d8_missing_delete_operation.sql diff --git a/cmd/cloudctl/README.en.md b/cmd/cloudctl/README.en.md index 58f1e146..ccab8ede 100644 --- a/cmd/cloudctl/README.en.md +++ b/cmd/cloudctl/README.en.md @@ -42,10 +42,13 @@ cloudctl -config residual get --tenant-id --intent-id Each `list` item, actual residual ledger rows from `get`, and the resource output when present expose persisted `stop_evidence` as `stopEvidence`; the resource is returned only when no intent is selected. Without evidence the field is an empty object `{}`; legacy D8 list items without a ledger also return `{}`, while a synthetic legacy D8 residual in `get` may omit the field. An existing unconfirmed-stop record retains `kind=stop_unconfirmed`, `source=cloud`, `observedAt`, and the original-identity `executionId`. This field is display-only: it grants no termination, deletion, or ledger-clearance authority and does not change reclamation responsibility. -Termination and data deletion are separate commands, each requiring a reason, idempotency key, and affirmative confirmation. `terminate` requires at least a Run or Workspace; all additional IDs must identify the same original resource. D8 also requires the original failed delete `--operation-id`. Project/tenant selectors cannot authorize batch writes. +Termination and data deletion are separate commands, each requiring a reason, idempotency key, and affirmative confirmation. `terminate` requires at least a Run or Workspace; all additional IDs must identify the same original resource. D8 with a historical deletion requires the original failed delete `--operation-id`. Project/tenant selectors cannot authorize batch writes. + +A D8 residual that reached `done/workspace_unavailable` before any deletion was created additionally requires explicit `--allow-missing-delete-operation`, both exact `--run-id` and `--workspace-id`, and no `--operation-id`. It cannot select a live Run, an absent delivery conclusion, any deletion history, open admission, or an existing residual ledger. The intent records `legacyDeleteOperationMissing=true` without inventing an operation; the default command still rejects this variant. Termination, fencing, and separately confirmed data deletion retain their evidence requirements. ```sh cloudctl -config residual terminate --tenant-id --run-id --workspace-id --reason 'operator abandoned execution' --key --impact-confirmed +cloudctl -config residual terminate --tenant-id --run-id --workspace-id --allow-missing-delete-operation --reason 'verified recovery copies retained' --key --impact-confirmed cloudctl -config residual delete-data --tenant-id --intent-id --reason 'delete retained data' --key --data-delete-confirmed ``` diff --git a/cmd/cloudctl/README.md b/cmd/cloudctl/README.md index fbcb3031..7e880d00 100644 --- a/cmd/cloudctl/README.md +++ b/cmd/cloudctl/README.md @@ -42,10 +42,13 @@ cloudctl -config residual get --tenant-id --intent-id `list` 的每个条目、`get` 的实际 residual 账本行,以及存在时的资源输出都通过 `stopEvidence` 展示持久化的 `stop_evidence`;资源仅在未选择意图时返回。没有证据时输出空对象 `{}`;没有账本的历史 D8 列表条目也输出 `{}`,但 `get` 中合成的历史 D8 residual 可能省略该字段。已有停止未确认记录会保留 `kind=stop_unconfirmed`、`source=cloud`、`observedAt` 和原身份的 `executionId`。此字段仅供查看,不授予终止、删除或清账权限,也不改变回收责任。 -终止和删数据是两次独立命令,各自需要 reason、幂等 key 和真实确认。`terminate` 至少指定 Run 或 Workspace;提供的其他 ID 必须属于同一个原资源。D8 还必须提供原 failed delete 的 `--operation-id`。Project/tenant 不能作为批量写入范围。 +终止和删数据是两次独立命令,各自需要 reason、幂等 key 和真实确认。`terminate` 至少指定 Run 或 Workspace;提供的其他 ID 必须属于同一个原资源。D8 有历史删除操作时必须提供原 failed delete 的 `--operation-id`。Project/tenant 不能作为批量写入范围。 + +对已按 D8 达到 `done/workspace_unavailable`、却从未创建任何删除操作的残留,必须额外显式选择 `--allow-missing-delete-operation`,同时提供精确的 `--run-id` 和 `--workspace-id`,且不提供 `--operation-id`。此选择不能用于仍在运行、没有交付结论、有删除历史、admission 开放或已有残留账本的资源。意图记录 `legacyDeleteOperationMissing=true`,不补造历史 operation;默认命令仍拒绝这一变体。后续终止、围栏和单独删数据的证据要求相同。 ```sh cloudctl -config residual terminate --tenant-id --run-id --workspace-id --reason 'operator abandoned execution' --key --impact-confirmed +cloudctl -config residual terminate --tenant-id --run-id --workspace-id --allow-missing-delete-operation --reason 'verified recovery copies retained' --key --impact-confirmed cloudctl -config residual delete-data --tenant-id --intent-id --reason 'delete retained data' --key --data-delete-confirmed ``` diff --git a/cmd/cloudctl/residual.go b/cmd/cloudctl/residual.go index cad0bb01..0e18d2df 100644 --- a/cmd/cloudctl/residual.go +++ b/cmd/cloudctl/residual.go @@ -40,6 +40,7 @@ func parseResidual(args []string) (core.ResidualManagementRequest, error) { flags.IntVar(&request.Limit, "limit", 100, "page size") flags.StringVar(&request.After, "after", "", "page cursor") flags.BoolVar(&request.ImpactConfirmed, "impact-confirmed", false, "confirm termination impact") + flags.BoolVar(&request.AllowMissingDeleteOperation, "allow-missing-delete-operation", false, "explicitly select a D8 Run with no historical delete operation") flags.BoolVar(&request.DataDeleteConfirmed, "data-delete-confirmed", false, "confirm permanent data deletion") flags.BoolVar(&request.ResponsibilityTransferred, "responsibility-transferred", false, "accept remaining external responsibility") flags.StringVar(&request.EvidenceNote, "evidence-note", "", "external disposition explanation") @@ -74,6 +75,9 @@ func parseResidual(args []string) (core.ResidualManagementRequest, error) { if (request.RunID == "" && request.WorkspaceID == "") || !request.ImpactConfirmed { return request, invalidArguments() } + if request.AllowMissingDeleteOperation && (request.RunID == "" || request.WorkspaceID == "" || request.OperationID != "") { + return request, invalidArguments() + } case "delete-data": request.Action = core.ResidualDeleteData if request.IntentID == "" || !request.DataDeleteConfirmed { @@ -101,7 +105,7 @@ func residualFlags(action string) map[string]bool { case "get": names = []string{"project-id", "run-id", "workspace-id", "operation-id", "reclaim-id", "intent-id"} case "terminate": - names = []string{"project-id", "run-id", "workspace-id", "operation-id", "reason", "key", "operator-label", "impact-confirmed"} + names = []string{"project-id", "run-id", "workspace-id", "operation-id", "reason", "key", "operator-label", "impact-confirmed", "allow-missing-delete-operation"} case "delete-data": names = []string{"intent-id", "reason", "key", "operator-label", "data-delete-confirmed"} case "confirm-external": diff --git a/cmd/cloudctl/residual_test.go b/cmd/cloudctl/residual_test.go index 62ea047b..38727c4d 100644 --- a/cmd/cloudctl/residual_test.go +++ b/cmd/cloudctl/residual_test.go @@ -48,6 +48,11 @@ func TestCloudctlResidualManagement(t *testing.T) { args: []string{"delete-data", "--tenant-id", "tenant", "--intent-id", "root", "--reason", "delete retained data", "--key", "delete-key", "--data-delete-confirmed=true"}, want: core.ResidualManagementRequest{Action: core.ResidualDeleteData, TenantID: "tenant", IntentID: "root", Reason: "delete retained data", Key: "delete-key", DataDeleteConfirmed: true, Limit: 100, ExternalFacts: map[string]string{}}, }, + { + name: "explicit D8 without a historical delete", + args: []string{"terminate", "--tenant-id", "tenant", "--run-id", "run", "--workspace-id", "workspace", "--allow-missing-delete-operation", "--reason", "verified recovery copies", "--key", "missing-delete", "--impact-confirmed"}, + want: core.ResidualManagementRequest{Action: core.ResidualTerminate, TenantID: "tenant", RunID: "run", WorkspaceID: "workspace", AllowMissingDeleteOperation: true, Reason: "verified recovery copies", Key: "missing-delete", ImpactConfirmed: true, Limit: 100, ExternalFacts: map[string]string{}}, + }, { name: "external facts preserve exact references", args: []string{"confirm-external", "--tenant-id", "tenant", "--intent-id", "unfinished", "--reason", "external disposal", "--key", "external-key", "--evidence-note", "host decommissioned", "--external-fact", "sandbox:original=ticket=123", "--external-fact", "data:workspace=ticket-456", "--responsibility-transferred"}, @@ -98,6 +103,8 @@ func TestCloudctlResidualRejectsUnsafeOrAmbiguousArguments(t *testing.T) { terminate, appendCopy(terminate, "--impact-confirmed=false"), appendCopy(terminate, "--impact-confirmed", "--data-delete-confirmed"), + appendCopy(terminate, "--impact-confirmed", "--allow-missing-delete-operation"), + appendCopy(terminate, "--impact-confirmed", "--run-id", "run", "--allow-missing-delete-operation", "--operation-id", "old-delete"), {"terminate", "--tenant-id", "tenant", "--project-id", "project", "--reason", "reason", "--key", "key", "--impact-confirmed"}, {"terminate", "--tenant-id", "tenant", "--run-id", "run", "--reason", " ", "--key", "key", "--impact-confirmed"}, {"terminate", "--tenant-id", "tenant", "--run-id", "run", "--reason", "reason", "--key", " ", "--impact-confirmed"}, diff --git a/docs/agent-runs/residual-reclamation.en.md b/docs/agent-runs/residual-reclamation.en.md index 47379cc2..b2050814 100644 --- a/docs/agent-runs/residual-reclamation.en.md +++ b/docs/agent-runs/residual-reclamation.en.md @@ -91,7 +91,7 @@ Workspaces continue to return 404. | Command | Scope and required confirmation | |---|---| | `list` / `get` | Tenant is required; intersect Project, Run, Workspace, operation or reclaim filters; details expose only operational state, targets, audit and evidence | -| `terminate` | One exact Run/Workspace, reason, idempotency key and explicit impact confirmation; D8 also requires the original failed delete operation; retains data by default | +| `terminate` | One exact Run/Workspace, reason, idempotency key and explicit impact confirmation; D8 requires its original failed deletion or explicit missing-delete selection; retains data by default | | `delete-data` | One root intent with successful physical termination and retained disposition, separate reason/key and data-deletion confirmation; no bare-Workspace deletion | | `confirm-external` | One existing unfinished root or dependent intent, reason/key, explanation, external fact references for every unfinished target and explicit acceptance of remaining responsibility | @@ -122,11 +122,16 @@ It transfers responsibility for an existing intent and stops new permits. It doe permit same-volume reuse or authorize data deletion. Missing intents, succeeded intents, empty explanations and missing targets are rejected. Late physical facts remain evidence; they cannot silently convert external disposition into system-proven success. -D8 takeover requires the original done Run, retained Workspace binding and failed/node_unavailable delete operation; it never backfills the D4/D5 +D8 with a historical deletion requires the original done Run, retained Workspace binding and failed/node_unavailable delete operation; it never backfills the D4/D5 ledger. Without a ledger, the root owns targets, evidence, conflicts and the new delete-operation binding. The old failed operation, Run terminal and settlement remain unchanged. Ordinary scans do not create controlled D8 responsibility. Missing ledger rows or retained disposition cannot bypass admission, Project-operation or data-reuse guards. +When D8 finished before a deletion was declared, the deployment operator must select the variant with exact Run/Workspace IDs and explicit +`--allow-missing-delete-operation`. Only an original `done/workspace_unavailable` Run with a delivery conclusion, no deletion history, +the retained binding and closed admission qualifies. Root and dependent deletion intents retain `legacyDeleteOperationMissing` provenance. +The default command does not expand admission, invent an original operation, or rewrite the business outcome. + ## Existing fact intake and conflicts Cloud uses existing `runtime_ack`, creation-fact, execution-result and Revision intake. This step adds no RPC, proto or Controller discovery diff --git a/docs/agent-runs/residual-reclamation.md b/docs/agent-runs/residual-reclamation.md index 1c0ee89d..512e509b 100644 --- a/docs/agent-runs/residual-reclamation.md +++ b/docs/agent-runs/residual-reclamation.md @@ -74,7 +74,7 @@ Workspace 软删除并写 `cleared/released`,不改 Run 的 phase/status/resul | 命令 | 范围与必要确认 | |---|---| | `list` / `get` | tenant 必填;查询可按 Project、Run、Workspace、operation 或 reclaim 相交过滤,详情只返回运维所需状态、目标、审计与证据 | -| `terminate` | 一个精确 Run/Workspace,reason、幂等键和显式影响确认;D8 还需原 failed delete operation 身份;默认保留数据 | +| `terminate` | 一个精确 Run/Workspace,reason、幂等键和显式影响确认;D8 需原 failed delete operation 身份或明确的无删除记录选择;默认保留数据 | | `delete-data` | 一个已真实终止成功且 retained 的根意图,独立 reason、幂等键和数据删除确认;不能按裸 Workspace 发起 | | `confirm-external` | 一个已存在且未完成的根或从属意图,reason、幂等键、说明、每个未完成目标的外部事实引用及承担剩余责任的确认 | @@ -102,7 +102,10 @@ Workspace 软删除并写 `cleared/released`,不改 Run 的 phase/status/resul 停止该责任的新许可,不伪造 `terminated`、`lateEnsureFenced` 或 `removed`,也不据此开放同卷复用或授权删卷。 没有意图、意图已 succeeded、说明为空或漏目标均拒绝;迟到物理事实继续留存,不能静默把外部处置改成系统证明。 -D8 只按原 done Run、保留的 Workspace 绑定及 failed/node_unavailable delete operation 接管,不回填 D4/D5 账本。 +D8 有原删除操作时,按原 done Run、保留的 Workspace 绑定及 failed/node_unavailable delete operation 接管,不回填 D4/D5 账本。 +若 D8 在删除声明建立前结束,部署运维须以精确 Run/Workspace 和显式 `--allow-missing-delete-operation` 选择无删除操作变体。 +仅原 `done/workspace_unavailable`、已有交付结论、没有任何删除历史、保留原绑定且 admission 关闭的资源可受理;意图与从属删除保留 +`legacyDeleteOperationMissing` 来源标记。默认命令不扩展准入,不补造原 operation,原业务结论仍不变。 无账本时根意图持有目标、证据、冲突和删除 operation 绑定;原 failed operation、Run 终态与结算保持不变。 普通扫描不会自动为 D8 创建受控责任;无账本或已 retained 也不能绕过 admission、Project 操作和数据复用保护。 diff --git a/internal/core/agent_run_controlled_missing_delete_db_test.go b/internal/core/agent_run_controlled_missing_delete_db_test.go new file mode 100644 index 00000000..b1f625f3 --- /dev/null +++ b/internal/core/agent_run_controlled_missing_delete_db_test.go @@ -0,0 +1,188 @@ +package core + +import ( + "reflect" + "testing" + "time" +) + +// The real D8 worker permits a releasing Run to finish before any delete was declared. +func seedControlledD8WithoutDelete(t *testing.T, s *Store) residualReclaimScene { + t.Helper() + legacy := seedReleaseScene(t, s, "releasing", "completed") + sandbox := newID() + seedResidualExec(t, s, `INSERT INTO sandbox_instances(id,workspace_id,generation,observed_state) VALUES($1,$2,1,'running')`, sandbox, legacy.runws) + // The session established control before release; no delete operation creates it in this case. + seedResidualExec(t, s, "INSERT INTO runtime_controls(workspace_id,state) VALUES($1,'reconciling')", legacy.runws) + seedResidualExec(t, s, `UPDATE issue_runs SET result='{"revisionId":null,"deliveryState":"failed"}' WHERE id=$1`, legacy.seed.run) + s.DeliveryUnreachableAfter = time.Second + if err := s.GiveUpStaleWorkspaceReleasesOnce(t.Context()); err != nil { + t.Fatal(err) + } + seedControllerLease(t, s) + return residualReclaimScene{store: s, abandonedCreation: abandonedCreation{provisioningSeed: provisioningSeed{dispSeed: legacy.seed, workspace: legacy.runws}, claims: controllerClaims(), effect: sandbox}} +} + +// Explicit opt-in must be enough to recover this real D8 state without inventing a delete. +func TestControlledD8MissingDeleteRequiresExplicitScopeAndKeepsHistory(t *testing.T) { + scene := seedControlledD8WithoutDelete(t, controlledStores(t)(t)) + r := seedControlledRequest(scene, "missing-delete") + listed := seedControlledManage(t, scene.store, ResidualManagementRequest{Action: ResidualList, TenantID: scene.tenant, RunID: scene.run}) + items := residualObjects(listed["items"]) + if len(items) != 1 || !items[0].B("legacyDeleteOperationMissing") { + t.Fatalf("operator cannot discover the unhandled D8 variant: %v", listed) + } + if _, err := scene.store.ManageResidual(t.Context(), &r); err == nil { + t.Fatal("default termination silently broadened D8 eligibility") + } + r.AllowMissingDeleteOperation = true + before := seedControlledRunSnapshot(t, scene) + root := seedControlledManage(t, scene.store, r).O("intent") + if root.S("source") != "legacy_d8" || !root.B("legacyDeleteOperationMissing") || root["legacyDeleteOperationId"] != nil { + t.Fatalf("missing-delete source was not recorded honestly: %v", root) + } + replay := seedControlledManage(t, scene.store, r).O("intent") + if !reflect.DeepEqual(root, replay) { + t.Fatalf("replay changed the durable intent: %v / %v", root, replay) + } + if got := seedControlledRunSnapshot(t, scene); got != before { + t.Fatalf("management rewrote the original Run: %s / %s", before, got) + } + var count int + if err := scene.store.Pool.QueryRow("SELECT count(*) FROM operations WHERE workspace_id=$1 AND kind='delete_workspace'", scene.workspace).Scan(&count); err != nil || count != 0 { + t.Fatalf("management invented a historical delete: %d / %v", count, err) + } +} + +// The new acknowledgement cannot reinterpret a live, mismatched, or already represented resource. +func TestControlledD8MissingDeleteRejectsOtherResources(t *testing.T) { + reset := controlledStores(t) + for _, name := range []string{"missing run", "missing workspace", "foreign tenant", "wrong run", "not D8", "still releasing", "live status", "no delivery conclusion", "admission open", "existing delete"} { + t.Run(name, func(t *testing.T) { + scene := seedControlledD8WithoutDelete(t, reset(t)) + r := seedControlledRequest(scene, "reject-"+name) + r.AllowMissingDeleteOperation = true + switch name { + case "missing run": + r.RunID = "" + case "missing workspace": + r.WorkspaceID = "" + case "foreign tenant": + r.TenantID = newID() + case "wrong run": + r.RunID = newID() + case "not D8": + seedResidualExec(t, scene.store, "UPDATE issue_runs SET failure_reason='' WHERE id=$1", scene.run) + case "still releasing": + seedResidualExec(t, scene.store, "UPDATE issue_runs SET phase='releasing' WHERE id=$1", scene.run) + case "live status": + seedResidualExec(t, scene.store, "UPDATE issue_runs SET status='queued' WHERE id=$1", scene.run) + case "no delivery conclusion": + seedResidualExec(t, scene.store, "UPDATE issue_runs SET result=NULL WHERE id=$1", scene.run) + case "admission open": + seedResidualExec(t, scene.store, "UPDATE workspaces SET desired_state='running',observed_state='ready',admission_open=true WHERE id=$1", scene.workspace) + case "existing delete": + // An old failed delete is a different variant and still requires its exact ID. + scene = seedControlledLegacyD8(t, reset(t)) + r = seedControlledRequest(scene, "existing-delete") + r.OperationID = "" + r.AllowMissingDeleteOperation = true + } + if _, err := scene.store.ManageResidual(t.Context(), &r); err == nil { + t.Fatalf("missing-delete acknowledgement admitted %s", name) + } + var count int + if err := scene.store.Pool.QueryRow("SELECT count(*) FROM agent_run_reclamation_intents").Scan(&count); err != nil || count != 0 { + t.Fatalf("refusal left a partial intent: %d / %v", count, err) + } + }) + } +} + +// Termination retains the volume; only a separate intent and physical cleanup can release it. +func TestControlledD8MissingDeleteRetainsThenReleasesWithoutRewritingRun(t *testing.T) { + scene := seedControlledD8WithoutDelete(t, controlledStores(t)(t)) + before := seedControlledRunSnapshot(t, scene) + r := seedControlledRequest(scene, "terminate-missing-delete") + r.AllowMissingDeleteOperation = true + root := seedControlledManage(t, scene.store, r).O("intent") + plan := seedControlledPlan(t, scene, root.S("id")) + for _, target := range residualObjects(plan["effects"]) { + if _, err := seedControlledConfirm(t, scene, root.S("id"), target.S("id"), Object{"terminated": true, "lateEnsureFenced": true}); err != nil { + t.Fatal(err) + } + } + root = seedControlledRow(t, scene.store, root.S("id")) + if root.S("state") != "succeeded" || root.S("disposition") != "terminated_data_retained" { + t.Fatalf("termination did not retain data: %v", root) + } + child, op, effect := seedControlledDeletePlan(t, scene, root) + if !child.B("legacyDeleteOperationMissing") || child["legacyDeleteOperationId"] != nil { + t.Fatalf("dependent deletion lost the honest missing-delete source: %v", child) + } + op = seedResidualAction(t, scene, op, "effect_result", Object{"state": "succeeded", "externalId": "missing-delete-cleanup", "result": Object{"removed": true}}, effect.S("id")).O("operation") + finished := seedResidualAction(t, scene, op, "advance", nil, "").O("operation") + if finished.S("state") != "succeeded" { + t.Fatalf("controlled cleanup did not finish: %v", finished) + } + if got := seedControlledRunSnapshot(t, scene); got != before { + t.Fatalf("cleanup rewrote the original Run: %s / %s", before, got) + } + var released bool + if err := scene.store.Pool.QueryRow(`SELECT deleted_at IS NOT NULL AND issue_run_id=$2 + AND (SELECT count(*) FROM operations WHERE workspace_id=$1 AND kind='delete_workspace')=1 + AND NOT EXISTS(SELECT 1 FROM agent_run_residuals WHERE run_id=$2) + FROM workspaces WHERE id=$1`, scene.workspace, scene.run).Scan(&released); err != nil || !released { + t.Fatalf("cleanup failed to release only its new operation: %v / %v", released, err) + } +} + +// Forward migration must preserve old intent identities while adding a safe default. +func TestControlledD8MissingDeleteMigrationUpgradesAndRepeats(t *testing.T) { + s := controlledStores(t)(t) + scene := seedControlledLegacyD8(t, s) + root := seedControlledRoot(t, scene) + var before string + if err := s.Pool.QueryRow("SELECT (to_jsonb(i)-'legacy_delete_operation_missing')::text FROM agent_run_reclamation_intents i WHERE id=$1", root.S("id")).Scan(&before); err != nil { + t.Fatal(err) + } + // Reconstruct the preceding schema only inside this isolated test database. + if _, err := s.Pool.Exec(`ALTER TABLE agent_run_reclamation_intents DROP CONSTRAINT agent_run_reclamation_legacy_delete_identity; + ALTER TABLE agent_run_reclamation_intents DROP COLUMN legacy_delete_operation_missing; + ALTER TABLE agent_run_reclamation_intents ADD CONSTRAINT agent_run_reclamation_intents_check1 + CHECK((source='legacy_d8' AND legacy_delete_operation_id IS NOT NULL) OR (source='residual_ledger' AND legacy_delete_operation_id IS NULL)); + DELETE FROM schema_migrations WHERE version='0070_d8_missing_delete_operation.sql'`); err != nil { + t.Fatal(err) + } + for range 2 { + if err := s.Migrate(t.Context()); err != nil { + t.Fatal(err) + } + if err := s.CheckSchema(t.Context()); err != nil { + t.Fatal(err) + } + } + var after string + var missing bool + if err := s.Pool.QueryRow("SELECT (to_jsonb(i)-'legacy_delete_operation_missing')::text,legacy_delete_operation_missing FROM agent_run_reclamation_intents i WHERE id=$1", root.S("id")).Scan(&after, &missing); err != nil || after != before || missing { + t.Fatalf("upgrade changed the old intent or enabled missing-delete mode: %s / %s / %v / %v", before, after, missing, err) + } + for _, query := range []string{ + "UPDATE agent_run_reclamation_intents SET legacy_delete_operation_missing=true WHERE id=$1", + "UPDATE agent_run_reclamation_intents SET legacy_delete_operation_id=NULL WHERE id=$1", + } { + if _, err := s.Pool.Exec(query, root.S("id")); err == nil { + t.Fatal("migration allowed contradictory original delete evidence") + } + } +} + +// Comparing the complete Run prevents an operational command from rewriting business history. +func seedControlledRunSnapshot(t *testing.T, scene residualReclaimScene) string { + t.Helper() + var out string + if err := scene.store.Pool.QueryRow("SELECT row_to_json(r)::text FROM issue_runs r WHERE id=$1", scene.run).Scan(&out); err != nil { + t.Fatal(err) + } + return out +} diff --git a/internal/core/agent_run_controlled_reclaim.go b/internal/core/agent_run_controlled_reclaim.go index 2f88d644..447ea24f 100644 --- a/internal/core/agent_run_controlled_reclaim.go +++ b/internal/core/agent_run_controlled_reclaim.go @@ -26,8 +26,10 @@ type ResidualManagementRequest struct { Key, Reason, OperatorLabel, After string Limit int ImpactConfirmed, DataDeleteConfirmed, ResponsibilityTransferred bool - EvidenceNote string - ExternalFacts map[string]string + // AllowMissingDeleteOperation explicitly selects the D8 variant with no historical delete. + AllowMissingDeleteOperation bool + EvidenceNote string + ExternalFacts map[string]string } // ManageResidual uses the deployment's database credential, independently of tenant identities. @@ -54,11 +56,18 @@ func manageResidualCommand(t *transaction, r *ResidualManagementRequest) Object return getManagedResidual(t, r) } require(r.Action == ResidualTerminate || r.Action == ResidualDeleteData || r.Action == ResidualConfirmExternal, 400, "invalid_residual_action") + if r.AllowMissingDeleteOperation { + require(r.Action == ResidualTerminate && r.RunID != "" && r.WorkspaceID != "" && r.OperationID == "", 400, "invalid_residual_scope") + } require(r.Key != "" && len(r.Key) <= 200, 400, "idempotency_key_required") r.Reason = validText(r.Reason, 2000) require(len(r.OperatorLabel) <= 200, 400, "invalid_operator_label") actor := t.one("SELECT session_user::text AS actor").S("actor") input := Object{"action": string(r.Action), "tenantId": r.TenantID, "projectId": r.ProjectID, "runId": r.RunID, "workspaceId": r.WorkspaceID, "operationId": r.OperationID, "reclaimId": r.ReclaimID, "intentId": r.IntentID, "reason": r.Reason, "impactConfirmed": r.ImpactConfirmed, "dataDeleteConfirmed": r.DataDeleteConfirmed, "responsibilityTransferred": r.ResponsibilityTransferred, "evidenceNote": r.EvidenceNote, "externalFacts": r.ExternalFacts, "operatorLabel": r.OperatorLabel} + if r.AllowMissingDeleteOperation { + // Omit the default so receipts written before this option keep their canonical input. + input["allowMissingDeleteOperation"] = true + } if old := t.one("SELECT * FROM agent_run_reclamation_command_receipts WHERE tenant_id=$1 AND idempotency_key=$2", r.TenantID, r.Key); old != nil { if jsonText(old.O("commandInput")) != jsonText(input) { event := Object{"kind": "idempotency_conflict", "actorSubject": actor, "incoming": input, "at": controlledNow(t)} @@ -99,18 +108,29 @@ func registerControlledTermination(t *transaction, r *ResidualManagementRequest, if root := controlledRootForWorkspace(t, w.S("id")); root != nil { require(controlledRootRow(t, root)["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") require(r.OperationID == "" || r.OperationID == root.S("legacyDeleteOperationId"), 409, "residual_identity_conflict") + require(r.AllowMissingDeleteOperation == root.B("legacyDeleteOperationMissing"), 409, "residual_identity_conflict") return root } row := t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", w.S("issueRunId")) source := "residual_ledger" if row == nil { source = "legacy_d8" - old := t.one(`SELECT o.* FROM operations o JOIN issue_runs i ON i.id=$2 WHERE o.id=$1 AND o.tenant_id=$3 + if r.AllowMissingDeleteOperation { + // D8 may finish before any delete was declared. Only its original closed Run and + // exact retained Workspace can select this explicitly acknowledged variant. + require(t.one(`SELECT id FROM issue_runs WHERE id=$1 AND tenant_id=$2 AND workspace_id=$3 + AND executor_type='agent' AND phase='done' AND failure_reason='workspace_unavailable' + AND status IN ('completed','failed','cancelled') + AND result->>'deliveryState' IN ('saved','unchanged','failed','skipped')`, w.S("issueRunId"), r.TenantID, w.S("id")) != nil, 409, "legacy_residual_identity_required") + require(t.one("SELECT id FROM operations WHERE workspace_id=$1 AND kind='delete_workspace' LIMIT 1", w.S("id")) == nil, 409, "legacy_residual_identity_required") + } else { + old := t.one(`SELECT o.* FROM operations o JOIN issue_runs i ON i.id=$2 WHERE o.id=$1 AND o.tenant_id=$3 AND o.workspace_id=$4 AND o.kind='delete_workspace' AND o.state='failed' AND o.error_code='node_unavailable' AND i.phase='done'`, nullable(r.OperationID), w.S("issueRunId"), r.TenantID, w.S("id")) - require(old != nil, 409, "legacy_residual_identity_required") + require(old != nil, 409, "legacy_residual_identity_required") + } row = Object{"runId": w.S("issueRunId"), "tenantId": r.TenantID, "projectId": w.S("projectId"), "workspaceId": w.S("id"), "runtimeGeneration": w.N("runtimeGeneration"), "responsibility": Object{}, "reclaimTargets": []Object{}} } else { - require(r.OperationID == "", 400, "invalid_residual_scope") + require(r.OperationID == "" && !r.AllowMissingDeleteOperation, 400, "invalid_residual_scope") require(row.S("reclaimState") == "reconciling", 409, "residual_mode_conflict") require(row["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") } @@ -120,8 +140,8 @@ func registerControlledTermination(t *transaction, r *ResidualManagementRequest, requireNoForceStop(t, w.S("id")) id, reclaim := newID(), newID() c := runtimeControl(t, w.S("id")) - t.exec(`INSERT INTO agent_run_reclamation_intents(id,command_kind,source,tenant_id,project_id,run_id,workspace_id,runtime_generation,reclaim_id,legacy_delete_operation_id,control_epoch,actor_subject,operator_label,reason,impact_confirmed) - VALUES($1,'terminate',$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,true)`, id, source, r.TenantID, p.S("id"), row.S("runId"), w.S("id"), row.N("runtimeGeneration"), reclaim, nullable(r.OperationID), c.N("controlEpoch"), actor, r.OperatorLabel, r.Reason) + t.exec(`INSERT INTO agent_run_reclamation_intents(id,command_kind,source,tenant_id,project_id,run_id,workspace_id,runtime_generation,reclaim_id,legacy_delete_operation_id,control_epoch,actor_subject,operator_label,reason,impact_confirmed,legacy_delete_operation_missing) + VALUES($1,'terminate',$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,true,$14)`, id, source, r.TenantID, p.S("id"), row.S("runId"), w.S("id"), row.N("runtimeGeneration"), reclaim, nullable(r.OperationID), c.N("controlEpoch"), actor, r.OperatorLabel, r.Reason, r.AllowMissingDeleteOperation) if source == "residual_ledger" { claimed := claimResidualReclamation(t, row.S("runId"), "controlled", Object{"kind": "controlled_intent", "intentId": id, "source": "cloudctl"}) require(claimed.B("claimed"), 409, "resource_in_use") @@ -134,7 +154,7 @@ func registerControlledTermination(t *transaction, r *ResidualManagementRequest, t.exec("UPDATE agent_run_reclamation_intents SET control_epoch=(SELECT control_epoch FROM runtime_controls WHERE workspace_id=$2) WHERE id=$1", id, w.S("id")) root := t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", id) refreshControlledTargets(t, root) - controlledAudit(t, id, "registered", actor, Object{"impactConfirmed": true, "operatorLabel": r.OperatorLabel, "source": source}) + controlledAudit(t, id, "registered", actor, Object{"impactConfirmed": true, "operatorLabel": r.OperatorLabel, "source": source, "legacyDeleteOperationMissing": r.AllowMissingDeleteOperation}) return t.one("SELECT * FROM agent_run_reclamation_intents WHERE id=$1", id) } diff --git a/internal/core/agent_run_controlled_reclaim_delete.go b/internal/core/agent_run_controlled_reclaim_delete.go index c9b771c1..4c53b5f7 100644 --- a/internal/core/agent_run_controlled_reclaim_delete.go +++ b/internal/core/agent_run_controlled_reclaim_delete.go @@ -16,8 +16,8 @@ func registerControlledDataDelete(t *transaction, r *ResidualManagementRequest, require(p.S("lifecycle") == "active" && w["deletedAt"] == nil && !w.B("admissionOpen"), 409, "resource_unavailable") require(t.one("SELECT id FROM operations WHERE project_id=$1 AND state IN ('queued','running','retry_wait','blocked')", p.S("id")) == nil, 409, "resource_in_use") id := newID() - t.exec(`INSERT INTO agent_run_reclamation_intents(id,root_intent_id,command_kind,source,tenant_id,project_id,run_id,workspace_id,runtime_generation,legacy_delete_operation_id,control_epoch,actor_subject,operator_label,reason,data_delete_confirmed) - VALUES($1,$2,'delete_data',$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,true)`, id, root.S("id"), root.S("source"), root.S("tenantId"), root.S("projectId"), root.S("runId"), root.S("workspaceId"), root.N("runtimeGeneration"), nullable(root.S("legacyDeleteOperationId")), root.N("controlEpoch"), actor, r.OperatorLabel, r.Reason) + t.exec(`INSERT INTO agent_run_reclamation_intents(id,root_intent_id,command_kind,source,tenant_id,project_id,run_id,workspace_id,runtime_generation,legacy_delete_operation_id,control_epoch,actor_subject,operator_label,reason,data_delete_confirmed,legacy_delete_operation_missing) + VALUES($1,$2,'delete_data',$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,true,$14)`, id, root.S("id"), root.S("source"), root.S("tenantId"), root.S("projectId"), root.S("runId"), root.S("workspaceId"), root.N("runtimeGeneration"), nullable(root.S("legacyDeleteOperationId")), root.N("controlEpoch"), actor, r.OperatorLabel, r.Reason, root.B("legacyDeleteOperationMissing")) snapshot := Object{} for key, value := range w { if key != "issueRunId" { diff --git a/internal/core/agent_run_controlled_reclaim_query.go b/internal/core/agent_run_controlled_reclaim_query.go index c3025ef5..3e26a76f 100644 --- a/internal/core/agent_run_controlled_reclaim_query.go +++ b/internal/core/agent_run_controlled_reclaim_query.go @@ -41,17 +41,21 @@ func listManagedResiduals(t *transaction, r *ResidualManagementRequest) Object { items := t.list(`WITH resources AS ( SELECT ar.tenant_id,ar.project_id,ar.run_id,ar.workspace_id,ar.reclaim_id,ar.reclaim_state AS state, ar.disposition,ar.reclaim_operation_id AS operation_id,NULL::uuid AS legacy_operation_id,'residual_ledger'::text AS source, - ar.reclaim_conflicted_at AS conflicted_at,ar.reclaim_checked_at AS checked_at,ar.stop_evidence + ar.reclaim_conflicted_at AS conflicted_at,ar.reclaim_checked_at AS checked_at,ar.stop_evidence,false AS legacy_delete_operation_missing FROM agent_run_residuals ar UNION ALL SELECT w.tenant_id,w.project_id,i.id,w.id,ri.reclaim_id,COALESCE(ri.state,'reconciling'),COALESCE(child.disposition,ri.disposition), - ri.delete_operation_id,o.id,'legacy_d8',ri.conflicted_at,ri.last_checked_at,'{}'::jsonb + ri.delete_operation_id,o.id,'legacy_d8',ri.conflicted_at,ri.last_checked_at,'{}'::jsonb, + COALESCE(ri.legacy_delete_operation_missing,o.id IS NULL) FROM workspaces w JOIN issue_runs i ON i.id=w.issue_run_id - JOIN LATERAL (SELECT op.id FROM operations op WHERE op.workspace_id=w.id AND op.kind='delete_workspace' + LEFT JOIN LATERAL (SELECT op.id FROM operations op WHERE op.workspace_id=w.id AND op.kind='delete_workspace' AND op.state='failed' AND op.error_code='node_unavailable' ORDER BY op.created_at,op.id LIMIT 1) o ON true LEFT JOIN agent_run_reclamation_intents ri ON ri.workspace_id=w.id AND ri.root_intent_id IS NULL LEFT JOIN agent_run_reclamation_intents child ON child.root_intent_id=ri.id AND child.state='succeeded' WHERE i.phase='done' AND i.executor_type='agent' AND w.kind='isolated' + AND (o.id IS NOT NULL OR ri.legacy_delete_operation_missing OR + (i.failure_reason='workspace_unavailable' AND w.deleted_at IS NULL AND NOT EXISTS( + SELECT 1 FROM operations previous WHERE previous.workspace_id=w.id AND previous.kind='delete_workspace'))) AND NOT EXISTS(SELECT 1 FROM agent_run_residuals ar WHERE ar.workspace_id=w.id) ) SELECT r.*,ri.id AS intent_id FROM resources r LEFT JOIN agent_run_reclamation_intents ri ON ri.workspace_id=r.workspace_id AND ri.root_intent_id IS NULL diff --git a/internal/core/migrations/0070_d8_missing_delete_operation.sql b/internal/core/migrations/0070_d8_missing_delete_operation.sql new file mode 100644 index 00000000..060b123c --- /dev/null +++ b/internal/core/migrations/0070_d8_missing_delete_operation.sql @@ -0,0 +1,12 @@ +-- Keep an absent D8 delete identity explicit instead of fabricating an operation or ledger. +ALTER TABLE agent_run_reclamation_intents + ADD COLUMN legacy_delete_operation_missing boolean NOT NULL DEFAULT false; +ALTER TABLE agent_run_reclamation_intents + DROP CONSTRAINT agent_run_reclamation_intents_check1; +ALTER TABLE agent_run_reclamation_intents + ADD CONSTRAINT agent_run_reclamation_legacy_delete_identity CHECK ( + (source='legacy_d8' AND ( + (legacy_delete_operation_id IS NOT NULL AND NOT legacy_delete_operation_missing) + OR (legacy_delete_operation_id IS NULL AND legacy_delete_operation_missing))) + OR (source='residual_ledger' AND legacy_delete_operation_id IS NULL AND NOT legacy_delete_operation_missing) + ); diff --git a/internal/core/migrations/README.en.md b/internal/core/migrations/README.en.md index 8ba6fda4..64242afd 100644 --- a/internal/core/migrations/README.en.md +++ b/internal/core/migrations/README.en.md @@ -6,7 +6,7 @@ SQL is embedded in the binary through `embed.FS` in `core/store.go`. `Store.Migr ## File index -There are currently 71 SQL files. +There are currently 72 SQL files. | SQL file | Schema change | |---|---| @@ -81,11 +81,14 @@ There are currently 71 SQL files. | [0067_scheduled_empty_artifacts.sql](0067_scheduled_empty_artifacts.sql) | Accept zero-length artifacts in verified registration | | [0068_scheduled_manifest_completeness.sql](0068_scheduled_manifest_completeness.sql) | Persist omitted rejection counts and walk truncation facts | | [0069_workflow_node_attempt.sql](0069_workflow_node_attempt.sql) | Per-attempt identity for workflow node work items, executions and end requests | +| [0070_d8_missing_delete_operation.sql](0070_d8_missing_delete_operation.sql) | Explicit D8 missing-delete provenance and mutually exclusive identity constraints | 0050 copies no personal connection or key and backfills no historical run. Clearing configuration retains its revision; run options are stored separately from shared run input. ## Immutability and upgrades +0070 adds the default-off `legacy_delete_operation_missing` marker to controlled reclamation intents. Existing D8 intents with original deletion identities and residual-ledger intents retain their values. Only an explicitly selected missing-delete variant records the absence; no Run history is backfilled or changed. Fresh schemas, preceding-schema upgrades, repeated migration, and rejection of contradictory provenance are covered by `TestControlledD8MissingDeleteMigrationUpgradesAndRepeats`. + - Never edit or rename applied SQL; corrections use new forward migrations. Repeated application skips applied files with matching checksums. - `CheckSchema` only reads and rejects missing migrations, checksum mismatches and unknown database versions. Production services do not run DDL; operators migrate through [cloudctl](../../../cmd/cloudctl/README.en.md). - Files describe their application-time changes, not the final schema alone. For example, 0017 supersedes 0013's nullable space scope; 0016 retains old storage/worktree records while retiring their steps. diff --git a/internal/core/migrations/README.md b/internal/core/migrations/README.md index 4c0b43fc..dd93c066 100644 --- a/internal/core/migrations/README.md +++ b/internal/core/migrations/README.md @@ -6,7 +6,7 @@ SQL 通过 `core/store.go` 的 `embed.FS` 编入二进制。`Store.Migrate` 在 ## 文件索引 -当前共 71 个 SQL 文件。 +当前共 72 个 SQL 文件。 | SQL 文件 | schema 变化 | |---|---| @@ -81,11 +81,14 @@ SQL 通过 `core/store.go` 的 `embed.FS` 编入二进制。`Store.Migrate` 在 | [0067_scheduled_empty_artifacts.sql](0067_scheduled_empty_artifacts.sql) | 产物大小下界改为零,允许空文件登记 | | [0068_scheduled_manifest_completeness.sql](0068_scheduled_manifest_completeness.sql) | 保存未列明的拒绝数和遍历截断事实 | | [0069_workflow_node_attempt.sql](0069_workflow_node_attempt.sql) | 工作流节点工作项、执行与结束请求的尝试序号 | +| [0070_d8_missing_delete_operation.sql](0070_d8_missing_delete_operation.sql) | D8 无原删除操作的显式来源标记与互斥约束 | 0050 不复制个人连接或 Key,不回填历史运行;清除配置保留 revision,运行选项独立于共享运行输入保存。 ## 不可变性与升级 +0070 给受控回收意图增加默认关闭的 `legacy_delete_operation_missing`。有原删除操作的 D8 和残留账本意图保持原值;仅明确选择无删除操作变体时才可记录缺失,不回填或修改 Run 历史。空库、前一版升级、重复迁移和矛盾来源拒绝由 `TestControlledD8MissingDeleteMigrationUpgradesAndRepeats` 覆盖。 + - 已应用 SQL 不改内容、不改文件名;更正用新的前向迁移。重复应用跳过 checksum 相同的已应用文件。 - `CheckSchema` 只读检查缺失迁移、checksum 不符和未知数据库版本,发现不一致拒绝启动。生产服务不执行 DDL,运维用 [cloudctl](../../../cmd/cloudctl/README.md) 应用迁移。 - 文件描述各自应用时的变化,不能单独当成最终 schema。例如 0013 的可空空间归属被 0017 取代;0016 保留旧存储/工作树记录但停用旧步骤。