diff --git a/cmd/server/README.en.md b/cmd/server/README.en.md index 531f6513..57fa4494 100644 --- a/cmd/server/README.en.md +++ b/cmd/server/README.en.md @@ -16,6 +16,8 @@ The production entrypoint composes configuration, databases, HTTP/gRPC adapters `main.go` owns loops with one cancellation context and WaitGroup: optional catalog sync and workflow asset imports, Agent dispatch/startup, blocked windows, Thread end/cancellation, delivery/release recovery and Workflow scheduling. Core methods still decide state; loops do not make process-local state authoritative. +The residual-ledger recovery loop calls `ReconcileAgentRunResidualsOnce` immediately at startup and then at `issue_runs.residual_reconcile_interval` (default `1m`), sharing that cancellation context and WaitGroup. It retains recovery responsibility after the business run ends and consumes existing fact entrypoints without adding a Controller discovery RPC or contract. Missing evidence leaves the ledger open; timeouts, reconnection and forced stops do not establish successful cleanup. + Cleanup differs between shutdown and failure paths: - Signal-exit branch (SIGINT/SIGTERM): cancel background tasks and send the Controller drain signal; stop gRPC gracefully, forcing it to stop at the shared 10-second shutdown deadline, then call HTTP Shutdown with that same deadline, join background loops, close the database and flush logs. diff --git a/cmd/server/README.md b/cmd/server/README.md index a91511f7..ff74c691 100644 --- a/cmd/server/README.md +++ b/cmd/server/README.md @@ -16,6 +16,8 @@ `main.go` 用同一个取消 context 与 WaitGroup 拥有循环:可选市场同步和工作流资产导入、Agent 派发/启动、blocked 窗口、Thread 结束/取消、交付/释放恢复,以及 Workflow 调度。具体状态判定仍由各 Core 方法负责;循环不把进程内状态当作权威。 +残留账本恢复循环调用 `ReconcileAgentRunResidualsOnce`,启动即运行一次,之后按 `issue_runs.residual_reconcile_interval`(缺省 `1m`)重试,并参与同一 context 的取消与 WaitGroup 等待。它在业务运行结束后继续承担恢复责任,只消费既有事实入口,不新增 Controller 发现 RPC 或契约;证据收不到就保留 open,不按超时、重连或强停推定清理成功。 + 停机与失败路径的清理范围不同: - 信号退出分支(SIGINT/SIGTERM):取消后台任务,发送 Controller drain 信号;gRPC 先优雅停止,到达与 HTTP 共用的 10 秒停机 deadline 时强制停止,然后调用 HTTP Shutdown,等待后台循环结束,再关闭数据库和刷新日志。 diff --git a/cmd/server/main.go b/cmd/server/main.go index b3c41a5f..a6f22746 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -263,6 +263,15 @@ func run() (runErr error) { pluginmarket.RunSyncLoop(ctx, store.GiveUpStaleWorkspaceReleasesOnce, agentDeliveryInterval, pluginmarket.ContextSleep, log) }() } + // Residual recovery remains independent of a run's business terminal state. Cloud consumes + // existing, identity-bound facts; a missing fact leaves its ledger open rather than treating + // a timer or reconnection as cleanup proof. This loop shares cancellation and shutdown joining + // with the delivery/release passes above and has no disable switch. + syncGroup.Add(1) + go func() { + defer syncGroup.Done() + pluginmarket.RunSyncLoop(ctx, store.ReconcileAgentRunResidualsOnce, cfg.IssueRuns.ResidualReconcileInterval, pluginmarket.ContextSleep, log) + }() // Workflow advance loop: the Cloud-side scheduler (specs#99 D1 条件②/④, landing order step 3). // A bounded rescan of pending and running workflow runs, each advanced in its own short // transaction; the ready set is recomputed from committed state only, so the pass is idempotent diff --git a/config.toml.template b/config.toml.template index 49cec212..8ef9759c 100644 --- a/config.toml.template +++ b/config.toml.template @@ -8,6 +8,11 @@ # `task setup` (cmd/devsetup) only turns the values below into what those services already accept: # CLOUD_*/GATEWAY_* environment overrides and TEST_DATABASE_URL in .local/dev.env, which # Taskfile.yml loads for every task, plus the client secret file the Gateway reads. +# Cloud's residual recovery pass runs every 1m by default. Configure +# issue_runs.residual_reconcile_interval in the Cloud YAML, or set +# CLOUD_ISSUE_RUNS_RESIDUAL_RECONCILE_INTERVAL in the server environment. This TOML setup +# template does not accept an [issue_runs] section. The interval must be positive; it is not +# a cleanup-success deadline and there is no disable switch. [database] # PostgreSQL DSN shared by Cloud, the Gateway, `task migrate` and the test suite. Any PostgreSQL 17 diff --git a/docs/README.en.md b/docs/README.en.md index f6697829..5f6f72e3 100644 --- a/docs/README.en.md +++ b/docs/README.en.md @@ -50,6 +50,7 @@ package live in that directory's `README.md`; the authoritative source for busin - [Agent IssueRuns: an Agent run from start to finish](agent-runs/issue-runs.en.md) - [Threads: the conversation of an Agent run](agent-runs/threads.en.md) - [Bounded startup: waiting limits and settling unknown outcomes](agent-runs/startup-settlement.en.md) +- [Agent Run residual reclamation: Cloud authority and evidence](agent-runs/residual-reclamation.en.md) - [Revisions: delivery, verified registration and resume](agent-runs/revisions.en.md) - [Session reads: session lists, archives and model request statistics](agent-runs/session-reads.en.md) diff --git a/docs/README.md b/docs/README.md index 5d2d4ba6..60d87111 100644 --- a/docs/README.md +++ b/docs/README.md @@ -48,6 +48,7 @@ - [Agent IssueRun:一次 Agent 运行的全过程](agent-runs/issue-runs.md) - [Thread:Agent 运行的对话](agent-runs/threads.md) - [有界启动:等待上限与未知结果的收尾](agent-runs/startup-settlement.md) +- [Agent Run 残留回收:Cloud 责任与证据](agent-runs/residual-reclamation.md) - [Revision:交付、校验登记与续接](agent-runs/revisions.md) - [会话读取:会话列表、归档与模型请求统计](agent-runs/session-reads.md) diff --git a/docs/agent-runs/issue-runs.en.md b/docs/agent-runs/issue-runs.en.md index 72a55f67..535a05f2 100644 --- a/docs/agent-runs/issue-runs.en.md +++ b/docs/agent-runs/issue-runs.en.md @@ -20,6 +20,7 @@ This document is the overview; individual parts are covered in: | Creation (@Agent, the Project gate, the input snapshot) | [Issue collaboration](../collaboration/interactions.en.md#task-mode-agent) | | Thread entries, user turns, idleness and ending | [Threads](threads.en.md) | | Startup waiting limits and settling unknown outcomes | [Bounded startup](startup-settlement.en.md) | +| D4/D5 residual ledger, system cleanup responsibility and evidence boundaries | [Residual reclamation](residual-reclamation.en.md) | | Object storage, upload grants, Revision registration and resume | [Revisions](revisions.en.md) | | Session lists, archives and model request statistics | [Session reads](session-reads.en.md) | @@ -67,6 +68,12 @@ and it accepts no other run or interactive session. ## Background loops +After D4/D5 business settlement, independent ledger reconciliation continues and grants one durable system owned-delete responsibility only +when evidence qualifies. Retries keep the same operation; cleanup does not settle the Run again. The current scope is +[Cloud step 1 of residual reclamation](residual-reclamation.en.md). Controlled operator endpoints, retained-data/external dispositions, +Controller/Sandbox Server physical acceptance, joint acceptance and D8 historical takeover remain incomplete. Neither `done` nor a completed +reconciliation pass proves that resources were reclaimed. + Besides the synchronous transitions taken while evidence is taken over, the Cloud server process runs a set of loops every 10 seconds for the progress and settlement that have no external event to follow: diff --git a/docs/agent-runs/issue-runs.md b/docs/agent-runs/issue-runs.md index 2e96044f..db48a6e2 100644 --- a/docs/agent-runs/issue-runs.md +++ b/docs/agent-runs/issue-runs.md @@ -15,6 +15,7 @@ Cloud 校验并登记之后删除运行 Workspace,释放沙盒和数据。 | 怎么创建(@Agent、Project 关口、输入快照) | [Issue 协作](../collaboration/interactions.md#task-modeagent) | | Thread 条目、用户轮次、空闲与结束 | [Thread](threads.md) | | 启动的等待上限与未知结果收尾 | [有界启动](startup-settlement.md) | +| D4/D5 残留账本、系统清理责任与证据边界 | [残留回收](residual-reclamation.md) | | 对象存储、上传授权、Revision 登记与续接 | [Revision](revisions.md) | | 会话列表、归档与模型请求统计 | [会话读取](session-reads.md) | @@ -49,6 +50,10 @@ Cloud 在接管上一阶段证据的事务里写入。 ## 后台循环 +D4/D5 业务收尾后的残留由独立账本对账继续处理,证据足够才授予一份持久的系统 owned-delete 责任;重试保持同一 operation, +完成回收不重新结算 Run。当前仅覆盖 [残留回收的 Cloud 第 1 步](residual-reclamation.md):受控运维入口、保留数据/外部确认、 +Controller/Sandbox Server 物理验收、跨仓联合验收和 D8 历史接管仍未完成;`done` 或对账已运行均不证明资源已回收。 + 除了接管证据时的同步转换,Cloud server 进程里有一组每 10 秒执行一次的循环,负责那些没有外部事件可跟随的推进和收尾: | 循环 | 作用 | diff --git a/docs/agent-runs/residual-reclamation.en.md b/docs/agent-runs/residual-reclamation.en.md new file mode 100644 index 00000000..155be8d6 --- /dev/null +++ b/docs/agent-runs/residual-reclamation.en.md @@ -0,0 +1,106 @@ +# Agent Run residual reclamation: Cloud authority and evidence + +English | [中文](residual-reclamation.md) + +D4/D5 [bounded startup settlement](startup-settlement.en.md) can finish the business Run while retaining unresolved resources in +`agent_run_residuals`. Cloud continues collecting evidence on that original ledger and only grants system cleanup authority when the +evidence qualifies. Deletion follows the existing owned-delete path. It neither settles the [IssueRun](issue-runs.en.md) again nor reopens its Workspace. + +This page describes the Cloud step 1 implementation boundary. It is not evidence of physical termination by the Controller or Sandbox Server, +deployment acceptance, or joint fault acceptance. The +[residual reclamation ADR](https://github.com/ora-space/specs/blob/main/decisions/cloud/issue-run/20261018-residual-agent-run-reclamation.md) +remains `approved`; this step does not declare the whole ADR implemented. + +## Ledger states and one durable responsibility + +| `reclaim_state` | Meaning | Authority fields | +|---|---|---| +| `reconciling` | Collect original-identity facts; deletion is not yet authorized | ID, mode and operation are NULL; authorization is `{}` | +| `reclaiming` | One durable responsibility has been granted; continue or await evidence | Fixed `reclaim_id`, mode and nonempty authorization; `owned_delete` binds exactly one operation | +| `cleared` | Disposition under the original responsibility is complete | Keep that responsibility, with both `cleared_at` and disposition | + +Step 1 production authorization uses only `owned_delete` and completes only as `released`. `controlled` reserves the shared internal arbitration +point; it is neither a controlled management endpoint nor operator authority. The schema's `terminated_data_retained` and `externally_confirmed` +dispositions are reserved for later controlled paths. + +Automatic responsibility stays bound to the same `delete_workspace` operation. Restarts, lease changes, retries and quiesce refusals do not +create another responsibility or operation, and do not revive a failed historical operation. Ordinary deletion, dispatch and permit paths retain +residual guards; an internal system responsibility is not a public deletion flag. The transaction rechecks the ledger, Run, Workspace, Project and +operation tenant, original bindings and generation; a UUID foreign key alone does not establish that scope. + +The [migration](../../internal/core/migrations/README.en.md) only adds columns. Defaults initialize existing D4/D5 rows as `reconciling`, without +inferring completion, backfilling D8, or changing old `reason`, `responsibility` or business settlement. A legacy row with non-null `cleared_at` +causes an explicit error and rollback pending compatibility handling (A2). + +## Original-identity evidence and deletion eligibility + +`reclaim_targets` is a union of original plans and fact targets: original operations/effects, preallocated sandbox ensure IDs, sandboxes, Nodes, +executions, clones, work and data. Late facts may add original targets; they cannot replace an old target with the current generation or drop an +unknown target because a query did not find it. + +New facts append to `reclaim_evidence` with logical event keys, facts, sources and database receipt times, instead of changing old +`responsibility.lateFacts`. An identical replay reuses its receipt; new evidence, additional targets or authority transitions increment +`reclaim_version`. Evidence intake, authorization, permits and completion use Cloud's existing serialized transaction boundary. Completion must +therefore reread the full target set and latest facts in its own transaction. + +System authorization requires located original identities, closed admission, no conflict, no blocking Project operation or other activity, and +qualifying delivery and execution evidence. Execution evidence includes input closure and an explicit unfinished-execution list for the same +runtime control identity, together with original execution terminals. Per-target termination evidence can also establish physical closure. +A no-target proof applies only when original plans and execution relationships truly do not exist; missing rows, NULL results and elapsed time +do not establish that proof. + +Step 1 accepts delivery eligibility only through a genuinely registered and verified Revision for the original `deliver_revision` execution; +every other actual delivery execution needs its own completed verification. There is currently no unified existing durable source for explicit +delivery abandonment. `failed`, `skipped`, business timeouts, absent object storage and missing results do not imply explicit abandonment (A1). +Entries with insufficient evidence remain open. + +Authorized cleanup follows existing quiesce → terminate → cleanup. A quiesce refusal leaves the original operation in `retry_wait`, retaining +the original authority and guards for evidence collection and retry (A3). It neither restores new execution eligibility, escalates to force-stop, +nor creates another operation. +When the existing Controller requests `blocked` for unconfirmed termination, this system responsibility instead persists claimable `retry_wait`; +actual plans and permits still recheck evidence. + +A failed `sandbox_terminate` / `workspace_data_delete` effect bound to this responsibility is a retryable observation, not a disposal conclusion. +After fresh evidence, the same effect may advance to success while retaining both observations. Historical provisioning/execution failures, +external identity bindings and successful conclusions remain protected against contradictions. +After completion, the existing effect-result intake only preserves original bound facts or audits conflicts; it cannot advance the operation, +change its version or grant execution authority again. + +Every original sandbox/ensure ID requires a corresponding successful termination result containing both `terminated` and `lateEnsureFenced`. +`terminated_at`, a failed ensure, a 404 or elapsed time alone is insufficient. The current safe target can reuse owned delete; an additional +historical ID without both confirmations remains open, without replacing its identity or dispatching another termination effect (A5). +Data-deletion planning, permits and completion recheck target coverage. One transaction confirms successful data deletion, succeeds the operation, +soft-deletes the Workspace and writes `cleared/released`, without changing the Run's phase/status/result or repeating business settlement activity. + +## Existing fact intake and conflicts + +Cloud uses existing `runtime_ack`, creation-fact, execution-result and Revision intake. This step adds no RPC, proto or Controller discovery +mechanism. After checking the original control epoch/version, Node, sandbox and generation, `runtime_ack` records input closure and the full +`unfinishedExecutionIds` list on the ledger. Session terminals and delivery results are accepted under their original execution/work/resource +identity. A done Run or cleared ledger is not a reason to reject a consistent late fact. Output rescue uses existing Revision verification; +it does not open admission or start a new session or execution. + +Consistent replay preserves the original conclusion. A contradictory terminal first appends a conflict audit and sets `reclaim_conflicted_at`; +the audit and block commit before the stable `409 residual_fact_conflict` is returned (A4). Canonical facts, existing `cleared_at` and disposition +remain intact. Further deletion or reuse is blocked; this step provides no conflict-resolution endpoint. + +## Reconciliation, recovery and remaining scope + +`ReconcileAgentRunResidualsOnce` scans `reconciling/reclaiming` ledger rows independently of whether the business Run is still `releasing`. +Bounded passes rotate by `reclaim_checked_at NULLS FIRST, created_at, run_id`; a checked time is scheduling metadata, not completion evidence. +Each entry rereads targets, eligibility and original bindings. An existing responsibility resumes its original operation, subject to the current +Controller lease and effect permits. Query failures, insufficient evidence and temporary refusals neither clear the ledger nor expand authority. + +If a failed create's original effects have no existing active-discovery path, they continue waiting for verifiable original-identity facts (A6). +Cloud's evidence intake and responsibility scheduling do not establish new Controller discovery capability, or production validation of real +containers, late-ensure fencing or physical data deletion. + +| Later scope | Step 1 boundary | +|---|---| +| Step 2: controlled operator endpoints, retained data, external confirmation | Not implemented; internal `controlled` arbitration does not supply authentication, intent or commands | +| Step 3: Controller / Sandbox Server physical contracts and per-target evidence | This page states Cloud evidence requirements, not completed physical acceptance | +| Step 4: cross-repository joint fault and recovery acceptance | Incomplete; single-repository implementation or database tests are not a substitute | +| Taking over D8 historical deletion abandonment and residuals without ledger rows | Not implemented; no ledger backfill, automatic retry or historical business-result rewrite | + +See the [core code map](../../internal/core/README.en.md) for implementation entry points. Test results require their execution artifacts; +this page does not claim migration, concurrency or end-to-end tests have passed. diff --git a/docs/agent-runs/residual-reclamation.md b/docs/agent-runs/residual-reclamation.md new file mode 100644 index 00000000..92271906 --- /dev/null +++ b/docs/agent-runs/residual-reclamation.md @@ -0,0 +1,88 @@ +# Agent Run 残留回收:Cloud 责任与证据 + +[English](residual-reclamation.en.md) | 中文 + +[有界启动](startup-settlement.md)的 D4/D5 收尾可以先结束业务运行,把尚未查明的资源留在 +`agent_run_residuals`。Cloud 的残留回收在原账本上继续取证,证据足够后才取得系统清理责任,沿已有 +owned delete 执行删除。它不重新结算 [IssueRun](issue-runs.md),也不重新开放运行 Workspace。 + +本文描述 Cloud 第 1 步的实现边界,不作为 Controller、Sandbox Server 的物理终止、部署或联合故障验收证据。 +[残留回收 ADR](https://github.com/ora-space/specs/blob/main/decisions/cloud/issue-run/20261018-residual-agent-run-reclamation.md) +仍为 `approved`;本步不是整份 ADR 已实施完成的声明。 + +## 账本状态和单一责任 + +| `reclaim_state` | 含义 | 取得责任的字段 | +|---|---|---| +| `reconciling` | 收集原身份事实,尚未授予删除资格 | ID、mode、operation 为空,authorization 为 `{}` | +| `reclaiming` | 已取得一份持久责任,继续推进或等待证据 | 固定 `reclaim_id`、mode 和非空 authorization;`owned_delete` 必须绑定唯一 operation | +| `cleared` | 原责任下的处置已完成 | 保留原责任,同时有 `cleared_at` 和 disposition | + +第 1 步的生产授权只使用 `owned_delete`,完成处置只写 `released`。`controlled` 只预留共用内部仲裁位置, +不构成受控管理端点或运维权限。数据库接受的 `terminated_data_retained`、`externally_confirmed` 留给后续受控路径。 + +自动责任始终绑定同一个 `delete_workspace` operation。重启、换 lease、重试和 quiesce 拒绝不会产生第二个责任或 operation, +也不会把失败的历史 operation 恢复为运行态。普通删除、派发和许可入口继续检查残留保护;内部系统责任不能被当成公开删除开关。 +账本、Run、Workspace、Project、operation 的租户、原绑定和 generation 在同一事务里重新比对;UUID 外键本身不证明这些范围一致。 + +[迁移](../../internal/core/migrations/README.md)只增列。存量 D4/D5 行通过默认值进入 `reconciling`,不推断完成、不回填 D8, +不修改旧 `reason`、`responsibility` 或业务结算。旧行已有非空 `cleared_at` 时明确报错回滚,等待显式兼容处理(A2)。 + +## 原身份取证与删除资格 + +`reclaim_targets` 是原计划和事实目标的并集,包括原 operation/effect、预先分配的 sandbox ensure ID、sandbox、Node、 +execution、clone、work 和数据目标。迟到事实可以补充原目标,不能将旧目标替换成当前 generation,不能用“查不到”删去未知目标。 + +新事实追加到 `reclaim_evidence`,包含逻辑事件键、事实、来源和数据库接收时间,不再写入旧 `responsibility.lateFacts`。 +相同事实重放复用原回执;新证据、目标补充或状态推进增加 `reclaim_version`。证据接管与授权、许可、完成都使用 Cloud 的既有 +串行事务边界,因此完成必须重新看本事务内的完整目标和最新事实。 + +系统授权要求原身份可定位、admission 已关闭、无冲突、无阻挡中的 Project 操作或其他活动,并满足交付和执行证据条件。 +执行侧需要同一 runtime control 身份的关闭输入确认及明确的未完成执行列表,并检查原执行终态;逐目标的终止证据也可证明其 +物理执行已封闭。仅在原计划和执行关系确实不存在时才能采用无目标证明,缺行、NULL result 或等待时间均不构成这种证明。 + +第 1 步只接受原 `deliver_revision` 执行真实登记、核验完成的 Revision 作为交付资格;其他实际交付执行也必须各自核验完成。 +当前没有可统一引用的既有“明确放弃交付”记录来源。`failed`、`skipped`、业务超时、没有对象存储或无结果都不自动等于明确放弃(A1), +证据不足的条目继续保持未完成。 + +获授权后沿已有 quiesce → terminate → cleanup 推进。quiesce 拒绝使原 operation 留在 `retry_wait`,保留原责任及保护,稍后重新取证再试(A3)。 +它不恢复新执行资格,不升级为强停,也不重新创建 operation。 +既有 Controller 因终止未确认而请求 `blocked` 时,此系统责任也保存为可领取的 `retry_wait`;实际计划和许可仍重新检查证据。 + +该责任绑定的 `sandbox_terminate` / `workspace_data_delete` effect 失败是可重试观察,不是已处置结论。 +重新取证后同一 effect 可推进到成功,失败与成功证据均保留;历史创建/执行失败、外部身份和已成功结论仍受冲突保护。 +责任完成后,既有 effect 结果入口只接管原绑定事实或提交冲突审计,不推进 operation、改变其版本或重新授予执行资格。 + +每个原计划 sandbox/ensure ID 都需要对应成功终止结果同时包含 `terminated` 和 `lateEnsureFenced`。 +仅有 `terminated_at`、ensure 失败、404 或经过一段时间都不够。当前安全目标可以复用已有 owned delete;额外历史 ID 缺少双确认时保持未完成, +不为它改写目标或另发终止 effect(A5)。数据删除 effect 的计划、许可和完成再次检查目标覆盖;同一事务确认数据删除成功、operation 成功、 +Workspace 软删除并写 `cleared/released`,不改 Run 的 phase/status/result 或重复追加业务结算活动。 + +## 既有事实入口与冲突 + +Cloud 使用现有 `runtime_ack`、创建事实、执行结果和 Revision intake;没有新增 RPC、proto 或 Controller 取证机制。 +`runtime_ack` 通过原 control epoch/version、Node、sandbox 和 generation 检查后,将关闭输入和 `unfinishedExecutionIds` 原样接入账本。 +会话终态和交付结果按原 execution/work/资源身份接管;Run 已 done 或账本已 cleared 也不能成为拒收一致迟到事实的理由。 +成果救援沿既有 Revision 核验路径,不开放 admission,不启动新会话或执行。 + +一致重放不改原结论。矛盾终态先追加冲突审计并设置 `reclaim_conflicted_at`,提交这些事实及阻挡后才返回稳定的 +`409 residual_fact_conflict`(A4)。规范事实、已有 `cleared_at` 和 disposition 保留,后续删除或复用受阻;本步没有解除冲突的入口。 + +## 对账、恢复与尚未覆盖的范围 + +`ReconcileAgentRunResidualsOnce` 独立扫描账本的 `reconciling/reclaiming`,不依赖业务 Run 仍处于 `releasing`。 +扫描按 `reclaim_checked_at NULLS FIRST, created_at, run_id` 有界轮转;检查时间只是调度记录,不是完成证据。 +每个条目重新读取目标、资格和原绑定;已有责任恢复原 operation,继续服从当前 Controller lease 和 effect 许可。 +查询失败、证据不足和暂时拒绝均不清账、不扩大授权。 + +failed create 的原 effects 如果没有现成可主动发现的路径,就继续等待可验证的原身份事实(A6)。Cloud 接收证据和调度责任不证明 +Controller 已新增发现能力,更不证明真实容器、迟到 ensure 围栏或物理数据删除已经通过生产验证。 + +| 后续范围 | 本步边界 | +|---|---| +| 第 2 步:受控运维入口、保留数据、外部确认 | 未实现;内部 `controlled` 仲裁预留不替代认证、意图或命令 | +| 第 3 步:Controller / Sandbox Server 物理契约与逐目标证据 | 本文仅描述 Cloud 的证据要求,不声明物理验收完成 | +| 第 4 步:跨仓联合故障与恢复验收 | 未完成,不以单仓实现或数据库测试代替 | +| D8 历史删除放弃、无账本残留的接管 | 未实现;不补账、不自动重试、不改历史业务结果 | + +核心代码入口见 [core 代码地图](../../internal/core/README.md)。测试结果应以对应执行产物为准,本文不声明迁移、并发或端到端测试已经通过。 diff --git a/integration/agent_run_provisioning_fact_test.go b/integration/agent_run_provisioning_fact_test.go index f67c6098..148c1898 100644 --- a/integration/agent_run_provisioning_fact_test.go +++ b/integration/agent_run_provisioning_fact_test.go @@ -45,6 +45,8 @@ func TestRecordProvisioningFactReceiptsLateFactsOverGRPCWithoutResurrectingRun(t _, e = f.store.Pool.Exec(`UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '1 second' WHERE id=$1`, run) must(t, e) must(t, f.store.GiveUpBlockedWorkspaceCreationsOnce(context.Background())) + var responsibility string + must(t, f.store.Pool.QueryRow(`SELECT responsibility::text FROM agent_run_residuals WHERE run_id=$1`, run).Scan(&responsibility)) var phase, status, threadState sql.NullString must(t, f.store.Pool.QueryRow(`SELECT phase, status, thread_state FROM issue_runs WHERE id=$1`, run).Scan(&phase, &status, &threadState)) if phase.String != "done" || status.String != "failed" || threadState.Valid { @@ -90,9 +92,12 @@ func TestRecordProvisioningFactReceiptsLateFactsOverGRPCWithoutResurrectingRun(t if open := f.scalar(`SELECT count(*) FROM workspaces WHERE id=$1 AND admission_open`, workspace); open != 0 { t.Fatal("late creation success reopened admission") } - if n := f.scalar(`SELECT count(*) FROM jsonb_object_keys((SELECT responsibility->'lateFacts' FROM agent_run_residuals WHERE run_id=$1))`, run); n != 2 { + if n := f.scalar(`SELECT jsonb_array_length(reclaim_evidence) FROM agent_run_residuals WHERE run_id=$1`, run); n != 2 { t.Fatalf("ledger receipts wrong: %d", n) } + if n := f.scalar(`SELECT count(*) FROM agent_run_residuals WHERE run_id=$1 AND responsibility=$2::jsonb`, run, responsibility); n != 1 { + t.Fatal("late facts changed the original residual responsibility") + } } // The gRPC contract carries a fact's target in dedicated fields, so the submission fingerprint @@ -122,6 +127,8 @@ func TestRecordProvisioningFactBindsTargetAndProgressesOverGRPC(t *testing.T) { _, e = f.store.Pool.Exec(`UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '1 second' WHERE id=$1`, run) must(t, e) must(t, f.store.GiveUpBlockedWorkspaceCreationsOnce(context.Background())) + var responsibility string + must(t, f.store.Pool.QueryRow(`SELECT responsibility::text FROM agent_run_residuals WHERE run_id=$1`, run).Scan(&responsibility)) // A recorded creation fact... _, e = c.operations.RecordProvisioningFact(c.ctx, &controlpb.RecordProvisioningFactRequest{SubmissionId: "bound-creation", Epoch: c.epoch, OperationId: operation, Kind: controlpb.ProvisioningFactKind_PROVISIONING_FACT_KIND_CREATION, Outcome: controlpb.ProvisioningOutcome_PROVISIONING_OUTCOME_READY}) @@ -147,14 +154,17 @@ func TestRecordProvisioningFactBindsTargetAndProgressesOverGRPC(t *testing.T) { if terminal.GetEffect().GetState() != controlpb.EffectState_EFFECT_STATE_SUCCEEDED { t.Fatalf("terminal after running not receipted: %s", terminal.GetEffect().GetState()) } - // The ledger holds the terminal on top of the retained observation; nothing reopened. - var state string - var prior int - must(t, f.store.Pool.QueryRow(`SELECT responsibility->'lateFacts'->('effect:'||$2)->>'state', - jsonb_array_length(responsibility->'lateFacts'->('effect:'||$2)->'prior') - FROM agent_run_residuals WHERE run_id=$1`, run, effectID).Scan(&state, &prior)) - if state != "succeeded" || prior != 1 { - t.Fatalf("ledger receipt wrong: state=%s prior=%d", state, prior) + // Provisional and terminal observations remain independent append-only evidence entries. + var states string + var entries int + must(t, f.store.Pool.QueryRow(`SELECT string_agg(e->'fact'->>'state', ',' ORDER BY ordinal), count(*) + FROM agent_run_residuals r, jsonb_array_elements(r.reclaim_evidence) WITH ORDINALITY AS facts(e,ordinal) + WHERE r.run_id=$1 AND e->>'key'='effect:'||$2`, run, effectID).Scan(&states, &entries)) + if states != "running,succeeded" || entries != 2 { + t.Fatalf("ledger observations wrong: states=%s entries=%d", states, entries) + } + if n := f.scalar(`SELECT count(*) FROM agent_run_residuals WHERE run_id=$1 AND responsibility=$2::jsonb`, run, responsibility); n != 1 { + t.Fatal("late observations changed the original residual responsibility") } if n := f.scalar(`SELECT count(*) FROM workspaces WHERE id=$1 AND admission_open`, workspace); n != 0 { t.Fatal("late terminal reopened admission") diff --git a/integration/agent_run_residual_clone_receipt_test.go b/integration/agent_run_residual_clone_receipt_test.go new file mode 100644 index 00000000..a4154018 --- /dev/null +++ b/integration/agent_run_residual_clone_receipt_test.go @@ -0,0 +1,261 @@ +package integration + +import ( + "context" + "encoding/base64" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + "github.com/wanglongan587/cloud/internal/controlpb" +) + +// Each state is a staged ledger precondition, not a claim that recovery reached it. The +// original result and receipt enter through the same real takeover RPC as the contradiction. +func TestResidualCloneReceiptConflictAcrossReclaimStatesOverGRPC(t *testing.T) { + for _, state := range []string{"reconciling", "reclaiming", "cleared"} { + t.Run(state, func(t *testing.T) { + f, c, run, workspace, original := cloneReceiptResidualScene(t) + + // A running disposal operation gives PlanEffect a valid operation/version. Its + // rejection must come from the newly committed conflict, not stale-operation fencing. + disposal := uuid.NewString() + _, err := f.store.Pool.Exec(`INSERT INTO operations(id,tenant_id,actor_user_id,project_id,workspace_id, + kind,state,step,request,idempotency_key,request_hash,controller_epoch) + SELECT $2::uuid,w.tenant_id,w.creator_user_id,w.project_id,w.id,'delete_workspace','running','terminate', + '{}','clone-receipt-disposal','clone-receipt-fixture',$3 FROM workspaces w WHERE w.id=$1`, workspace, disposal, c.epoch) + must(t, err) + if state != "reconciling" { + _, err = f.store.Pool.Exec(`UPDATE agent_run_residuals SET reclaim_state=$2,reclaim_id=$3, + reclaim_mode='controlled',reclaim_operation_id=$3,reclaim_authorization='{"source":"test-prestate"}', + cleared_at=CASE WHEN $2='cleared' THEN now() END, + disposition=CASE WHEN $2='cleared' THEN 'externally_confirmed' END WHERE run_id=$1`, run, state, disposal) + must(t, err) + } + before := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()) + var responsibility string + must(t, f.store.Pool.QueryRow(`SELECT responsibility::text FROM agent_run_residuals WHERE run_id=$1`, run).Scan(&responsibility)) + conflict := proto.Clone(original).(*controlpb.TakeOverNodeEventRequest) + conflict.SubmissionId, conflict.Event = c.sub(), []byte("clone event B") + response, err := f.executions.TakeOverNodeEvent(c.ctx, conflict) + expectStatus(t, err, codes.Aborted, controlpb.ErrorCode_ERROR_CODE_CONFLICT) + if response != nil || status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("receipt-only conflict was not durably rejected: response=%v error=%v", response, err) + } + + var retained bool + must(t, f.store.Pool.QueryRow(`SELECT r.reclaim_conflicted_at IS NOT NULL + AND r.reclaim_state=$2 AND (r.cleared_at IS NOT NULL)=($2='cleared') AND r.responsibility=$3::jsonb + AND (SELECT count(*) FROM jsonb_array_elements(r.reclaim_evidence) x + WHERE x->>'key'=$4 AND x->'fact'->>'kind'='conflict' + AND x->'fact'->'original'->>'kind'='receipt' AND x->'fact'->'original'->>'event'=$5 + AND x->'fact'->'incoming'->>'kind'='receipt' AND x->'fact'->'incoming'->>'event'=$6 + AND x->>'receivedAt'<>'' AND x->'source'->>'submissionId'=$7)=1 + AND EXISTS(SELECT 1 FROM control_submissions WHERE submission_id=$7 AND response->>'residualFactConflict'='true') + FROM agent_run_residuals r WHERE r.run_id=$1`, run, state, responsibility, + "clone_receipt:"+original.GetExecutionId()+":1", base64.StdEncoding.EncodeToString(original.GetEvent()), + base64.StdEncoding.EncodeToString(conflict.GetEvent()), conflict.GetSubmissionId()).Scan(&retained)) + if !retained { + t.Fatal("conflict error rolled back receipt evidence, replay response, or original ledger state") + } + if after := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()); after != before { + t.Fatalf("receipt conflict changed original clone result or event receipt:\nbefore=%s\nafter=%s", before, after) + } + + var evidence string + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object('evidence',reclaim_evidence, + 'version',reclaim_version,'conflictedAt',reclaim_conflicted_at)::text FROM agent_run_residuals WHERE run_id=$1`, run).Scan(&evidence)) + for _, submission := range []string{conflict.GetSubmissionId(), c.sub()} { + conflict.SubmissionId = submission + response, err = f.executions.TakeOverNodeEvent(c.ctx, conflict) + expectStatus(t, err, codes.Aborted, controlpb.ErrorCode_ERROR_CODE_CONFLICT) + if response != nil || status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("conflicting receipt replay lost its stable result: response=%v error=%v", response, err) + } + } + var replayStable bool + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object('evidence',reclaim_evidence, + 'version',reclaim_version,'conflictedAt',reclaim_conflicted_at)=$2::jsonb FROM agent_run_residuals WHERE run_id=$1`, run, evidence).Scan(&replayStable)) + if !replayStable { + t.Fatal("receipt conflict replay duplicated evidence or moved conflict/version metadata") + } + + var disposalVersion int64 + var disposalBefore string + must(t, f.store.Pool.QueryRow(`SELECT version,jsonb_build_object('state',state,'step',step, + 'version',version,'updatedAt',updated_at,'effects',(SELECT count(*) FROM external_effects WHERE operation_id=$1))::text + FROM operations WHERE id=$1`, disposal).Scan(&disposalVersion, &disposalBefore)) + planned, err := c.operations.PlanEffect(c.ctx, &controlpb.PlanEffectRequest{ + SubmissionId: c.sub(), Epoch: c.epoch, OperationId: disposal, Version: disposalVersion, + Kind: controlpb.EffectKind_EFFECT_KIND_SANDBOX_TERMINATE, WorkspaceId: workspace, + }) + expectStatus(t, err, codes.Aborted, controlpb.ErrorCode_ERROR_CODE_CONFLICT) + if planned != nil || status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("committed receipt conflict did not block disposal: response=%v error=%v", planned, err) + } + var disposalAfter string + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object('state',state,'step',step, + 'version',version,'updatedAt',updated_at,'effects',(SELECT count(*) FROM external_effects WHERE operation_id=$1))::text + FROM operations WHERE id=$1`, disposal).Scan(&disposalAfter)) + if disposalAfter != disposalBefore { + t.Fatalf("blocked disposal changed its operation or planned an effect:\nbefore=%s\nafter=%s", disposalBefore, disposalAfter) + } + }) + } +} + +func TestResidualCloneReceiptConflictDoesNotImportLegacyResultOverGRPC(t *testing.T) { + f, c, run, _, original := cloneReceiptResidualScene(t) + // Model a pre-migration terminal fact: its result exists only in lateFacts, + // with no accepted clone fact in the new evidence ledger. + _, err := f.store.Pool.Exec(`UPDATE agent_run_residuals r SET + responsibility=jsonb_set(r.responsibility,'{lateFacts}',jsonb_build_object( + 'clone:'||e.execution_id,jsonb_build_object('kind','clone','state','failed','result',e.result))), + reclaim_version=7 FROM clone_executions e WHERE r.run_id=$1 AND e.execution_id=$2`, run, original.GetExecutionId()) + must(t, err) + var responsibility string + var version int64 + var emptyEvidence bool + must(t, f.store.Pool.QueryRow(`SELECT responsibility::text,reclaim_version, + reclaim_evidence='[]'::jsonb FROM agent_run_residuals WHERE run_id=$1`, run).Scan(&responsibility, &version, &emptyEvidence)) + if !emptyEvidence { + t.Fatal("legacy fixture already has evidence in the reclaim ledger") + } + before := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()) + conflict := proto.Clone(original).(*controlpb.TakeOverNodeEventRequest) + conflict.SubmissionId, conflict.Event = c.sub(), []byte("clone event B") + response, err := f.executions.TakeOverNodeEvent(c.ctx, conflict) + expectStatus(t, err, codes.Aborted, controlpb.ErrorCode_ERROR_CODE_CONFLICT) + if response != nil || status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("legacy receipt conflict was not durably rejected: response=%v error=%v", response, err) + } + + var retained bool + must(t, f.store.Pool.QueryRow(`SELECT r.responsibility=$2::jsonb + AND r.reclaim_version=$3+1 AND r.reclaim_conflicted_at IS NOT NULL + AND jsonb_array_length(r.reclaim_evidence)=1 + AND r.reclaim_evidence->0->>'key'=$4 + AND r.reclaim_evidence->0->'fact'=jsonb_build_object('kind','conflict','state','conflict', + 'original',jsonb_build_object('kind','receipt','event',$5::text), + 'incoming',jsonb_build_object('kind','receipt','event',$6::text)) + AND r.reclaim_evidence->0->>'receivedAt'<>'' + AND r.reclaim_evidence->0->'source'->>'submissionId'=$7 + AND EXISTS(SELECT 1 FROM control_submissions WHERE submission_id=$7 AND response->>'residualFactConflict'='true') + FROM agent_run_residuals r WHERE r.run_id=$1`, run, responsibility, version, + "clone_receipt:"+original.GetExecutionId()+":1", base64.StdEncoding.EncodeToString(original.GetEvent()), + base64.StdEncoding.EncodeToString(conflict.GetEvent()), conflict.GetSubmissionId()).Scan(&retained)) + if !retained { + t.Fatal("rejected receipt must append exactly one conflict and one version, preserving lateFacts without importing an ordinary clone fact") + } + if after := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()); after != before { + t.Fatalf("legacy receipt conflict changed original clone result or event receipt:\nbefore=%s\nafter=%s", before, after) + } +} + +func TestCloneReceiptConflictWithoutResidualWritesNothingOverGRPC(t *testing.T) { + f, c, _, _, _, original := cloneReceiptScene(t) + before := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()) + var bookkeepingBefore string + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object( + 'residuals',(SELECT jsonb_agg(to_jsonb(r) ORDER BY run_id) FROM agent_run_residuals r), + 'submissions',(SELECT jsonb_agg(jsonb_build_object('id',submission_id,'holder',holder_id, + 'response',response,'createdAt',created_at) ORDER BY submission_id) FROM control_submissions))::text`).Scan(&bookkeepingBefore)) + conflict := proto.Clone(original).(*controlpb.TakeOverNodeEventRequest) + conflict.SubmissionId, conflict.Event = c.sub(), []byte("clone event B") + response, err := f.executions.TakeOverNodeEvent(c.ctx, conflict) + expectStatus(t, err, codes.Aborted, controlpb.ErrorCode_ERROR_CODE_CONFLICT) + if response != nil || status.Convert(err).Message() != "receipt_conflict" { + t.Fatalf("ordinary receipt conflict changed behavior: response=%v error=%v", response, err) + } + if after := cloneReceiptCanonicalSnapshot(t, f, original.GetExecutionId()); after != before { + t.Fatalf("ordinary receipt conflict changed canonical data:\nbefore=%s\nafter=%s", before, after) + } + var bookkeepingAfter string + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object( + 'residuals',(SELECT jsonb_agg(to_jsonb(r) ORDER BY run_id) FROM agent_run_residuals r), + 'submissions',(SELECT jsonb_agg(jsonb_build_object('id',submission_id,'holder',holder_id, + 'response',response,'createdAt',created_at) ORDER BY submission_id) FROM control_submissions))::text`).Scan(&bookkeepingAfter)) + if bookkeepingAfter != bookkeepingBefore { + t.Fatalf("ordinary receipt conflict wrote residual evidence or a control submission:\nbefore=%s\nafter=%s", bookkeepingBefore, bookkeepingAfter) + } +} + +func cloneReceiptResidualScene(t *testing.T) (*fixture, *grpcController, string, string, *controlpb.TakeOverNodeEventRequest) { + t.Helper() + f, c, run, workspace, operation, original := cloneReceiptScene(t) + _, err := c.operations.DeferOperation(c.ctx, &controlpb.DeferOperationRequest{ + SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation.GetId(), Version: operation.GetVersion(), + State: controlpb.DeferState_DEFER_STATE_BLOCKED, Reason: controlpb.DeferReason_DEFER_REASON_NODE_UNAVAILABLE, RetrySeconds: 60, + }) + must(t, err) + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '1 second' WHERE id=$1`, run) + must(t, err) + must(t, f.store.GiveUpBlockedWorkspaceCreationsOnce(context.Background())) + return f, c, run, workspace, original +} + +func cloneReceiptScene(t *testing.T) (*fixture, *grpcController, string, string, *controlpb.Operation, *controlpb.TakeOverNodeEventRequest) { + t.Helper() + f := setup(t) + f.bindBusinessHooks() + created := f.create("clone-receipt-conflict") + f.drain() + run := f.insertAgentRun(t, created.O("resource").S("id")) + provisioned, err := f.store.CreateRunWorkspace(context.Background(), run) + must(t, err) + workspace, operation := provisioned.O("workspace").S("id"), provisioned.O("operation").S("id") + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET phase='provisioning',status='dispatched',workspace_id=$2,version=version+1 WHERE id=$1`, run, workspace) + must(t, err) + c := newGRPCController(t, f) + snapshot := c.claim() + if snapshot.GetOperation().GetId() != operation { + t.Fatalf("claim picked %s, want %s", snapshot.GetOperation().GetId(), operation) + } + op, ensured := c.effect(snapshot.GetOperation(), controlpb.EffectKind_EFFECT_KIND_SANDBOX_ENSURE, workspace) + op, err = c.advance(op) + must(t, err) + nodeID := ensured.GetEvidence().GetSandboxEnsured().GetNodeId() + node, err := c.register(ensured.GetId(), 1, nodeID, "inc-late") + must(t, err) + _, err = c.status(node, controlpb.NodeConnection_NODE_CONNECTION_CONNECTED) + must(t, err) + _, err = c.advance(op) + must(t, err) + snapshot = c.claim() + op = snapshot.GetOperation() + if op.GetStep() != controlpb.OperationStep_OPERATION_STEP_CLONE || snapshot.GetCloneInput() == nil { + t.Fatalf("receipt fixture must dispatch the Workspace's frozen clone step: %v", snapshot) + } + execution := "clone-receipt-execution" + _, err = f.executions.RecordDispatch(c.ctx, &controlpb.RecordDispatchRequest{ + SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, ExecutionId: execution, NodeId: nodeID, + Input: &controlpb.ExecutionInput{Spec: &controlpb.ExecutionInput_Clone{Clone: snapshot.GetCloneInput()}}, + }) + must(t, err) + original := &controlpb.TakeOverNodeEventRequest{ + SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, ExecutionId: execution, Sequence: 1, + Result: &controlpb.ExecutionResult{ + Node: &controlpb.NodeIdentity{NodeId: nodeID, NodeIncarnationId: "inc-late"}, + Outcome: &controlpb.ExecutionResult_CloneFailed{CloneFailed: &controlpb.CloneFailed{Reason: controlpb.CloneFailureReason_CLONE_FAILURE_REASON_SNAPSHOT_UNAVAILABLE}}, + }, + Event: []byte("clone event A"), + } + taken, err := f.executions.TakeOverNodeEvent(c.ctx, original) + must(t, err) + if taken.GetRecord().GetExecutionId() != execution || !proto.Equal(taken.GetRecord().GetResult(), original.GetResult()) { + t.Fatalf("initial clone takeover did not preserve its original result: %v", taken) + } + return f, c, run, workspace, op, original +} + +func cloneReceiptCanonicalSnapshot(t *testing.T, f *fixture, execution string) string { + t.Helper() + var snapshot string + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object('clone',to_jsonb(e), + 'receipts',(SELECT jsonb_agg(to_jsonb(r) ORDER BY sequence) FROM clone_event_receipts r WHERE r.execution_id=e.execution_id))::text + FROM clone_executions e WHERE e.execution_id=$1`, execution).Scan(&snapshot)) + return snapshot +} diff --git a/integration/agent_run_residual_delivery_test.go b/integration/agent_run_residual_delivery_test.go new file mode 100644 index 00000000..31e775e7 --- /dev/null +++ b/integration/agent_run_residual_delivery_test.go @@ -0,0 +1,260 @@ +package integration + +import ( + "strings" + "testing" + + "github.com/google/uuid" + "google.golang.org/grpc/status" + + "github.com/wanglongan587/cloud/internal/controlpb" +) + +// seedResidualTerminalIntake fixes only the test's pre-existing business/ledger state. The original +// execution, frozen work input/target and receipt sequence were registered through production +// dispatch before this fixture runs. In particular this does not claim to execute or prove the +// reclamation completion topology: the cleared branch is a fact-intake precondition only. +func seedResidualTerminalIntake(t *testing.T, f *fixture, scene liveThreadScene, state string) { + t.Helper() + reclaimID, operationID := uuid.NewString(), uuid.NewString() + tx, err := f.store.Pool.Begin() + must(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + _, err = tx.Exec(`UPDATE issue_runs SET phase='done',status='failed',thread_state='ended',idle_since=NULL, + failure_reason='agent_start_failed',completed_at=clock_timestamp(),version=version+1, + result='{"deliveryState":"failed","testBoundary":"preexisting_business_terminal"}'::jsonb WHERE id=$1`, scene.runID) + must(t, err) + _, err = tx.Exec(`UPDATE workspaces SET admission_open=false,admission_epoch=admission_epoch+1, + desired_state='deleted',observed_state=CASE WHEN $2='cleared' THEN 'deleted' ELSE 'deleting' END, + deleted_at=CASE WHEN $2='cleared' THEN clock_timestamp() ELSE NULL END,version=version+1 WHERE issue_run_id=$1`, scene.runID, state) + must(t, err) + _, err = tx.Exec(`INSERT INTO operations(id,tenant_id,actor_user_id,project_id,workspace_id,kind,state,step, + request,idempotency_key,request_hash,controller_epoch) + SELECT $2::uuid,w.tenant_id,w.creator_user_id,w.project_id,w.id,'delete_workspace', + CASE WHEN $4='cleared' THEN 'succeeded' ELSE 'running' END, + CASE WHEN $4='cleared' THEN 'done' ELSE 'quiesce' END, + jsonb_build_object('reclaimId',$3::uuid::text),'terminal-intake-'||$3::uuid::text,'terminal-intake-fixture',1 + FROM workspaces w WHERE w.issue_run_id=$1::uuid`, scene.runID, operationID, reclaimID, state) + must(t, err) + _, err = tx.Exec(`INSERT INTO agent_run_residuals(run_id,tenant_id,project_id,workspace_id,runtime_generation, + responsibility,reason,reclaim_state,reclaim_id,reclaim_mode,reclaim_operation_id,reclaim_authorization, + reclaim_targets,cleared_at,disposition) + SELECT $1::uuid,w.tenant_id,w.project_id,w.id,w.runtime_generation, + jsonb_build_object('executions',(SELECT jsonb_agg(jsonb_build_object('executionId',e.execution_id)) + FROM node_executions e WHERE e.workspace_id=w.id)), + 'agent_start_failed',$2,$3::uuid,'owned_delete',$4::uuid, + jsonb_build_object('kind','preexisting_intake_fixture','runId',$1::uuid::text,'workspaceId',w.id), + (SELECT jsonb_agg(jsonb_build_object('kind','sandbox','id',s.id,'workspaceId',w.id,'generation',s.generation)) + FROM sandbox_instances s WHERE s.workspace_id=w.id), + CASE WHEN $2='cleared' THEN clock_timestamp() ELSE NULL END, + CASE WHEN $2='cleared' THEN 'released' ELSE NULL END + FROM workspaces w WHERE w.issue_run_id=$1::uuid`, scene.runID, state, reclaimID, operationID) + must(t, err) + must(t, tx.Commit()) +} + +// residualIntakeSnapshot deliberately excludes the ledger/effect/result facts being accepted. It +// covers the whole business row and every dispatch counter that a late fact must not resurrect. +func residualIntakeSnapshot(t *testing.T, f *fixture, scene liveThreadScene) string { + t.Helper() + var snapshot string + must(t, f.store.Pool.QueryRow(`SELECT jsonb_build_object( + 'run',to_jsonb(r),'workspace',jsonb_build_object('admission',w.admission_open,'desired',w.desired_state, + 'observed',w.observed_state,'deleted',w.deleted_at,'generation',w.runtime_generation,'version',w.version), + 'operations',(SELECT count(*) FROM operations WHERE workspace_id=w.id), + 'ensures',(SELECT count(*) FROM external_effects WHERE workspace_id=w.id AND kind='sandbox_ensure'), + 'sandboxes',(SELECT count(*) FROM sandbox_instances WHERE workspace_id=w.id), + 'executions',(SELECT count(*) FROM node_executions WHERE workspace_id=w.id), + 'work',(SELECT count(*) FROM execution_work WHERE run_id=r.id), + 'activities',(SELECT count(*) FROM issue_activities WHERE issue_id=r.issue_id))::text + FROM issue_runs r JOIN workspaces w ON w.id=r.workspace_id WHERE r.id=$1`, scene.runID).Scan(&snapshot)) + return snapshot +} + +func TestResidualSessionTerminalTakeoverOverGRPC(t *testing.T) { + for _, state := range []string{"reclaiming", "cleared"} { + t.Run(state, func(t *testing.T) { + f := setup(t) + scene := seedLiveThreadScene(t, f) + scene.start(t, f) + f.runningThread(t, scene) + seedResidualTerminalIntake(t, f, scene, state) + residualTerminalIntakeCase(t, f, scene, scene.executionID, 2, false) + }) + } +} + +func TestResidualVerifiedDeliveryTakeoverOverGRPC(t *testing.T) { + for _, state := range []string{"reclaiming", "cleared"} { + t.Run(state, func(t *testing.T) { + f := setup(t) + scene := deliveringScene(t, f) + execution := f.deliverRevision(t, scene) + seedResidualTerminalIntake(t, f, scene, state) + residualTerminalIntakeCase(t, f, scene, execution, 1, true) + }) + } +} + +func residualTerminalIntakeCase(t *testing.T, f *fixture, scene liveThreadScene, execution string, sequence uint64, delivery bool) { + t.Helper() + before := residualIntakeSnapshot(t, f, scene) + in := f.deliveryInputOf(execution) + result := sessionEndResult(scene, controlpb.AgentSessionEndReason_AGENT_SESSION_END_REASON_USER_ENDED, "") + conflict := sessionEndResult(scene, controlpb.AgentSessionEndReason_AGENT_SESSION_END_REASON_INTERRUPTED, "") + kind := "agent_session" + if delivery { + kind = "deliver_revision" + result = deliveredResult(scene, in, revisionFinalCommit) + conflict = deliveredResult(scene, in, strings.Repeat("2", 40)) + f.push(in, revisionFinalCommit) + } + request := func(submission string, fact *controlpb.ExecutionResult) *controlpb.TakeOverNodeEventRequest { + return &controlpb.TakeOverNodeEventRequest{ + SubmissionId: submission, Epoch: 1, + OperationId: scene.runID, ExecutionId: execution, Sequence: sequence, + Result: fact, Event: []byte("original residual terminal event"), + } + } + ctx := asController("ctrl-a") + + // Naming a real execution is insufficient: its original work target must still resolve to + // the original stable sandbox. An unrelated UUID is rejected before any result or receipt. + var target string + must(t, f.store.Pool.QueryRow("SELECT target::text FROM execution_work WHERE execution_id=$1", execution).Scan(&target)) + _, err := f.store.Pool.Exec(`UPDATE execution_work SET target=jsonb_set(target,'{sandboxInstanceId}',to_jsonb($2::text)) WHERE execution_id=$1`, execution, uuid.NewString()) + must(t, err) + response, err := f.executions.TakeOverNodeEvent(ctx, request("residual-wrong-target", result)) + if err == nil || response != nil { + t.Fatalf("foreign original target was acknowledged: response=%v error=%v", response, err) + } + _, err = f.store.Pool.Exec("UPDATE execution_work SET target=$2::jsonb WHERE execution_id=$1", execution, target) + must(t, err) + if got := f.scalar(`SELECT count(*) FROM node_event_receipts WHERE execution_id=$1 AND sequence=$2`, execution, sequence); got != 0 { + t.Fatalf("invalid target created a receipt: %d", got) + } + if delivery && (len(f.refs.queried()) != 0 || len(f.objects.probed()) != 0) { + t.Fatal("invalid original target reached Git or object verification") + } + + // A successful transport response is the ACK. Force the ledger write to fail, proving the + // canonical result, event receipt and (for delivery) verified Revision cannot commit before it. + _, err = f.store.Pool.Exec(`CREATE FUNCTION reject_residual_intake_evidence() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'injected residual evidence persistence failure'; END $$; + CREATE TRIGGER reject_residual_intake_evidence BEFORE UPDATE OF reclaim_evidence ON agent_run_residuals + FOR EACH ROW EXECUTE FUNCTION reject_residual_intake_evidence()`) + must(t, err) + response, err = f.executions.TakeOverNodeEvent(ctx, request("residual-terminal", result)) + if err == nil || response != nil { + t.Fatalf("failed evidence persistence was acknowledged: response=%v error=%v", response, err) + } + var rolledBack bool + must(t, f.store.Pool.QueryRow(`SELECT e.result IS NULL + AND NOT EXISTS(SELECT 1 FROM node_event_receipts WHERE execution_id=e.execution_id AND sequence=$2) + AND NOT EXISTS(SELECT 1 FROM revision_verifications WHERE execution_id=e.execution_id) + AND NOT EXISTS(SELECT 1 FROM revisions WHERE execution_id=e.execution_id) + AND r.reclaim_evidence='[]'::jsonb + FROM node_executions e JOIN agent_run_residuals r ON r.run_id=e.operation_id + WHERE e.execution_id=$1`, execution, sequence).Scan(&rolledBack)) + if !rolledBack { + t.Fatal("failed evidence persistence left a result, receipt, Revision or evidence") + } + _, err = f.store.Pool.Exec("DROP TRIGGER reject_residual_intake_evidence ON agent_run_residuals; DROP FUNCTION reject_residual_intake_evidence()") + must(t, err) + + response, err = f.executions.TakeOverNodeEvent(ctx, request("residual-terminal", result)) + must(t, err) + if response.GetRecord().GetExecutionId() != execution || response.GetRecord().GetResult() == nil { + t.Fatalf("successful ACK lost its original execution/result: %v", response) + } + var accepted bool + must(t, f.store.Pool.QueryRow(`SELECT e.result->>'outcome'=$3 AND e.last_event_sequence=$2 + AND EXISTS(SELECT 1 FROM node_event_receipts WHERE execution_id=e.execution_id AND sequence=$2) + AND EXISTS(SELECT 1 FROM jsonb_array_elements(r.reclaim_evidence) x + WHERE x->>'key'='execution:'||e.execution_id AND x->'fact'->>'kind'=$4 + AND x->'fact'->>'state'='succeeded' AND x->>'receivedAt' IS NOT NULL) + FROM node_executions e JOIN agent_run_residuals r ON r.run_id=e.operation_id + WHERE e.execution_id=$1`, execution, sequence, map[bool]string{false: "agent_session_ended", true: "revision_delivered"}[delivery], kind).Scan(&accepted)) + if !accepted { + t.Fatal("ACK preceded durable canonical result, matching receipt, or residual evidence") + } + if got := f.receiptEvent(execution, int64(sequence)); got != "original residual terminal event" { + t.Fatalf("original event bytes changed: %q", got) + } + + if delivery { + row := f.revisionRow(scene.runID) + project, repository := f.runPlacement(scene.runID) + if row == nil || row.RunID != scene.runID || row.TenantID != scene.tenantID || row.WorkspaceID != f.runWorkspaceID(scene.runID) || row.ProjectID != project || row.RepositoryURL != repository || row.FinalCommit != revisionFinalCommit || row.BaseCommit != in.S("baseCommit") || row.RevisionRef != in.S("revisionRef") || row.DeliveryRef != in.S("deliveryRef") { + t.Fatalf("late verified delivery registered wrong original Revision: %#v", row) + } + if row.HistoryKey != in.S("historyKey") || row.HistorySize != revisionHistorySize || row.HistorySHA256 != revisionHistorySHA { + t.Fatalf("late verified delivery lost its declared history: %#v", row) + } + if row.BundleKey != "" || row.BundleSize != nil || row.BundleSHA256 != "" { + t.Fatalf("late Git delivery registered a retired bundle: %#v", row) + } + if verdict := f.revisionVerification(execution); verdict.S("outcome") != "delivered" { + t.Fatalf("actual Git and history verification missing: %v", verdict) + } + // Both the failed transaction's attempt and its successful retry queried the attempt's own + // delivery ref and verified its history via the real-HTTP HEAD fixture. The pinned Git ref + // stands for an accepted Node push; no pre-seeded Cloud verification verdict is used. + refs := f.refs.queried() + if len(refs) != 2 || refs[0] != in.S("deliveryRef") || refs[1] != in.S("deliveryRef") { + t.Fatalf("wrong actual Git verification requests: %v", refs) + } + probes := f.objects.probed() + if len(probes) != 2 { + t.Fatalf("wrong actual storage verification requests: %v", probes) + } + for _, probe := range probes { + if probe.S("key") != in.S("historyKey") || probe.N("size") != revisionHistorySize || probe.S("sha256") != revisionHistorySHA { + t.Fatalf("wrong actual history verification request: %v", probe) + } + } + if got := f.objects.nonHeadRequests(); got != 0 { + t.Fatalf("late verification sent %d non-HEAD object-store requests", got) + } + } + + var evidenceVersion int64 + var canonical string + must(t, f.store.Pool.QueryRow(`SELECT r.reclaim_version,e.result::text FROM agent_run_residuals r + JOIN node_executions e ON e.operation_id=r.run_id WHERE e.execution_id=$1`, execution).Scan(&evidenceVersion, &canonical)) + // Replays use the same persisted event identity. A different submission must also be idempotent, + // and successful delivery replay must not contact the Git Server or object storage again. + _, err = f.executions.TakeOverNodeEvent(ctx, request("residual-terminal", result)) + must(t, err) + _, err = f.executions.TakeOverNodeEvent(ctx, request("residual-terminal-replay", result)) + must(t, err) + var replayStable bool + must(t, f.store.Pool.QueryRow(`SELECT reclaim_version=$2 AND + (SELECT count(*) FROM node_event_receipts WHERE execution_id=$1 AND sequence=$3)=1 + FROM agent_run_residuals WHERE run_id=$4`, execution, evidenceVersion, sequence, scene.runID).Scan(&replayStable)) + if !replayStable { + t.Fatal("same original terminal duplicated evidence or receipt") + } + + response, err = f.executions.TakeOverNodeEvent(ctx, request("residual-terminal-conflict", conflict)) + if response != nil || status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("canonical conflict did not return committed residual conflict: response=%v error=%v", response, err) + } + var audited bool + must(t, f.store.Pool.QueryRow(`SELECT r.reclaim_conflicted_at IS NOT NULL AND e.result=$2::jsonb + AND EXISTS(SELECT 1 FROM jsonb_array_elements(r.reclaim_evidence) x + WHERE x->>'key'='execution:'||e.execution_id AND x->'fact'->>'kind'='conflict' + AND x->'fact'->'original'->'result'=$2::jsonb AND x->'fact'->'incoming'->'result'<>$2::jsonb) + AND (SELECT count(*) FROM node_event_receipts WHERE execution_id=e.execution_id AND sequence=$3)=1 + FROM node_executions e JOIN agent_run_residuals r ON r.run_id=e.operation_id + WHERE e.execution_id=$1`, execution, canonical, sequence).Scan(&audited)) + if !audited { + t.Fatal("conflict response rolled back audit or rewrote canonical result/receipt") + } + if delivery && (len(f.refs.queried()) != 2 || len(f.objects.probed()) != 2) { + t.Fatal("known replay or canonical conflict repeated Git or object verification") + } + if after := residualIntakeSnapshot(t, f, scene); after != before { + t.Fatalf("late facts changed business terminal/version, admission, resource identity or dispatch counts:\nbefore=%s\nafter=%s", before, after) + } +} diff --git a/integration/agent_run_residual_facts_test.go b/integration/agent_run_residual_facts_test.go new file mode 100644 index 00000000..b09a1cff --- /dev/null +++ b/integration/agent_run_residual_facts_test.go @@ -0,0 +1,134 @@ +package integration + +import ( + "context" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + "github.com/wanglongan587/cloud/internal/controlpb" +) + +// These scenes stage each ledger prestate to exercise the real fact entrypoint over bufconn and +// PostgreSQL. Staging reclaiming/cleared is not evidence that the recovery loop reaches them. +func TestResidualProvisioningFactsAcrossReclaimStatesOverGRPC(t *testing.T) { + for _, state := range []string{"reconciling", "reclaiming", "cleared"} { + t.Run(state, func(t *testing.T) { + f := setup(t) + f.bindBusinessHooks() + created := f.create("residual-facts-" + state) + f.drain() + run := f.insertAgentRun(t, created.O("resource").S("id")) + provisioned, err := f.store.CreateRunWorkspace(context.Background(), run) + must(t, err) + workspace, operation := provisioned.O("workspace").S("id"), provisioned.O("operation").S("id") + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET phase='provisioning',status='dispatched',workspace_id=$2,version=version+1 WHERE id=$1`, run, workspace) + must(t, err) + c := newGRPCController(t, f) + snapshot := c.claim() + if snapshot.GetOperation().GetId() != operation { + t.Fatalf("claim picked %s, want %s", snapshot.GetOperation().GetId(), operation) + } + planned, err := c.operations.PlanEffect(c.ctx, &controlpb.PlanEffectRequest{SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, Version: snapshot.GetOperation().GetVersion(), Kind: controlpb.EffectKind_EFFECT_KIND_SANDBOX_ENSURE, WorkspaceId: workspace}) + must(t, err) + effect := planned.GetEffect().GetId() + // Older ensure plans may carry no Workspace on the effect itself. Its original + // operation still binds the run Workspace, so intake must use that durable identity. + _, err = f.store.Pool.Exec(`UPDATE external_effects SET workspace_id=NULL WHERE id=$1 AND operation_id=$2`, effect, operation) + must(t, err) + clone, plugin := "late-clone-"+state, "late-plugin-"+state + // Only pre-seed the execution row; the entry under test is the real RPC. + _, err = f.store.Pool.Exec(`INSERT INTO clone_executions(execution_id,operation_id,workspace_id,node_id,input,dispatched_epoch,node_operation_id) + VALUES($1,$2,$3,'node-late','{"kind":"clone"}',1,$1::text)`, clone, operation, workspace) + must(t, err) + _, err = f.store.Pool.Exec(`INSERT INTO node_executions(execution_id,kind,operation_id,workspace_id,node_id,input,dispatched_epoch,node_operation_id) + VALUES($1,'install_plugins',$2,$3,'node-late','{"kind":"install_plugins"}',1,$1::text)`, plugin, operation, workspace) + must(t, err) + _, err = c.operations.DeferOperation(c.ctx, &controlpb.DeferOperationRequest{SubmissionId: c.sub(), Epoch: c.epoch, OperationId: operation, Version: planned.GetOperation().GetVersion(), State: controlpb.DeferState_DEFER_STATE_BLOCKED, Reason: controlpb.DeferReason_DEFER_REASON_NODE_UNAVAILABLE, RetrySeconds: 60}) + must(t, err) + _, err = f.store.Pool.Exec(`UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '1 second' WHERE id=$1`, run) + must(t, err) + must(t, f.store.GiveUpBlockedWorkspaceCreationsOnce(context.Background())) + if state != "reconciling" { + _, err = f.store.Pool.Exec(`UPDATE agent_run_residuals SET reclaim_state=$2,reclaim_id=$3,reclaim_mode='controlled', + reclaim_authorization='{"source":"test-prestate"}',cleared_at=CASE WHEN $2='cleared' THEN now() END, + disposition=CASE WHEN $2='cleared' THEN 'externally_confirmed' END WHERE run_id=$1`, run, state, operation) + must(t, err) + } + var responsibility string + must(t, f.store.Pool.QueryRow(`SELECT responsibility::text FROM agent_run_residuals WHERE run_id=$1`, run).Scan(&responsibility)) + facts := []*controlpb.RecordProvisioningFactRequest{ + { + Epoch: c.epoch, OperationId: operation, Kind: controlpb.ProvisioningFactKind_PROVISIONING_FACT_KIND_EFFECT, EffectId: effect, + State: controlpb.EffectState_EFFECT_STATE_SUCCEEDED, ExternalId: "substrate-late", + Evidence: &controlpb.EffectEvidence{Evidence: &controlpb.EffectEvidence_SandboxEnsured{SandboxEnsured: &controlpb.SandboxEnsured{SandboxInstanceId: effect, NodeId: "node-late"}}}, + }, + { + Epoch: c.epoch, OperationId: operation, Kind: controlpb.ProvisioningFactKind_PROVISIONING_FACT_KIND_CLONE, ExecutionId: clone, + ExecutionResult: &controlpb.ExecutionResult{Node: &controlpb.NodeIdentity{NodeId: "node-late", NodeIncarnationId: "inc-late"}, Outcome: &controlpb.ExecutionResult_CloneFailed{CloneFailed: &controlpb.CloneFailed{Reason: controlpb.CloneFailureReason_CLONE_FAILURE_REASON_SNAPSHOT_UNAVAILABLE}}}, + }, + { + Epoch: c.epoch, OperationId: operation, Kind: controlpb.ProvisioningFactKind_PROVISIONING_FACT_KIND_PLUGIN, ExecutionId: plugin, + ExecutionResult: &controlpb.ExecutionResult{Node: &controlpb.NodeIdentity{NodeId: "node-late", NodeIncarnationId: "inc-late"}, Outcome: &controlpb.ExecutionResult_PluginsFailed{PluginsFailed: &controlpb.PluginsFailed{Reason: controlpb.PluginsFailureReason_PLUGINS_FAILURE_REASON_PLUGIN_ROOT_UNAVAILABLE}}}, + }, + {Epoch: c.epoch, OperationId: operation, Kind: controlpb.ProvisioningFactKind_PROVISIONING_FACT_KIND_CREATION, Outcome: controlpb.ProvisioningOutcome_PROVISIONING_OUTCOME_READY}, + } + for _, fact := range facts { + fact.SubmissionId = c.sub() + _, err = c.operations.RecordProvisioningFact(c.ctx, fact) + must(t, err) + _, err = c.operations.RecordProvisioningFact(c.ctx, fact) + must(t, err) + replay := proto.Clone(fact).(*controlpb.RecordProvisioningFactRequest) + replay.SubmissionId = c.sub() + _, err = c.operations.RecordProvisioningFact(c.ctx, replay) + must(t, err) + } + var canonicalClone, canonicalPlugin string + must(t, f.store.Pool.QueryRow(`SELECT result::text FROM clone_executions WHERE execution_id=$1`, clone).Scan(&canonicalClone)) + must(t, f.store.Pool.QueryRow(`SELECT result::text FROM node_executions WHERE execution_id=$1`, plugin).Scan(&canonicalPlugin)) + conflicts := make([]*controlpb.RecordProvisioningFactRequest, len(facts)) + for i, fact := range facts { + conflicts[i] = proto.Clone(fact).(*controlpb.RecordProvisioningFactRequest) + conflicts[i].SubmissionId = c.sub() + } + failure := "late contradiction" + conflicts[0].State, conflicts[0].Evidence, conflicts[0].Failure = controlpb.EffectState_EFFECT_STATE_FAILED, nil, &failure + conflicts[1].ExecutionResult.Outcome = &controlpb.ExecutionResult_CloneFailed{CloneFailed: &controlpb.CloneFailed{Reason: controlpb.CloneFailureReason_CLONE_FAILURE_REASON_INTERRUPTED}} + conflicts[2].ExecutionResult.Outcome = &controlpb.ExecutionResult_PluginsFailed{PluginsFailed: &controlpb.PluginsFailed{Reason: controlpb.PluginsFailureReason_PLUGINS_FAILURE_REASON_INTERRUPTED}} + conflicts[3].Outcome = controlpb.ProvisioningOutcome_PROVISIONING_OUTCOME_FAILED + for _, conflict := range conflicts { + for _, submission := range []string{conflict.SubmissionId, conflict.SubmissionId, c.sub()} { + conflict.SubmissionId = submission + _, err = c.operations.RecordProvisioningFact(c.ctx, conflict) + expectStatus(t, err, codes.Aborted, controlpb.ErrorCode_ERROR_CODE_CONFLICT) + if status.Convert(err).Message() != "residual_fact_conflict" { + t.Fatalf("conflict did not return its stable residual code: %v", err) + } + } + } + // Read after the rejected RPCs: all conflicts must already be committed, deduplicated, + // and separate from the frozen responsibility and canonical execution results. + var retained bool + must(t, f.store.Pool.QueryRow(`SELECT r.responsibility=$2::jsonb AND r.reclaim_state=$3 + AND (r.cleared_at IS NOT NULL)=($3='cleared') AND r.reclaim_conflicted_at IS NOT NULL + AND jsonb_array_length(r.reclaim_evidence)=8 + AND (SELECT count(*) FROM jsonb_array_elements(r.reclaim_evidence) e WHERE e->'fact'->>'kind'='conflict' + AND e->'fact' ? 'incoming' AND e->'fact' ? 'original' AND e->>'source'<>'' AND e->>'receivedAt'<>'')=4 + AND (SELECT count(DISTINCT e->>'key') FROM jsonb_array_elements(r.reclaim_evidence) e WHERE e->>'key'=ANY($7::text[]))=4 + AND (SELECT result=$5::jsonb FROM clone_executions WHERE execution_id=$4) + AND (SELECT result=$6::jsonb FROM node_executions WHERE execution_id=$8) + AND (SELECT workspace_id IS NULL AND state='succeeded' AND external_id='substrate-late' + AND result=jsonb_build_object('sandboxInstanceId',$9::text,'nodeId','node-late') FROM external_effects WHERE id=$9::uuid) + AND (SELECT phase='done' AND status='failed' FROM issue_runs WHERE id=$1) + AND (SELECT NOT admission_open FROM workspaces WHERE id=$10) + FROM agent_run_residuals r WHERE r.run_id=$1`, run, responsibility, state, clone, canonicalClone, canonicalPlugin, + "{effect:"+effect+",clone:"+clone+",plugin:"+plugin+",creation}", plugin, effect, workspace).Scan(&retained)) + if !retained { + t.Fatal("late fact intake changed original state, lost durable conflict evidence, or duplicated a replay") + } + }) + } +} diff --git a/integration/agent_run_residual_reclaim_migration_test.go b/integration/agent_run_residual_reclaim_migration_test.go new file mode 100644 index 00000000..2d4f8358 --- /dev/null +++ b/integration/agent_run_residual_reclaim_migration_test.go @@ -0,0 +1,222 @@ +package integration + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" +) + +const residualReclaimMigration = "0057_agent_run_residual_reclamation.sql" + +func seedResidualMigrationRow(t *testing.T, pool *sql.DB, seed skeletonSeed, reason string) { + t.Helper() + _, err := pool.Exec(`INSERT INTO agent_run_residuals + (run_id,tenant_id,project_id,workspace_id,runtime_generation,responsibility,reason,session_ended_at) + VALUES($1,$2,$3,$4,27,'{"lateFacts":{"original":{"state":"ended"}},"original":"keep"}',$5,'2026-01-02T03:04:05Z')`, + seed.runID, seed.tenantID, seed.projectID, seed.runWorkspaceID, reason) + must(t, err) +} + +// Compare the complete old ledger and business rows, rather than only the newly added defaults. +func residualMigrationHistory(t *testing.T, pool *sql.DB) string { + t.Helper() + var snapshot string + must(t, pool.QueryRow(`SELECT jsonb_build_object( + 'residuals',(SELECT jsonb_agg(to_jsonb(r)-ARRAY['reclaim_state','disposition','reclaim_id','reclaim_mode', + 'reclaim_operation_id','reclaim_authorization','reclaim_targets','reclaim_evidence','reclaim_version', + 'reclaim_checked_at','reclaim_conflicted_at'] ORDER BY run_id) FROM agent_run_residuals r), + 'runs',(SELECT jsonb_agg(to_jsonb(r) ORDER BY id) FROM issue_runs r), + 'workspaces',(SELECT jsonb_agg(to_jsonb(w) ORDER BY id) FROM workspaces w), + 'operations',(SELECT jsonb_agg(to_jsonb(o) ORDER BY id) FROM operations o))::text`).Scan(&snapshot)) + return snapshot +} + +func TestMigration0057ResidualReclamationFreshAndConstraints(t *testing.T) { + pool, _ := testSchema(t, "test_residual57f_") + must(t, newStoreOnSchema(t, pool).Migrate(context.Background())) + seed := seedAgentIssueRunSkeleton(t, pool) + seedResidualMigrationRow(t, pool, seed, "agent_start_failed") + var columns int + must(t, pool.QueryRow(`SELECT count(*) FROM information_schema.columns + WHERE table_schema=current_schema() AND table_name='agent_run_residuals' + AND column_name IN ('reclaim_state','disposition','reclaim_id','reclaim_mode','reclaim_operation_id', + 'reclaim_authorization','reclaim_targets','reclaim_evidence','reclaim_version','reclaim_checked_at','reclaim_conflicted_at')`).Scan(&columns)) + if columns != 11 { + t.Fatalf("fresh schema has %d reclamation columns, want 11", columns) + } + var scanDefinition, scanPredicate string + must(t, pool.QueryRow(`SELECT pg_get_indexdef(indexrelid),pg_get_expr(indpred,indrelid) + FROM pg_index WHERE indexrelid='agent_run_residuals_reclaim_scan'::regclass`).Scan(&scanDefinition, &scanPredicate)) + if !strings.Contains(scanDefinition, "(reclaim_checked_at NULLS FIRST, created_at, run_id)") || + !strings.Contains(scanPredicate, "reconciling") || !strings.Contains(scanPredicate, "reclaiming") || strings.Contains(scanPredicate, "cleared") { + t.Fatalf("residual scan must order pending rows fairly: %s WHERE %s", scanDefinition, scanPredicate) + } + var initialized bool + must(t, pool.QueryRow(`SELECT reclaim_state='reconciling' AND disposition IS NULL AND reclaim_id IS NULL + AND reclaim_mode IS NULL AND reclaim_operation_id IS NULL AND reclaim_authorization='{}'::jsonb + AND reclaim_targets='[]'::jsonb AND reclaim_evidence='[]'::jsonb AND reclaim_version=0 + AND reclaim_checked_at IS NULL AND reclaim_conflicted_at IS NULL AND cleared_at IS NULL + FROM agent_run_residuals WHERE run_id=$1`, seed.runID).Scan(&initialized)) + if !initialized { + t.Fatal("fresh residual did not receive empty reconciling defaults") + } + op := uuid.NewString() + _, err := pool.Exec(`INSERT INTO operations(id,tenant_id,actor_user_id,project_id,workspace_id,kind,state,step,request,idempotency_key,request_hash) + VALUES($1,$2,$3,$4,$5,'delete_workspace','succeeded','done','{}','residual-migration','fixture-request')`, + op, seed.tenantID, seed.userID, seed.projectID, seed.runWorkspaceID) + must(t, err) + claim := "reclaim_state='reclaiming',reclaim_id='" + uuid.NewString() + "',reclaim_mode='owned_delete',reclaim_authorization='{\"source\":\"test\"}',reclaim_operation_id='" + op + "'" + for _, tc := range []struct { + name, set string + }{ + {"unknown state", "reclaim_state='unknown'"}, + {"null state", "reclaim_state=NULL"}, + {"unknown disposition", "disposition='unknown'"}, + {"unknown mode", "reclaim_mode='unknown'"}, + {"authorization array", "reclaim_authorization='[]'"}, + {"authorization null", "reclaim_authorization=NULL"}, + {"targets object", "reclaim_targets='{}'"}, + {"targets null", "reclaim_targets=NULL"}, + {"evidence object", "reclaim_evidence='{}'"}, + {"evidence null", "reclaim_evidence=NULL"}, + {"negative version", "reclaim_version=-1"}, + {"null version", "reclaim_version=NULL"}, + {"reconciling id", "reclaim_id='" + uuid.NewString() + "'"}, + {"reconciling mode", "reclaim_mode='controlled'"}, + {"reconciling operation", "reclaim_operation_id='" + op + "'"}, + {"reconciling authorization", "reclaim_authorization='{\"source\":\"test\"}'"}, + {"missing id", "reclaim_state='reclaiming',reclaim_mode='controlled',reclaim_authorization='{\"source\":\"test\"}'"}, + {"missing mode", "reclaim_state='reclaiming',reclaim_id='" + uuid.NewString() + "',reclaim_authorization='{\"source\":\"test\"}'"}, + {"empty authorization", "reclaim_state='reclaiming',reclaim_id='" + uuid.NewString() + "',reclaim_mode='controlled'"}, + {"owned delete without operation", "reclaim_state='reclaiming',reclaim_id='" + uuid.NewString() + "',reclaim_mode='owned_delete',reclaim_authorization='{\"source\":\"test\"}'"}, + {"cleared missing timestamp", strings.Replace(claim, "reclaiming", "cleared", 1) + ",disposition='released'"}, + {"cleared missing disposition", strings.Replace(claim, "reclaiming", "cleared", 1) + ",cleared_at=now()"}, + {"reconciling timestamp", "cleared_at=now()"}, + {"reconciling disposition", "disposition='released'"}, + {"reclaiming timestamp", claim + ",cleared_at=now()"}, + {"reclaiming disposition", claim + ",disposition='released'"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := pool.Exec("UPDATE agent_run_residuals SET "+tc.set+" WHERE run_id=$1", seed.runID) + code := "23514" + if strings.Contains(tc.name, "null") { + code = "23502" + } + wantPGError(t, err, code) + }) + } + _, err = pool.Exec("UPDATE agent_run_residuals SET "+strings.Replace(claim, op, uuid.NewString(), 1)+" WHERE run_id=$1", seed.runID) + wantPGError(t, err, "23503") + _, err = pool.Exec("UPDATE agent_run_residuals SET "+claim+" WHERE run_id=$1", seed.runID) + must(t, err) + other := seed + other.runID, other.runWorkspaceID = newBoundRun(t, pool, seed) + seedResidualMigrationRow(t, pool, other, "workspace_unavailable") + _, err = pool.Exec(`UPDATE agent_run_residuals SET reclaim_state='reclaiming',reclaim_mode='controlled', + reclaim_authorization='{"source":"test"}',reclaim_id=(SELECT reclaim_id FROM agent_run_residuals WHERE run_id=$1) + WHERE run_id=$2`, seed.runID, other.runID) + wantPGError(t, err, "23505") + _, err = pool.Exec(`UPDATE agent_run_residuals SET reclaim_state='reclaiming',reclaim_mode='controlled', + reclaim_authorization='{"source":"test"}',reclaim_id=$1,reclaim_operation_id=$2 WHERE run_id=$3`, uuid.NewString(), op, other.runID) + wantPGError(t, err, "23505") + // Controlled responsibility may omit an operation; all three explicit dispositions are legal. + _, err = pool.Exec(`UPDATE agent_run_residuals SET reclaim_state='reclaiming',reclaim_mode='controlled', + reclaim_authorization='{"source":"test"}',reclaim_id=$1 WHERE run_id=$2`, uuid.NewString(), other.runID) + must(t, err) + for _, disposition := range []string{"released", "terminated_data_retained", "externally_confirmed"} { + _, err = pool.Exec(`UPDATE agent_run_residuals SET reclaim_state='cleared',cleared_at=now(),disposition=$1 WHERE run_id=$2`, disposition, other.runID) + must(t, err) + } +} + +func TestMigration0057ResidualReclamationUpgradePreservesHistory(t *testing.T) { + pool, _ := testSchema(t, "test_residual57u_") + // Start after the mainline 0054 schema, including the 0052 settlement and 0053 clone retirement. + applyMigrationsUpTo(t, pool, migrationsBefore(t, residualReclaimMigration)) + seed := seedAgentIssueRunSkeleton(t, pool) + seedResidualMigrationRow(t, pool, seed, "agent_start_failed") + other := seed + other.runID, other.runWorkspaceID = newBoundRun(t, pool, seed) + seedResidualMigrationRow(t, pool, other, "workspace_unavailable") + before := residualMigrationHistory(t, pool) + must(t, newStoreOnSchema(t, pool).Migrate(context.Background())) + if after := residualMigrationHistory(t, pool); before != after { + t.Fatal("reclamation upgrade rewrote legacy residual or business facts") + } + var count int + must(t, pool.QueryRow(`SELECT count(*) FROM agent_run_residuals WHERE reclaim_state='reconciling' + AND reclaim_authorization='{}'::jsonb AND reclaim_targets='[]'::jsonb AND reclaim_evidence='[]'::jsonb + AND reclaim_version=0 AND reclaim_id IS NULL AND reclaim_mode IS NULL AND reclaim_operation_id IS NULL + AND disposition IS NULL AND reclaim_checked_at IS NULL AND reclaim_conflicted_at IS NULL`).Scan(&count)) + if count != 2 { + t.Fatalf("upgrade initialized %d legacy residuals, want both D4 and D5 without invented evidence", count) + } +} + +func TestMigration0057ResidualReclamationReplayPreservesProgress(t *testing.T) { + pool, _ := testSchema(t, "test_residual57r_") + store := newStoreOnSchema(t, pool) + must(t, store.Migrate(context.Background())) + base := seedAgentIssueRunSkeleton(t, pool) + for _, state := range []string{"reconciling", "reclaiming", "cleared"} { + seed := base + if state != "reconciling" { + seed.runID, seed.runWorkspaceID = newBoundRun(t, pool, base) + } + seedResidualMigrationRow(t, pool, seed, "agent_start_failed") + if state != "reconciling" { + _, err := pool.Exec(`UPDATE agent_run_residuals SET reclaim_state=$1,reclaim_id=$2,reclaim_mode='controlled', + reclaim_authorization='{"source":"test"}',reclaim_targets='[{"sandboxId":"original"}]', + reclaim_evidence='[{"eventKey":"original"}]',reclaim_version=7,reclaim_checked_at=now(),reclaim_conflicted_at=now(), + cleared_at=CASE WHEN $1='cleared' THEN now() END,disposition=CASE WHEN $1='cleared' THEN 'released' END + WHERE run_id=$3`, state, uuid.NewString(), seed.runID) + must(t, err) + } + } + var before, after string + query := "SELECT jsonb_agg(to_jsonb(r) ORDER BY run_id)::text FROM agent_run_residuals r" + must(t, pool.QueryRow(query).Scan(&before)) + // Exercise both runner replay and direct reentrant DDL on already progressed rows. + must(t, store.Migrate(context.Background())) + ddl, err := os.ReadFile(filepath.Join("..", "internal", "core", "migrations", residualReclaimMigration)) + must(t, err) + tx, err := pool.Begin() + must(t, err) + defer func() { _ = tx.Rollback() }() + _, err = tx.Exec(string(ddl)) + must(t, err) + must(t, tx.Commit()) + must(t, pool.QueryRow(query).Scan(&after)) + if before != after { + t.Fatal("migration replay reset reclamation progress or evidence") + } +} + +func TestMigration0057ResidualReclamationRejectsLegacyClearedRows(t *testing.T) { + pool, _ := testSchema(t, "test_residual57a2_") + applyMigrationsUpTo(t, pool, migrationsBefore(t, residualReclaimMigration)) + seed := seedAgentIssueRunSkeleton(t, pool) + seedResidualMigrationRow(t, pool, seed, "agent_start_failed") + _, err := pool.Exec("UPDATE agent_run_residuals SET cleared_at='2026-01-03T03:04:05Z' WHERE run_id=$1", seed.runID) + must(t, err) + before := residualMigrationHistory(t, pool) + err = newStoreOnSchema(t, pool).Migrate(context.Background()) + if err == nil || !strings.Contains(err.Error(), "non-null cleared_at") { + t.Fatalf("legacy cleared row must produce the explicit A2 compatibility error, got %v", err) + } + if after := residualMigrationHistory(t, pool); before != after { + t.Fatal("failed A2 migration changed original data") + } + var columns, recorded int + must(t, pool.QueryRow(`SELECT count(*) FROM information_schema.columns + WHERE table_schema=current_schema() AND table_name='agent_run_residuals' AND column_name='reclaim_state'`).Scan(&columns)) + must(t, pool.QueryRow("SELECT count(*) FROM schema_migrations WHERE version=$1", residualReclaimMigration).Scan(&recorded)) + if columns != 0 || recorded != 0 { + t.Fatalf("failed A2 migration left schema/data state: columns=%d recorded=%d", columns, recorded) + } +} diff --git a/internal/config/README.en.md b/internal/config/README.en.md index 7ce46eb2..52c9aa5a 100644 --- a/internal/config/README.en.md +++ b/internal/config/README.en.md @@ -16,4 +16,6 @@ Without an explicit path it searches `./configs`, `../configs` and `.` for `conf Configuration stores deployment references; raw private keys and secrets load from separate files and never enter configuration bodies, logs or databases. This package validates startup configuration; Core owns durable window freezing, database clocks and recovery decisions. Enabled object storage does not establish online health. +`issue_runs.residual_reconcile_interval` sets the cadence of Cloud's residual-ledger recovery pass and defaults to `1m`; `CLOUD_ISSUE_RUNS_RESIDUAL_RECONCILE_INTERVAL` overrides it. Explicit zero, negative and malformed durations reject startup, and there is no disable switch. The interval only schedules scans; it is not a timeout proving successful resource cleanup. Cloud consumes real facts through existing entrypoints without adding a Controller discovery contract; missing evidence leaves the ledger open. + Tests cover defaults, environment overrides, invalid durations, TLS, storage and repository configuration. See [server](../../cmd/server/README.en.md), [Bounded startup](../../docs/agent-runs/startup-settlement.en.md) and [Deployment egress](../../docs/operations/deployment-egress.en.md). diff --git a/internal/config/README.md b/internal/config/README.md index ceefe47b..8b61f71b 100644 --- a/internal/config/README.md +++ b/internal/config/README.md @@ -16,4 +16,6 @@ 配置文件保存部署引用,原始私钥和机密通过独立文件加载,不写进配置正文、日志或数据库。包负责启动期配置有效性;业务窗口的冻结、数据库时钟与恢复判定在 Core。对象存储开启不等于在线健康探测。 +`issue_runs.residual_reconcile_interval` 控制 Cloud 残留账本恢复扫描的间隔,缺省为 `1m`;环境变量 `CLOUD_ISSUE_RUNS_RESIDUAL_RECONCILE_INTERVAL` 可覆盖它。显式零、负数和非法时长均拒绝启动,没有关闭开关。该间隔只安排扫描,不是资源清理成功的超时判据。Cloud 只接收既有入口提供的真实事实,不新增 Controller 发现契约;证据未到时账本保持 open。 + 测试覆盖默认值、环境覆盖、非法时长、TLS、对象存储与仓库配置。参见 [server](../../cmd/server/README.md)、[有界启动](../../docs/agent-runs/startup-settlement.md) 与 [部署出网配置](../../docs/operations/deployment-egress.md)。 diff --git a/internal/config/config.go b/internal/config/config.go index 50ea02a4..20c1d589 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -50,6 +50,10 @@ type IssueRunsConfig struct { // rejects startup rather than silently selecting a zero-length window that would abandon every // blocked creation the moment it blocked. AgentWorkspaceBlockedTimeout time.Duration `mapstructure:"agent_workspace_blocked_timeout"` + // ResidualReconcileInterval is the cadence for Cloud's residual-ledger recovery pass. Omission + // selects one minute; an explicit non-positive or malformed duration rejects startup. This + // schedules reconciliation and is never a deadline proving external cleanup succeeded. + ResidualReconcileInterval time.Duration `mapstructure:"residual_reconcile_interval"` // ThreadIdleTimeout is Thread D4's idle window: how long a Thread may sit `idle` before Cloud // asks the session to end. It is a duration configured per deployment, never a per-run column, // and the comparison that judges it runs on database time — no Cloud, Controller or Node process @@ -187,6 +191,7 @@ func Load(configPath string) (*Config, error) { v.SetDefault("issue_runs.agent_restore_timeout", DefaultAgentRestoreTimeout) v.SetDefault("issue_runs.agent_end_confirm_timeout", DefaultAgentEndConfirmTimeout) v.SetDefault("issue_runs.agent_workspace_blocked_timeout", DefaultAgentWorkspaceBlockedTimeout) + v.SetDefault("issue_runs.residual_reconcile_interval", DefaultResidualReconcileInterval) v.SetDefault("workflow.session_turn_timeout", DefaultWorkflowSessionTurnTimeout) v.SetDefault("plugins.asset_import_interval", DefaultAssetImportInterval) var cfg Config @@ -211,6 +216,9 @@ func Load(configPath string) (*Config, error) { if cfg.IssueRuns.AgentStartTimeout <= 0 || cfg.IssueRuns.AgentRestoreTimeout <= 0 || cfg.IssueRuns.AgentEndConfirmTimeout <= 0 || cfg.IssueRuns.AgentWorkspaceBlockedTimeout <= 0 { return nil, fmt.Errorf("issue_runs startup, restore, end confirmation and workspace blocked timeouts must be positive durations") } + if cfg.IssueRuns.ResidualReconcileInterval <= 0 { + return nil, fmt.Errorf("issue_runs.residual_reconcile_interval must be a positive duration") + } // IssueRun D5's two approved defaults, applied for the same reason: an operator who omits the keys // gets the limits the decision fixes, not a zero-length window that would abandon every delivery // immediately. @@ -247,6 +255,9 @@ const DefaultAgentEndConfirmTimeout = 5 * time.Minute // issue_runs.agent_workspace_blocked_timeout is absent or not positive. const DefaultAgentWorkspaceBlockedTimeout = 30 * time.Minute +// DefaultResidualReconcileInterval is the residual-ledger recovery cadence when omitted. +const DefaultResidualReconcileInterval = time.Minute + // DefaultWorkflowSessionTurnTimeout is circus#299 option A's first-version window, applied when // workflow.session_turn_timeout is absent or not positive. It is the same order of magnitude as // IssueRun D5's delivery give-up limit: far longer than a normal single turn, short enough that a diff --git a/internal/config/residual_reconcile_test.go b/internal/config/residual_reconcile_test.go new file mode 100644 index 00000000..1b5209ea --- /dev/null +++ b/internal/config/residual_reconcile_test.go @@ -0,0 +1,60 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestResidualReconcileIntervalDefaultsAndOverrides(t *testing.T) { + for _, tc := range []struct { + name, yaml, environment string + want time.Duration + }{ + {"omitted", "", "", time.Minute}, + {"file", "issue_runs: {residual_reconcile_interval: 45s}\n", "", 45 * time.Second}, + {"environment with omitted key", "", "2m", 2 * time.Minute}, + {"environment overrides file", "issue_runs: {residual_reconcile_interval: 45s}\n", "3m", 3 * time.Minute}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("CLOUD_ISSUE_RUNS_RESIDUAL_RECONCILE_INTERVAL", tc.environment) + cfg, err := Load(writeResidualReconcileConfig(t, tc.yaml)) + if err != nil { + t.Fatal(err) + } + if cfg.IssueRuns.ResidualReconcileInterval != tc.want { + t.Fatalf("residual reconcile interval = %s, want %s", cfg.IssueRuns.ResidualReconcileInterval, tc.want) + } + }) + } +} + +func TestResidualReconcileIntervalRejectsInvalidConfiguration(t *testing.T) { + for _, source := range []string{"file", "environment"} { + for _, value := range []string{"0s", "-1m", "infinite", "999999999999999999h"} { + t.Run(source+"/"+value, func(t *testing.T) { + yaml, environment := "issue_runs: {residual_reconcile_interval: "+value+"}\n", "" + if source == "environment" { + yaml, environment = "", value + } + t.Setenv("CLOUD_ISSUE_RUNS_RESIDUAL_RECONCILE_INTERVAL", environment) + _, err := Load(writeResidualReconcileConfig(t, yaml)) + if err == nil || !strings.Contains(err.Error(), "residual_reconcile_interval") { + t.Fatalf("invalid residual reconcile interval %q from %s must reject configuration, got %v", value, source, err) + } + }) + } + } +} + +func writeResidualReconcileConfig(t *testing.T, issueRuns string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "config.yaml") + content := "server: {read_timeout: 10s, write_timeout: 15s}\ndatabase: {conn_max_lifetime: 1h}\ncontrol: {grpc_addr: '127.0.0.1:8082'}\n" + issueRuns + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + return path +} diff --git a/internal/core/README.en.md b/internal/core/README.en.md index eb9e26a8..392dcae9 100644 --- a/internal/core/README.en.md +++ b/internal/core/README.en.md @@ -14,6 +14,8 @@ This package owns business state, authorization, transactions and recovery decis | `repository_*.go`, `git_*.go` | Bindings, sync/publish Operations, GitGrant and credential metadata | [External Git line](../../docs/repositories/external-git.en.md) | | `issue*.go`, `collaboration*.go`, `form_descriptor.go` | Issue aggregates, timeline, interactions and consuming collaboration ports | [Issue collaboration](../../docs/collaboration/interactions.en.md) | | `agent_run_*.go`, `execution*.go`, `thread*.go`, `business_hooks.go` | Run lifecycle, execution registry, Thread and transaction-scoped settlement hooks | [IssueRun](../../docs/agent-runs/issue-runs.en.md), [Controller](../../docs/runtime/controller-integration.en.md) | +| `agent_run_residual.go`, `agent_run_residual_reclaim.go` | Original identity/target coverage and evidence eligibility, one reclamation responsibility and one owned-delete operation, effect/permit and completion checks | [Residual reclamation](../../docs/agent-runs/residual-reclamation.en.md) | +| `agent_run_residual_fact.go`, `agent_run_residual_loop.go` | Appended facts without rewriting old responsibility, replay and durable conflict blocking; ledger scans independent of Run phase and original-operation recovery | [Residual reclamation](../../docs/agent-runs/residual-reclamation.en.md) | | `task_conversation_*.go`, `thread_preview.go` | Ordered task messages, serial same-task Agent handoff and pause barriers, follow-up prompts frozen from Node visible text, configuration-gap hints and a bounded in-process live preview; initial Agent prompts read only the frozen run input | [Thread](../../docs/agent-runs/threads.en.md), [IssueRun](../../docs/agent-runs/issue-runs.en.md) | | `revision*.go`, `session_*.go` | Revision registration, archives, session listing and statistics | [Revision](../../docs/agent-runs/revisions.en.md), [Session reads](../../docs/agent-runs/session-reads.en.md) | | `model_*.go` | Personal configuration, frozen bindings, grant digests and request observations; no API Key decryption | [Model access](../../docs/models/model-access.en.md) | @@ -30,3 +32,9 @@ This package owns business state, authorization, transactions and recovery decis - Commands wire Store ports and policy durations. See [collab](../collab/README.en.md) for development ports; simulated results do not replace physical execution evidence. See [migrations](migrations/README.en.md) for complete migration filenames. Package `*_db_test.go` tests need isolated real PostgreSQL schemas; [integration](../../integration/README.en.md) covers HTTP, Git and storage boundaries. + +Residual facts retain their original intake: `agent_run_provisioning_fact.go` handles creation facts, `node_execution.go` and `revision.go` handle +original execution terminals and delivery, and `runtime_control_delivery.go` accepts full runtime evidence through existing `runtime_ack`. +`run_workspace.go`, `control.go` and effect/permit paths enforce the original responsibility binding. No new RPC/proto or public system-delete +flag is introduced. This is Cloud step 1 only; see the dedicated page for conservative A1–A6 eligibility and remaining scope, without implying +physical or joint acceptance. diff --git a/internal/core/README.md b/internal/core/README.md index c78cee3c..93b74f2a 100644 --- a/internal/core/README.md +++ b/internal/core/README.md @@ -14,6 +14,8 @@ | `repository_*.go`、`git_*.go` | Binding、同步/发布 Operation、GitGrant 与凭据元数据 | [外部 Git 线](../../docs/repositories/external-git.md) | | `issue*.go`、`collaboration*.go`、`form_descriptor.go` | Issue 聚合、时间线、交互和消费方协作端口 | [Issue 协作](../../docs/collaboration/interactions.md) | | `agent_run_*.go`、`execution*.go`、`thread*.go`、`business_hooks.go` | 运行生命周期、执行登记、Thread 与同事务结算钩子 | [IssueRun](../../docs/agent-runs/issue-runs.md)、[Controller](../../docs/runtime/controller-integration.md) | +| `agent_run_residual.go`、`agent_run_residual_reclaim.go` | 原身份/目标覆盖与证据资格、单一回收责任和唯一 owned-delete operation、effect/许可及完成检查 | [残留回收](../../docs/agent-runs/residual-reclamation.md) | +| `agent_run_residual_fact.go`、`agent_run_residual_loop.go` | 不改旧责任的追加事实、重放与持久冲突阻挡;独立于 Run phase 的账本扫描和原 operation 恢复 | [残留回收](../../docs/agent-runs/residual-reclamation.md) | | `task_conversation_*.go`、`thread_preview.go` | 任务级有序消息、同任务 Agent 串行交接与暂停屏障、从 Node 可见正文冻结的后续提示、配置缺口提示和有界进程内实时预览;Agent 初始提示只读冻结的运行输入 | [Thread](../../docs/agent-runs/threads.md)、[IssueRun](../../docs/agent-runs/issue-runs.md) | | `revision*.go`、`session_*.go` | Revision 登记、归档读取、会话列表与统计 | [Revision](../../docs/agent-runs/revisions.md)、[会话读取](../../docs/agent-runs/session-reads.md) | | `model_*.go` | 个人配置、冻结绑定、授权摘要与请求观察;不解密 API Key | [模型访问](../../docs/models/model-access.md) | @@ -30,3 +32,7 @@ - `Store` 的端口与策略时长由命令装配。开发协作端口见 [collab](../collab/README.md);物理执行证据不由模拟结果代替。 迁移目录与完整文件名见 [migrations](migrations/README.md)。包内 `*_db_test.go` 需独立真实 PostgreSQL schema;跨 HTTP、Git 与存储边界的覆盖在 [integration](../../integration/README.md)。 + +残留事实继续由原入口接管:`agent_run_provisioning_fact.go` 处理创建侧,`node_execution.go` 与 `revision.go` 处理原执行终态和交付, +`runtime_control_delivery.go` 接收已有 `runtime_ack` 的完整运行时证据。`run_workspace.go`、`control.go` 及 effect/许可路径检查原责任绑定; +不新增 RPC/proto 或公开的系统删除开关。当前仅为 Cloud 第 1 步实现,A1–A6 的保守资格和后续未实现范围见上述专页,不代表物理或联合验收完成。 diff --git a/internal/core/agent_run_provisioning_fact.go b/internal/core/agent_run_provisioning_fact.go index eaf120ce..f7ebed8d 100644 --- a/internal/core/agent_run_provisioning_fact.go +++ b/internal/core/agent_run_provisioning_fact.go @@ -24,7 +24,7 @@ func provisioningFact(t *transaction, r *ControlRequest) Object { w := t.one("SELECT * FROM workspaces WHERE id=$1", o.S("workspaceId")) require(w != nil && w.S("issueRunId") != "", 409, "operation_not_abandoned") run := w.S("issueRunId") - require(t.one("SELECT run_id FROM agent_run_residuals WHERE run_id=$1 AND cleared_at IS NULL", run) != nil, + require(t.one("SELECT run_id FROM agent_run_residuals WHERE run_id=$1", run) != nil, 409, "operation_not_abandoned") switch r.Body.S("kind") { @@ -59,9 +59,7 @@ func provisioningEffectFact(t *transaction, r *ControlRequest, o Object, run str factOp["controllerEpoch"] = r.Body.N("epoch") out := effectResult(t, r, factOp) e := out.O("effect") - appendResidualFact(t, run, "effect:"+e.S("id"), Object{ - "kind": "effect", "effectKind": e.S("kind"), "state": e.S("state"), "externalId": e.S("externalId"), "result": e.O("result"), - }) + // effectResult appends the same validated fact on both the live and late intake paths. return Object{"operation": t.one("SELECT * FROM operations WHERE id=$1", o.S("id")), "effect": e} } @@ -86,9 +84,7 @@ func provisioningExecutionFact(t *transaction, r *ControlRequest, run string) Ob require(e.S("kind") == "install_plugins" || e.S("kind") == "remove_plugins", 409, "invalid_execution") stored = nodeExecutionResult(t, factRequest, false) } - appendResidualFact(t, run, r.Body.S("kind")+":"+stored.S("executionId"), Object{ - "kind": r.Body.S("kind"), "state": executionState(stored), "result": stored.O("result"), - }) + // The execution applier owns the ledger receipt as well as the canonical result. return Object{"operation": t.one("SELECT * FROM operations WHERE id=$1", r.OperationID), "execution": stored} } @@ -121,65 +117,3 @@ func provisioningCreationFact(t *transaction, r *ControlRequest, run string) Obj appendResidualFact(t, run, "creation", fact) return Object{"operation": t.one("SELECT * FROM operations WHERE id=$1", r.OperationID)} } - -// appendResidualFact receipts one late fact onto the abandoned creation's residual ledger. The -// fact key is its stable identity; the same fact replays as a no-op. A provisional observation -// (running, absent/planned, pending) does not lock the identity: the real world progresses, so a -// later fact — above all the real terminal — takes over and the superseded observation is kept -// as prior evidence. A recorded terminal is final: only its identical replay is accepted and any -// different fact for the identity is a conflict, so a retried submission can never stack -// duplicates or rewrite an outcome. The fact is durable before the caller's ACK: this write is -// part of the submission's transaction. -func appendResidualFact(t *transaction, run, key string, fact Object) { - row := t.one("SELECT responsibility FROM agent_run_residuals WHERE run_id=$1 AND cleared_at IS NULL", run) - require(row != nil, 409, "operation_not_abandoned") - resp := row.O("responsibility") - facts := Object{} - for k, v := range resp.O("lateFacts") { - if m, ok := v.(map[string]any); ok { - facts[k] = Object(m) - } - } - var prev Object - switch v := facts[key].(type) { - case map[string]any: - prev = Object(v) - case Object: - prev = v - } - if prev != nil { - if prev.S("state") == fact.S("state") && prev.S("externalId") == fact.S("externalId") && jsonText(prev.O("result")) == jsonText(fact.O("result")) { - return - } - // The recorded outcome is final: a different fact contradicts it. - require(!residualFactTerminal(prev), 409, "fact_conflict") - // A provisional observation is superseded, not contradicted: keep it as prior evidence - // while the later fact becomes the identity's current receipt. - superseded := Object{} - for k, v := range prev { - if k != "prior" { - superseded[k] = v - } - } - prior := []any{} - if old, ok := prev["prior"].([]any); ok { - prior = append(prior, old...) - } - fact["prior"] = append(prior, superseded) - } - fact["receivedAt"] = t.one("SELECT clock_timestamp()::text AS received_at").S("receivedAt") - facts[key] = fact - resp["lateFacts"] = facts - t.exec("UPDATE agent_run_residuals SET responsibility=$2 WHERE run_id=$1", run, jsonText(resp)) -} - -// residualFactTerminal reports whether one ledger fact is an outcome rather than an observation: -// effects and executions end at succeeded/failed, creations at ready/failed; running, -// absent/planned and pending are states the real world can still move on from. -func residualFactTerminal(fact Object) bool { - switch fact.S("state") { - case "succeeded", "failed", "ready": - return true - } - return false -} diff --git a/internal/core/agent_run_provisioning_fact_db_test.go b/internal/core/agent_run_provisioning_fact_db_test.go index 7d3cdd4a..d1005285 100644 --- a/internal/core/agent_run_provisioning_fact_db_test.go +++ b/internal/core/agent_run_provisioning_fact_db_test.go @@ -2,6 +2,7 @@ package core import ( "context" + "errors" "strings" "testing" ) @@ -15,8 +16,9 @@ import ( // deadline, abandonment committed. The lease stays seeded for the fact submissions. type abandonedCreation struct { provisioningSeed - claims *Claims - effect string + claims *Claims + effect string + responsibility string } // seedAbandonedCreation parks a creation in blocked (freezing the deadline), optionally plans one @@ -61,7 +63,30 @@ func seedAbandonedCreation(t *testing.T, store *Store, withEffect bool) abandone if phase.String != "done" || status != "failed" || reason != "workspace_unavailable" { t.Fatalf("seed did not abandon the creation: %s/%s/%s", phase.String, status, reason) } - return abandonedCreation{provisioningSeed: p, claims: claims, effect: effect} + var responsibility string + if err := store.Pool.QueryRow(`SELECT responsibility::text FROM agent_run_residuals WHERE run_id=$1`, p.run).Scan(&responsibility); err != nil { + t.Fatal(err) + } + return abandonedCreation{provisioningSeed: p, claims: claims, effect: effect, responsibility: responsibility} +} + +func assertProvisioningResponsibilityUnchanged(t *testing.T, store *Store, scene abandonedCreation) { + t.Helper() + var unchanged bool + if err := store.Pool.QueryRow(`SELECT responsibility=$2::jsonb FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.responsibility).Scan(&unchanged); err != nil { + t.Fatal(err) + } + if !unchanged { + t.Fatal("late facts changed the original residual responsibility") + } +} + +func assertResidualFactConflict(t *testing.T, err error) { + t.Helper() + var fault *Fault + if !errors.As(err, &fault) || fault.Status != 409 || fault.Code != "residual_fact_conflict" { + t.Fatalf("want stable residual_fact_conflict Fault409, got %v", err) + } } // submitFact drives the provisioning_fact action; a non-nil error is returned as-is for conflict @@ -101,21 +126,36 @@ func TestProvisioningFactReceiptsLateEffectWithoutResurrecting(t *testing.T) { } var facts int var state, external string - if err := store.Pool.QueryRow(`SELECT (SELECT count(*) FROM jsonb_object_keys(responsibility->'lateFacts')), - responsibility->'lateFacts'->('effect:'||$2)->>'state', responsibility->'lateFacts'->('effect:'||$2)->>'externalId' - FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.effect).Scan(&facts, &state, &external); err != nil { + if err := store.Pool.QueryRow(`SELECT jsonb_array_length(reclaim_evidence), + reclaim_evidence->0->'fact'->>'state', reclaim_evidence->0->'fact'->>'externalId' + FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&facts, &state, &external); err != nil { t.Fatalf("ledger receipt read failed: %v", err) } if facts != 1 || state != "succeeded" || external != "sub-123" { t.Fatalf("ledger receipt wrong: count=%d state=%s external=%s", facts, state, external) } - // A different fact for the same effect identity is refused — by the live path's own evidence - // rules here (a succeeded effect cannot become failed), which is exactly the guarantee the - // entry reuses — and the ledger keeps the original receipt. + // A conflicting fact is durable before the stable error is returned. The canonical effect + // remains unchanged, and both a lost-response replay and a new submission deduplicate it. conflicting := Object{"epoch": 1, "kind": "effect", "effectId": scene.effect, "state": "failed", "externalId": "sub-123", "result": Object{"error": "substrate said no"}} - if _, err := submitFact(t, store, scene, "fact-3", conflicting); err == nil { - t.Fatal("contradicting fact accepted") + for _, submission := range []string{"fact-3", "fact-3", "fact-4"} { + _, err := submitFact(t, store, scene, submission, conflicting) + assertResidualFactConflict(t, err) + } + var committed bool + if err := store.Pool.QueryRow(`SELECT jsonb_array_length(r.reclaim_evidence)=2 AND r.reclaim_conflicted_at IS NOT NULL + AND r.reclaim_evidence->1->>'key'='effect:'||$2 + AND r.reclaim_evidence->1->'fact'->>'kind'='conflict' + AND r.reclaim_evidence->1->'fact'->'incoming'->>'state'='failed' + AND r.reclaim_evidence->1->'fact'->'original'->>'state'='succeeded' + AND e.state='succeeded' AND e.external_id='sub-123' + AND e.result=jsonb_build_object('sandboxInstanceId',$2::text,'nodeId','node-late') + FROM agent_run_residuals r JOIN external_effects e ON e.id=$2::uuid WHERE r.run_id=$1`, scene.run, scene.effect).Scan(&committed); err != nil { + t.Fatal(err) + } + if !committed { + t.Fatal("conflict was not retained exactly once before its error, or changed the canonical effect") } + assertProvisioningResponsibilityUnchanged(t, store, scene) phase, status, reason, _, _, _ := runFields(t, store, scene.run) if phase.String != "done" || status != "failed" || reason != "workspace_unavailable" { t.Fatalf("late fact resurrected the run: %s/%s/%s", phase.String, status, reason) @@ -163,16 +203,23 @@ func TestProvisioningFactReceiptsLateCloneAndCreationOutcomes(t *testing.T) { if _, err := submitFact(t, store, scene, "creation-1", Object{"epoch": 1, "kind": "creation", "outcome": "ready"}); err != nil { t.Fatalf("creation fact submission replay failed: %v", err) } - if _, err := submitFact(t, store, scene, "creation-2", Object{"epoch": 1, "kind": "creation", "outcome": "failed"}); err == nil || !strings.Contains(err.Error(), "fact_conflict") { - t.Fatalf("contradicting creation outcome accepted: %v", err) + for _, submission := range []string{"creation-2", "creation-2", "creation-3"} { + _, err := submitFact(t, store, scene, submission, Object{"epoch": 1, "kind": "creation", "outcome": "failed"}) + assertResidualFactConflict(t, err) } var creationState string - if err := store.Pool.QueryRow(`SELECT responsibility->'lateFacts'->'creation'->>'state' FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&creationState); err != nil { + var conflicts int + if err := store.Pool.QueryRow(`SELECT + (SELECT e->'fact'->>'state' FROM jsonb_array_elements(reclaim_evidence) e WHERE e->>'key'='creation' AND e->'fact'->>'kind'='creation'), + (SELECT count(*) FROM jsonb_array_elements(reclaim_evidence) e WHERE e->>'key'='creation' AND e->'fact'->>'kind'='conflict' + AND e->'fact'->'incoming'->>'state'='failed' AND e->'fact'->'original'->>'state'='ready' AND reclaim_conflicted_at IS NOT NULL) + FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&creationState, &conflicts); err != nil { t.Fatal(err) } - if creationState != "ready" { - t.Fatalf("creation fact not receipted: %s", creationState) + if creationState != "ready" || conflicts != 1 { + t.Fatalf("creation fact or durable deduplicated conflict wrong: state=%s conflicts=%d", creationState, conflicts) } + assertProvisioningResponsibilityUnchanged(t, store, scene) // A late success is evidence only: the run keeps its business terminal and the Workspace keeps // its admission fence. var admissionOpen bool @@ -286,22 +333,22 @@ func TestProvisioningEffectFactProgressesFromProvisionalToTerminal(t *testing.T) if _, err := submitFact(t, store, scene, "progress-replay", terminal); err != nil { t.Fatalf("identical terminal must replay: %v", err) } - // One receipt for the identity: the terminal on top, both provisional observations retained. - var state string - var prior int - if err := store.Pool.QueryRow(`SELECT responsibility->'lateFacts'->('effect:'||$2)->>'state', - jsonb_array_length(responsibility->'lateFacts'->('effect:'||$2)->'prior') - FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.effect).Scan(&state, &prior); err != nil { + // Each observation remains an independent append-only entry in arrival order. + var states string + var entries int + if err := store.Pool.QueryRow(`SELECT string_agg(e->'fact'->>'state', ',' ORDER BY ordinal), count(*) + FROM agent_run_residuals r, jsonb_array_elements(r.reclaim_evidence) WITH ORDINALITY AS facts(e,ordinal) + WHERE r.run_id=$1 AND e->>'key'='effect:'||$2`, scene.run, scene.effect).Scan(&states, &entries); err != nil { t.Fatalf("ledger receipt read failed: %v", err) } - if state != "succeeded" || prior != 2 { - t.Fatalf("ledger receipt wrong: state=%s prior=%d", state, prior) + if states != "planned,running,succeeded" || entries != 3 { + t.Fatalf("ledger observations wrong: states=%s entries=%d", states, entries) } // A terminal is final: a contradicting outcome stays refused, by the live path's own evidence // rules for the effect row, and the ledger keeps the recorded terminal. - if _, err := submitFact(t, store, scene, "progress-conflict", Object{"epoch": 1, "kind": "effect", "effectId": scene.effect, "state": "failed", "externalId": "sub-123", "result": Object{"error": "substrate said no"}}); err == nil { - t.Fatal("contradicting terminal accepted") - } + _, err = submitFact(t, store, scene, "progress-conflict", Object{"epoch": 1, "kind": "effect", "effectId": scene.effect, "state": "failed", "externalId": "sub-123", "result": Object{"error": "substrate said no"}}) + assertResidualFactConflict(t, err) + assertProvisioningResponsibilityUnchanged(t, store, scene) phase, status, reason, _, _, _ := runFields(t, store, scene.run) if phase.String != "done" || status != "failed" || reason != "workspace_unavailable" { t.Fatalf("progression resurrected the run: %s/%s/%s", phase.String, status, reason) @@ -341,7 +388,8 @@ func TestProvisioningFactSubmissionBindsItsTarget(t *testing.T) { // The ledger still holds exactly the two facts receipted for their original targets. var facts int - if err := store.Pool.QueryRow(`SELECT count(*) FROM jsonb_object_keys((SELECT responsibility->'lateFacts' FROM agent_run_residuals WHERE run_id=$1))`, scene.run).Scan(&facts); err != nil || facts != 2 { + if err := store.Pool.QueryRow(`SELECT jsonb_array_length(reclaim_evidence) FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&facts); err != nil || facts != 2 { t.Fatalf("ledger receipts wrong: count=%d err=%v", facts, err) } + assertProvisioningResponsibilityUnchanged(t, store, scene) } diff --git a/internal/core/agent_run_residual.go b/internal/core/agent_run_residual.go new file mode 100644 index 00000000..cab7db3f --- /dev/null +++ b/internal/core/agent_run_residual.go @@ -0,0 +1,317 @@ +package core + +import "sort" + +// residualIdentity re-reads the original ownership in the caller's serialized transaction. +// Neither a request's reclaim ID nor a UUID foreign key alone establishes this scope. +func residualIdentity(t *transaction, row Object) (run, workspace, project Object) { + require(row != nil, 404, "residual_not_found") + run = t.one("SELECT * FROM issue_runs WHERE id=$1", row.S("runId")) + workspace = t.one("SELECT * FROM workspaces WHERE id=$1", row.S("workspaceId")) + project = t.one("SELECT * FROM projects WHERE id=$1", row.S("projectId")) + require(run != nil && workspace != nil && project != nil && run.S("executorType") == "agent" && + run.S("tenantId") == row.S("tenantId") && run.S("workspaceId") == workspace.S("id") && + workspace.S("issueRunId") == run.S("id") && workspace.S("tenantId") == row.S("tenantId") && + workspace.S("projectId") == project.S("id") && project.S("tenantId") == row.S("tenantId") && + workspace.S("kind") == "isolated" && workspace.N("runtimeGeneration") == row.N("runtimeGeneration"), 409, "residual_identity_conflict") + return run, workspace, project +} + +// collectResidualTargets is a union, never a replacement snapshot. It includes original NULL-scope +// effects by their original operation, and retains missing historical references so absence cannot +// silently turn an unknown target into permission to delete data. The bound delete is not a new +// provisioning responsibility. Each target has kind/id plus its original associations. +func collectResidualTargets(t *transaction, row Object) []Object { + _, w, _ := residualIdentity(t, row) + byKey := map[string]Object{} + add := func(target Object) { + if target.S("id") == "" { + return + } + key := target.S("kind") + ":" + target.S("id") + if previous := byKey[key]; previous != nil { + for k, v := range target { + if _, exists := previous[k]; !exists { + previous[k] = v + } + } + return + } + byKey[key] = target + } + for _, target := range residualObjects(row["reclaimTargets"]) { + add(target) + } + ref := row.O("responsibility") + for _, spec := range []struct{ list, kind, key string }{ + {"operations", "operation", "id"}, + {"effects", "effect", "id"}, + {"sandboxes", "sandbox", "id"}, + {"nodes", "node", "id"}, + {"executions", "execution", "executionId"}, + {"clones", "clone", "executionId"}, + } { + for _, original := range residualObjects(ref[spec.list]) { + target := Object{"kind": spec.kind, "id": original.S(spec.key), "workspaceId": w.S("id")} + for _, key := range []string{"operationId", "sandboxInstanceId", "generation", "nodeId", "workId"} { + if v, exists := original[key]; exists { + target[key] = v + } + } + add(target) + } + } + for _, op := range t.list("SELECT * FROM operations WHERE workspace_id=$1 AND id IS DISTINCT FROM $2::uuid", w.S("id"), nullable(row.S("reclaimOperationId"))) { + add(Object{"kind": "operation", "id": op.S("id"), "workspaceId": w.S("id")}) + } + // Resolve all original operations, including a Project-scoped operation named in responsibility. + operations := []string{} + for _, target := range byKey { + if target.S("kind") == "operation" { + operations = append(operations, target.S("id")) + } + } + for _, id := range operations { + for _, e := range t.list("SELECT * FROM external_effects WHERE operation_id=$1 AND (workspace_id=$2 OR workspace_id IS NULL)", id, w.S("id")) { + add(Object{"kind": "effect", "id": e.S("id"), "operationId": id, "workspaceId": w.S("id")}) + } + } + for _, e := range t.list("SELECT * FROM external_effects WHERE workspace_id=$1 AND operation_id IS DISTINCT FROM $2::uuid", w.S("id"), nullable(row.S("reclaimOperationId"))) { + add(Object{"kind": "effect", "id": e.S("id"), "operationId": e.S("operationId"), "workspaceId": w.S("id")}) + } + // Every planned ensure is itself a stable sandbox identity, even before a sandbox row arrives. + effects := []string{} + for _, target := range byKey { + if target.S("kind") == "effect" { + effects = append(effects, target.S("id")) + } + } + for _, id := range effects { + e := t.one("SELECT * FROM external_effects WHERE id=$1", id) + if e != nil && e.S("kind") == "sandbox_ensure" { + add(Object{"kind": "sandbox", "id": id, "ensureEffectId": id, "operationId": e.S("operationId"), "workspaceId": w.S("id")}) + } + } + for _, s := range t.list("SELECT * FROM sandbox_instances WHERE workspace_id=$1", w.S("id")) { + add(Object{"kind": "sandbox", "id": s.S("id"), "generation": s.N("generation"), "workspaceId": w.S("id")}) + } + for _, n := range t.list("SELECT * FROM node_instances WHERE workspace_id=$1", w.S("id")) { + add(Object{"kind": "node", "id": n.S("id"), "sandboxInstanceId": n.S("sandboxInstanceId"), "nodeId": n.S("nodeId"), "workspaceId": w.S("id")}) + } + for _, e := range t.list("SELECT * FROM node_executions WHERE workspace_id=$1 OR operation_id=$2", w.S("id"), row.S("runId")) { + add(Object{"kind": "execution", "id": e.S("executionId"), "operationId": e.S("operationId"), "nodeId": e.S("nodeId"), "workId": e.S("workId"), "workspaceId": w.S("id")}) + } + for _, e := range t.list("SELECT * FROM clone_executions WHERE workspace_id=$1", w.S("id")) { + add(Object{"kind": "clone", "id": e.S("executionId"), "operationId": e.S("operationId"), "nodeId": e.S("nodeId"), "workspaceId": w.S("id")}) + } + for _, work := range t.list("SELECT * FROM execution_work WHERE run_id=$1", row.S("runId")) { + add(Object{"kind": "work", "id": work.S("id"), "workspaceId": w.S("id")}) + if sid := work.O("target").S("sandboxInstanceId"); sid != "" { + add(Object{"kind": "sandbox", "id": sid, "workspaceId": w.S("id"), "generation": work.O("target").N("runtimeGeneration")}) + } + } + add(Object{"kind": "data", "id": w.S("id"), "workspaceId": w.S("id")}) + keys := make([]string, 0, len(byKey)) + for key := range byKey { + keys = append(keys, key) + } + sort.Strings(keys) + targets := make([]Object, 0, len(keys)) + for _, key := range keys { + targets = append(targets, byKey[key]) + } + return targets +} + +func refreshResidualTargets(t *transaction, row Object) Object { + targets := collectResidualTargets(t, row) + if jsonText(targets) != jsonText(row["reclaimTargets"]) { + t.exec("UPDATE agent_run_residuals SET reclaim_targets=$2,reclaim_version=reclaim_version+1 WHERE run_id=$1", row.S("runId"), jsonText(targets)) + } + return t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", row.S("runId")) +} + +// residualSandboxTerminated requires the two distinct physical assertions for the exact stable +// sandbox ID. terminated_at, failed ensures, elapsed time and a missing sandbox row prove neither. +func residualSandboxTerminated(t *transaction, row Object, sid string) bool { + for _, e := range t.list(`SELECT * FROM external_effects WHERE project_id=$1 AND workspace_id=$2 + AND kind='sandbox_terminate' AND request->>'sandboxInstanceId'=$3 AND state='succeeded'`, row.S("projectId"), row.S("workspaceId"), sid) { + if e.O("result").B("terminated") && e.O("result").B("lateEnsureFenced") { + return true + } + } + return false +} + +func residualAllSandboxesTerminated(t *transaction, row Object) bool { + count := 0 + for _, target := range residualObjects(row["reclaimTargets"]) { + if target.S("kind") == "sandbox" { + count++ + if !residualSandboxTerminated(t, row, target.S("id")) { + return false + } + } + } + return count > 0 +} + +// residualTargetsLocated checks references against the real tables. Unknown historical references +// cannot be normalized to the newest generation or dropped from the responsibility. +func residualTargetsLocated(t *transaction, row Object) bool { + for _, target := range residualObjects(row["reclaimTargets"]) { + id, wid := target.S("id"), row.S("workspaceId") + switch target.S("kind") { + case "sandbox": + s := t.one("SELECT * FROM sandbox_instances WHERE id=$1 AND workspace_id=$2", id, wid) + if s == nil { + e := t.one(`SELECT e.id FROM external_effects e JOIN operations o ON o.id=e.operation_id + WHERE e.id=$1 AND e.kind='sandbox_ensure' AND e.project_id=$2 AND o.tenant_id=$3 + AND (e.workspace_id=$4 OR (e.workspace_id IS NULL AND o.workspace_id=$4))`, id, row.S("projectId"), row.S("tenantId"), wid) + if e == nil { + return false + } + } else if target.N("generation") > 0 && target.N("generation") != s.N("generation") { + return false + } + case "operation": + if t.one("SELECT id FROM operations WHERE id=$1 AND tenant_id=$2 AND project_id=$3 AND (workspace_id=$4 OR workspace_id IS NULL)", id, row.S("tenantId"), row.S("projectId"), wid) == nil { + return false + } + case "effect": + if t.one(`SELECT e.id FROM external_effects e JOIN operations o ON o.id=e.operation_id + WHERE e.id=$1 AND o.tenant_id=$2 AND e.project_id=$3 AND (e.workspace_id=$4 OR (e.workspace_id IS NULL AND o.workspace_id=$4))`, id, row.S("tenantId"), row.S("projectId"), wid) == nil { + return false + } + case "node": + if t.one("SELECT id FROM node_instances WHERE id=$1 AND workspace_id=$2 AND sandbox_instance_id=$3", id, wid, target.S("sandboxInstanceId")) == nil { + return false + } + case "execution": + if t.one("SELECT execution_id FROM node_executions WHERE execution_id=$1 AND workspace_id=$2", id, wid) == nil { + return false + } + case "clone": + if t.one("SELECT execution_id FROM clone_executions WHERE execution_id=$1 AND workspace_id=$2", id, wid) == nil { + return false + } + case "work": + if t.one("SELECT id FROM execution_work WHERE id=$1 AND run_id=$2 AND target->>'workspaceId'=$3", id, row.S("runId"), wid) == nil { + return false + } + case "data": + if id != wid { + return false + } + default: + return false + } + } + return true +} + +func residualExecutionsTerminal(t *transaction, row Object) bool { + for _, target := range residualObjects(row["reclaimTargets"]) { + var e Object + switch target.S("kind") { + case "execution": + e = t.one("SELECT * FROM node_executions WHERE execution_id=$1", target.S("id")) + case "clone": + e = t.one("SELECT * FROM clone_executions WHERE execution_id=$1", target.S("id")) + default: + continue + } + if e == nil || e.O("result").S("outcome") == "" { + return false + } + } + return true +} + +func residualInputClosed(t *transaction, row Object) bool { + c := t.one("SELECT * FROM runtime_controls WHERE workspace_id=$1", row.S("workspaceId")) + if c == nil || !c.B("inputClosed") { + return false + } + for _, receipt := range residualObjects(row["reclaimEvidence"]) { + fact := receipt.O("fact") + if fact.S("kind") != "runtime_closed" || !fact.B("inputClosed") || fact.S("workspaceId") != row.S("workspaceId") || + fact.N("controlEpoch") != c.N("controlEpoch") || fact.S("sandboxId") != c.S("boundSandboxId") { + continue + } + // The full list is retained by the verified runtime_ack path; an omitted list is unknown. + ids, exists := fact["unfinishedExecutionIds"] + if !exists { + continue + } + switch values := ids.(type) { + case []any: + if len(values) == 0 { + return true + } + case []string: + if len(values) == 0 { + return true + } + } + } + return false +} + +func residualDeliveryComplete(t *transaction, row Object) bool { + // There is presently no durable explicit delivery-abandonment record type. Startup/creation + // abandoned, failed/skipped, timeout and revision_delivery_skips are not such a record. Until + // an existing explicit source is identified, only the already verified Revision path qualifies. + verified := t.one(`SELECT r.id FROM revisions r JOIN revision_verifications v ON v.execution_id=r.execution_id + JOIN node_executions e ON e.execution_id=r.execution_id JOIN execution_work ew ON ew.id=e.work_id + WHERE r.run_id=$1 AND r.tenant_id=$2 AND r.project_id=$3 AND r.workspace_id=$4 + AND e.operation_id=r.run_id AND e.workspace_id=r.workspace_id AND e.kind='deliver_revision' + AND ew.run_id=r.run_id AND ew.kind='deliver_revision' AND ew.execution_id=e.execution_id + AND v.outcome IN ('delivered','unchanged') AND e.result->>'outcome' IN ('revision_delivered','revision_unchanged')`, row.S("runId"), row.S("tenantId"), row.S("projectId"), row.S("workspaceId")) + if verified == nil { + return false + } + // Every actual delivery must have its own successful verification; a first saved result cannot + // cover a different late delivery that still needs rescue. + return t.one(`SELECT e.execution_id FROM node_executions e WHERE e.workspace_id=$1 AND e.kind='deliver_revision' + AND NOT EXISTS(SELECT 1 FROM revision_verifications v JOIN revisions r ON r.execution_id=v.execution_id + WHERE v.execution_id=e.execution_id AND v.outcome IN ('delivered','unchanged') AND r.run_id=$2 + AND r.workspace_id=$1 AND r.tenant_id=$3 AND r.project_id=$4)`, row.S("workspaceId"), row.S("runId"), row.S("tenantId"), row.S("projectId")) == nil +} + +func residualNeverPlanned(t *transaction, row Object) bool { + for _, target := range residualObjects(row["reclaimTargets"]) { + switch target.S("kind") { + case "sandbox", "node", "execution", "clone", "work": + return false + case "effect": + e := t.one("SELECT kind FROM external_effects WHERE id=$1", target.S("id")) + if e == nil || e.S("kind") == "sandbox_ensure" { + return false + } + } + } + return true +} + +// residualEligibility is deliberately conservative about additional historical IDs: only the one +// actual current sandbox may be passed to owned delete. Every other ID needs prior dual confirmation. +func residualEligibility(t *transaction, row Object) bool { + _, w, p := residualIdentity(t, row) + if row["reclaimConflictedAt"] != nil || w.B("admissionOpen") || p.S("lifecycle") != "active" || !residualTargetsLocated(t, row) || !residualDeliveryComplete(t, row) { + return false + } + allGone := residualAllSandboxesTerminated(t, row) + if !allGone && !residualNeverPlanned(t, row) && (!residualInputClosed(t, row) || !residualExecutionsTerminal(t, row)) { + return false + } + for _, target := range residualObjects(row["reclaimTargets"]) { + if target.S("kind") != "sandbox" || residualSandboxTerminated(t, row, target.S("id")) { + continue + } + if t.one("SELECT id FROM sandbox_instances WHERE id=$1 AND workspace_id=$2 AND generation=$3 AND terminated_at IS NULL", target.S("id"), w.S("id"), row.N("runtimeGeneration")) == nil { + return false + } + } + return t.one("SELECT id FROM execution_tickets WHERE workspace_id=$1 AND state='active' AND terminated_by_force_stop_id IS NULL", w.S("id")) == nil +} diff --git a/internal/core/agent_run_residual_fact.go b/internal/core/agent_run_residual_fact.go new file mode 100644 index 00000000..d103825e --- /dev/null +++ b/internal/core/agent_run_residual_fact.go @@ -0,0 +1,241 @@ +package core + +import ( + "fmt" + "strings" +) + +// residualForWorkspace also finds cleared responsibilities: cleanup never closes the original +// identity's evidence intake. Soft-deleted Workspace rows remain the identity authority. +func residualForWorkspace(t *transaction, workspaceID string) Object { + if workspaceID == "" { + return nil + } + return t.one("SELECT * FROM agent_run_residuals WHERE workspace_id=$1", workspaceID) +} + +func residualObjects(value any) []Object { + var out []Object + switch values := value.(type) { + case []Object: + return values + case []any: + for _, item := range values { + switch v := item.(type) { + case map[string]any: + out = append(out, Object(v)) + case Object: + out = append(out, v) + } + } + } + return out +} + +// appendResidualFact records one validated fact without editing the original responsibility. +// False means a contradictory terminal was durably audited; callers must not apply it to the +// canonical execution/effect. Control returns the stable error only after this transaction commits. +func appendResidualFact(t *transaction, run, key string, fact Object) bool { + row := t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", run) + require(row != nil, 409, "operation_not_abandoned") + var previous Object + entries := residualObjects(row["reclaimEvidence"]) + for _, entry := range entries { + if entry.S("key") != key { + continue + } + old := entry.O("fact") + if old.S("kind") == "conflict" { + if jsonText(old.O("incoming")) == jsonText(fact) { + t.residualFactConflict = true + return false + } + continue + } + if jsonText(old) == jsonText(fact) { + return true + } + previous = old + } + // Pre-migration lateFacts stay intact. They may supply the original terminal, never a reason + // to rewrite the legacy JSON or to infer cleanup eligibility during migration. + if previous == nil { + previous = row.O("responsibility").O("lateFacts").O(key) + } + if len(previous) > 0 && residualFactTerminal(previous) && !residualCleanupFailureRetryable(t, row, key, previous) && !sameResidualFact(previous, fact) { + return appendResidualConflict(t, row, key, previous, fact) + } + appendResidualEntry(t, row, key, fact, false) + return true +} + +// A3 retries the same bound cleanup operation and effect. Its failed observation does not prove +// disposal and can advance after fresh evidence; keep that observation instead of treating the +// later success as an A4 contradiction. Historical provisioning failures and every successful +// conclusion remain immutable, and external binding conflicts are checked before this path. +func residualCleanupFailureRetryable(t *transaction, row Object, key string, fact Object) bool { + if fact.S("state") != "failed" || fact.S("kind") != "effect" || row.S("reclaimMode") != "owned_delete" || + row.S("reclaimOperationId") == "" || !strings.HasPrefix(key, "effect:") { + return false + } + id := strings.TrimPrefix(key, "effect:") + return validID(id) && t.one(`SELECT id FROM external_effects WHERE id=$1 AND operation_id=$2 + AND kind IN ('sandbox_terminate','workspace_data_delete')`, id, row.S("reclaimOperationId")) != nil +} + +func appendResidualConflict(t *transaction, row Object, key string, previous, incoming Object) bool { + t.residualFactConflict = true + for _, entry := range residualObjects(row["reclaimEvidence"]) { + old := entry.O("fact") + if entry.S("key") == key && old.S("kind") == "conflict" && jsonText(old.O("incoming")) == jsonText(incoming) { + return false + } + } + appendResidualEntry(t, row, key, Object{"kind": "conflict", "state": "conflict", "original": previous, "incoming": incoming}, true) + return false +} + +func appendResidualEntry(t *transaction, row Object, key string, fact Object, conflict bool) { + source := fact.O("source") + if len(source) == 0 { + source = t.residualFactSource + if len(source) == 0 { + source = Object{"kind": "cloud_accepted_fact", "identity": key} + } + } + entry := Object{ + "key": key, "fact": fact, "source": source, + "receivedAt": t.one("SELECT clock_timestamp()::text AS received_at").S("receivedAt"), + } + t.exec(`UPDATE agent_run_residuals SET reclaim_evidence=reclaim_evidence||jsonb_build_array($2::jsonb), + reclaim_version=reclaim_version+1, + reclaim_conflicted_at=CASE WHEN $3 THEN COALESCE(reclaim_conflicted_at,clock_timestamp()) ELSE reclaim_conflicted_at END + WHERE run_id=$1`, row.S("runId"), jsonText(entry), conflict) +} + +func residualEffectRow(t *transaction, e Object) Object { + if row := residualForWorkspace(t, e.S("workspaceId")); row != nil { + return row + } + // Some legacy preallocated ensure plans have no Workspace binding on the effect row. + return t.one(`SELECT ar.* FROM agent_run_residuals ar JOIN operations o ON o.workspace_id=ar.workspace_id + WHERE o.id=$1 AND o.tenant_id=ar.tenant_id AND o.project_id=ar.project_id`, e.S("operationId")) +} + +func acceptResidualEffectFact(t *transaction, row, e, incoming Object) bool { + key := "effect:" + e.S("id") + if e.S("externalId") != "" && incoming.S("externalId") != "" && e.S("externalId") != incoming.S("externalId") { + return appendResidualConflict(t, row, key, + Object{"kind": "effect", "state": e.S("state"), "externalId": e.S("externalId"), "result": e.O("result")}, incoming) + } + for _, entry := range residualObjects(row["reclaimEvidence"]) { + if entry.S("key") == key && jsonText(entry.O("fact")) == jsonText(incoming) && e.S("state") != incoming.S("state") { + // An earlier observation can replay without downgrading the canonical terminal. + return false + } + } + if e.S("state") == "succeeded" || e.S("state") == "failed" { + original := Object{ + "kind": "effect", "effectKind": e.S("kind"), "state": e.S("state"), + "externalId": e.S("externalId"), "result": e.O("result"), + } + if !appendResidualFact(t, row.S("runId"), key, original) { + return false + } + } + return appendResidualFact(t, row.S("runId"), key, incoming) +} + +// receipts is a constant selected by the node/clone registry call sites, never request input. +func acceptResidualReceipt(t *transaction, e Object, sequence int64, event, receipts string) bool { + row := residualForWorkspace(t, e.S("workspaceId")) + if row == nil { + return true + } + old := t.one("SELECT event FROM "+receipts+" WHERE execution_id=$1 AND sequence=$2", e.S("executionId"), sequence) + if old != nil && old.S("event") != event { + prefix := "receipt:" + // Separate primary keys do not guarantee uniqueness across the two execution registries. + if receipts == "clone_event_receipts" { + prefix = "clone_receipt:" + } + key := prefix + e.S("executionId") + ":" + fmt.Sprint(sequence) + return appendResidualConflict(t, row, key, Object{"kind": "receipt", "event": old.S("event")}, Object{"kind": "receipt", "event": event}) + } + return true +} + +func sameResidualFact(a, b Object) bool { + // Legacy facts also contain receipt timestamps and provisional history. Neither changes the + // terminal's semantic identity when importing an existing receipt into the append-only column. + for _, key := range []string{"kind", "state", "externalId", "detail"} { + if a.S(key) != b.S(key) { + return false + } + } + return jsonText(a.O("result")) == jsonText(b.O("result")) +} + +func residualFactTerminal(fact Object) bool { + switch fact.S("state") { + case "succeeded", "failed", "ready": + return true + } + return false +} + +// requireNoResidualConflict fences future mutations even after a responsibility was cleared. +func requireNoResidualConflict(t *transaction, workspaceID string) { + row := residualForWorkspace(t, workspaceID) + require(row == nil || row["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") +} + +// residualExecutionIdentity verifies the original work and resource binding without requiring a +// live Node, open admission, an unfinished Run, or an undeleted Workspace. +func residualExecutionIdentity(t *transaction, e Object) Object { + row := residualForWorkspace(t, e.S("workspaceId")) + if row == nil { + return nil + } + w := t.one("SELECT * FROM workspaces WHERE id=$1", row.S("workspaceId")) + require(w != nil && w.S("tenantId") == row.S("tenantId") && w.S("projectId") == row.S("projectId") && w.S("issueRunId") == row.S("runId"), 409, "residual_identity_conflict") + if e.S("kind") == "agent_session" || e.S("kind") == "deliver_revision" { + require(e.S("operationId") == row.S("runId"), 409, "residual_identity_conflict") + work := t.one("SELECT * FROM execution_work WHERE id=$1 AND run_id=$2", e.S("workId"), row.S("runId")) + require(work != nil && work.S("executionId") == e.S("executionId") && work.S("kind") == e.S("kind"), 409, "residual_identity_conflict") + target := work.O("target") + require(target.S("workspaceId") == w.S("id") && target.S("nodeId") == e.S("nodeId"), 409, "residual_identity_conflict") + sandbox := t.one("SELECT * FROM sandbox_instances WHERE id=$1 AND workspace_id=$2", target.S("sandboxInstanceId"), w.S("id")) + require(sandbox != nil && sandbox.N("generation") <= row.N("runtimeGeneration"), 409, "residual_identity_conflict") + } else { + require(t.one("SELECT id FROM operations WHERE id=$1 AND workspace_id=$2 AND tenant_id=$3 AND project_id=$4", e.S("operationId"), w.S("id"), row.S("tenantId"), row.S("projectId")) != nil, 409, "residual_identity_conflict") + } + return row +} + +func residualExecutionFact(e, result Object) (string, Object) { + kind, prefix := e.S("kind"), "execution:" + switch kind { + case "install_plugins", "remove_plugins": + kind, prefix = "plugin", "plugin:" + case "": + kind, prefix = "clone", "clone:" + } + state := executionState(Object{"result": result}) + return prefix + e.S("executionId"), Object{"kind": kind, "state": state, "result": result} +} + +func acceptResidualExecutionFact(t *transaction, e, result Object) bool { + row := residualExecutionIdentity(t, e) + if row == nil { + return true + } + key, incoming := residualExecutionFact(e, result) + if previous := e.O("result"); len(previous) > 0 { + _, original := residualExecutionFact(e, previous) + if !appendResidualFact(t, row.S("runId"), key, original) { + return false + } + } + return appendResidualFact(t, row.S("runId"), key, incoming) +} diff --git a/internal/core/agent_run_residual_loop.go b/internal/core/agent_run_residual_loop.go new file mode 100644 index 00000000..342c8749 --- /dev/null +++ b/internal/core/agent_run_residual_loop.go @@ -0,0 +1,55 @@ +package core + +import "context" + +// ReconcileAgentRunResidualsOnce scans the ledger, independently of the business Run's phase. +// Scheduling time has no evidentiary meaning. Old unresolved rows rotate behind rows not yet +// checked, so one bounded batch of unknowns cannot starve the remainder of the ledger. +func (s *Store) ReconcileAgentRunResidualsOnce(ctx context.Context) error { + rows, err := s.Pool.QueryContext(ctx, `SELECT run_id FROM agent_run_residuals + WHERE reclaim_state IN ('reconciling','reclaiming') + ORDER BY reclaim_checked_at NULLS FIRST,created_at,run_id LIMIT $1`, agentDeliveryBatchSize) + if err != nil { + return err + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return err + } + ids = append(ids, id) + } + err = rows.Err() + rows.Close() + if err != nil { + return err + } + for _, id := range ids { + if ctx.Err() != nil { + return ctx.Err() + } + // Claim the scheduling position separately: even an invalid historical ownership row + // whose reconciliation rolls back must yield its scan slot to the next bounded pass. + _, markErr := s.transact(ctx, func(t *transaction) Object { + t.exec("UPDATE agent_run_residuals SET reclaim_checked_at=clock_timestamp() WHERE run_id=$1 AND reclaim_state IN ('reconciling','reclaiming')", id) + return nil + }) + if markErr != nil { + continue + } + _, _ = s.transact(ctx, func(t *transaction) Object { + row := t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1 AND reclaim_state IN ('reconciling','reclaiming')", id) + if row == nil || row["reclaimConflictedAt"] != nil { + return nil + } + row = refreshResidualTargets(t, row) + if row.S("reclaimState") == "reconciling" { + return authorizeResidualOwnedDelete(t, id) + } + return resumeResidualReclamation(t, row) + }) + } + return nil +} diff --git a/internal/core/agent_run_residual_reclaim.go b/internal/core/agent_run_residual_reclaim.go new file mode 100644 index 00000000..3aad6640 --- /dev/null +++ b/internal/core/agent_run_residual_reclaim.go @@ -0,0 +1,207 @@ +package core + +// claimResidualReclamation is the single serialized arbitration point for automatic cleanup and +// a future already-authorized controlled intent. It is internal, not a caller-selectable public +// delete flag. Controlled callers must have authenticated/persisted their intent before entry; +// this step implements only its reservation, never a controlled action or effect. +func claimResidualReclamation(t *transaction, runID, mode string, authorization Object) Object { + require(mode == "owned_delete" || mode == "controlled", 400, "invalid_reclaim_mode") + row := t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", runID) + _, w, p := residualIdentity(t, row) + require(row["reclaimConflictedAt"] == nil, 409, "residual_fact_conflict") + if row.S("reclaimState") != "reconciling" { + require(row.S("reclaimMode") == mode, 409, "residual_mode_conflict") + return Object{"residual": row, "operation": residualBoundOperation(t, row), "claimed": false} + } + row = refreshResidualTargets(t, row) + require(!w.B("admissionOpen") && w["deletedAt"] == nil && p.S("lifecycle") == "active", 409, "resource_unavailable") + if t.one("SELECT id FROM operations WHERE project_id=$1 AND state IN ('queued','running','retry_wait','blocked')", p.S("id")) != nil { + return Object{"busy": true, "residual": row} + } + requireNoForceStop(t, w.S("id")) + if mode == "owned_delete" && !residualEligibility(t, row) { + return Object{"eligible": false, "residual": row} + } + if mode == "controlled" { + // No boolean or arbitrary nonempty object is an authority. The future authenticated entry + // passes the stable identity of its already accepted operator intent. This reservation by + // itself does not authorize a terminate, data-delete, new operation or final disposition. + require(authorization.S("kind") == "controlled_intent" && validID(authorization.S("intentId")) && authorization.S("source") != "", 409, "controlled_intent_required") + } + id := newID() + auth := Object{"kind": "residual_owned_delete", "runId": runID, "workspaceId": w.S("id"), "tenantId": row.S("tenantId"), "projectId": p.S("id"), "runtimeGeneration": row.N("runtimeGeneration"), "evidenceVersion": row.N("reclaimVersion")} + if mode == "controlled" { + auth["kind"], auth["intentId"], auth["source"] = "controlled_intent", authorization.S("intentId"), authorization.S("source") + } + auth["claimedAt"] = t.one("SELECT clock_timestamp()::text AS at").S("at") + var op Object + if mode == "owned_delete" { + snapshot := Object{} + for key, value := range w { + if key != "issueRunId" { + snapshot[key] = value + } + } + closeAdmission(t, w, "deleted") + request := Object{"previous": Object{w.S("id"): snapshot}, "reclaimId": id} + op = newOperation(t, &PublicRequest{TenantID: row.S("tenantId"), Key: "residual-" + id}, w.S("creatorUserId"), p.S("id"), w.S("id"), "delete_workspace", "quiesce", requestHash("residual-owned-delete", runID, Object{"reclaimId": id}), request) + reserveRuntimeMaintenance(t, w.S("id"), op.S("id")) + } + t.exec(`UPDATE agent_run_residuals SET reclaim_state='reclaiming',reclaim_id=$2,reclaim_mode=$3, + reclaim_operation_id=$4,reclaim_authorization=$5,reclaim_version=reclaim_version+1 WHERE run_id=$1 AND reclaim_state='reconciling'`, runID, id, mode, nullable(op.S("id")), jsonText(auth)) + return Object{"claimed": true, "residual": t.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", runID), "operation": op} +} + +func authorizeResidualOwnedDelete(t *transaction, runID string) Object { + return claimResidualReclamation(t, runID, "owned_delete", nil) +} + +func residualBoundOperation(t *transaction, row Object) Object { + if row.S("reclaimOperationId") == "" { + return nil + } + return t.one("SELECT * FROM operations WHERE id=$1", row.S("reclaimOperationId")) +} + +// residualOperation resolves a persisted binding, including a cleared row for idempotent fact +// reception. A request's reclaimId without the database binding can never obtain this authority. +func residualOperation(t *transaction, op Object) Object { + if op == nil { + return nil + } + return t.one("SELECT * FROM agent_run_residuals WHERE reclaim_operation_id=$1", op.S("id")) +} + +// checkResidualOperation is called before plan, permit and advance, even when returning an existing +// effect. Ordinary operations return nil only when their Workspace has no residual responsibility. +// A conflicting cleared record remains a durable block, rather than reopening ordinary cleanup. +func checkResidualOperation(t *transaction, op Object) Object { + row := residualOperation(t, op) + if row == nil { + if op.S("workspaceId") != "" { + other := residualForWorkspace(t, op.S("workspaceId")) + require(other == nil || (other.S("reclaimState") == "cleared" && other["reclaimConflictedAt"] == nil), 409, "residual_operation_required") + } + require(op.O("request").S("reclaimId") == "", 409, "residual_operation_required") + return nil + } + _, w, _ := residualIdentity(t, row) + require(row.S("reclaimState") == "reclaiming" && row.S("reclaimMode") == "owned_delete" && + row["reclaimConflictedAt"] == nil && row["clearedAt"] == nil && op.S("kind") == "delete_workspace" && + op.S("workspaceId") == w.S("id") && op.S("projectId") == row.S("projectId") && op.S("tenantId") == row.S("tenantId") && + op.O("request").S("reclaimId") == row.S("reclaimId") && !w.B("admissionOpen"), 409, "residual_operation_conflict") + row = refreshResidualTargets(t, row) + require(residualTargetsLocated(t, row), 409, "residual_target_unknown") + return row +} + +// checkResidualCleanupActivities exempts only this binding's own residual guard. All other activity +// guards remain. Unknown executions are physically closed only when every writable original +// sandbox/ensure identity is dual-confirmed, never by result=NULL or a ticket timeout. +func checkResidualCleanupActivities(t *transaction, w, op Object) { + row := checkResidualOperation(t, op) + if row == nil { + checkActivities(t, w) + return + } + require(w.S("id") == row.S("workspaceId") && w.S("kind") == "isolated", 409, "residual_identity_conflict") + require(t.one("SELECT run_id FROM agent_run_residuals WHERE workspace_id=$1 AND run_id<>$2 AND cleared_at IS NULL", w.S("id"), row.S("runId")) == nil, 409, "resource_in_use") + require(t.one("SELECT id FROM execution_tickets WHERE workspace_id=$1 AND state='active' AND terminated_by_force_stop_id IS NULL", w.S("id")) == nil, 409, "resource_in_use") + if !residualAllSandboxesTerminated(t, row) { + require(residualExecutionsTerminal(t, row), 409, "resource_in_use") + } + require(residualDeliveryComplete(t, row), 409, "residual_delivery_unknown") +} + +// residualDeletionReady is the additional per-effect/permit gate. Planning the one current +// sandbox's termination is allowed; cleanup waits for every original ID, including historical +// ensures which never produced a sandbox row. Additional unconfirmed IDs get no delete effects. +func residualDeletionReady(t *transaction, row Object) bool { + if row == nil || row["reclaimConflictedAt"] != nil || !residualTargetsLocated(t, row) || !residualDeliveryComplete(t, row) { + return false + } + for _, target := range residualObjects(row["reclaimTargets"]) { + if target.S("kind") == "sandbox" && !residualSandboxTerminated(t, row, target.S("id")) { + return false + } + } + return true +} + +// checkResidualEffect authorizes only the owned-delete topology and the original current sandbox. +// Use this in both planEffect and effectPermit; pass the actual persisted effect for a permit. +func checkResidualEffect(t *transaction, op Object, kind, workspaceID string, effect Object) { + row := checkResidualOperation(t, op) + if row == nil { + return + } + require(workspaceID == row.S("workspaceId"), 409, "residual_identity_conflict") + w := t.one("SELECT * FROM workspaces WHERE id=$1", workspaceID) + checkResidualCleanupActivities(t, w, op) + require(kind == "sandbox_terminate" || kind == "workspace_data_delete", 409, "residual_effect_forbidden") + if kind == "workspace_data_delete" { + require(residualDeletionReady(t, row), 409, "termination_unconfirmed") + return + } + sid := "" + if effect != nil { + sid = effect.O("request").S("sandboxInstanceId") + } + current := t.one("SELECT * FROM sandbox_instances WHERE workspace_id=$1 AND terminated_at IS NULL", workspaceID) + if sid == "" { + sid = current.S("id") + } + require(sid != "" && (current.S("id") == sid || residualSandboxTerminated(t, row, sid)), 409, "residual_target_unknown") + found := false + for _, target := range residualObjects(row["reclaimTargets"]) { + if target.S("kind") != "sandbox" { + continue + } + if target.S("id") == sid { + found = true + continue + } + require(residualSandboxTerminated(t, row, target.S("id")), 409, "residual_target_unknown") + } + require(found, 409, "residual_target_unknown") +} + +// completeResidualReclamation is called from cleanup->done in the SAME transaction that marks the +// operation succeeded and soft-deletes the Workspace. Failure rolls those writes back together. +func completeResidualReclamation(t *transaction, op Object) bool { + row := residualOperation(t, op) + if row == nil { + return true + } + if row.S("reclaimState") == "cleared" { + require(row["reclaimConflictedAt"] == nil && row.S("disposition") == "released", 409, "residual_fact_conflict") + return true + } + row = checkResidualOperation(t, op) + require(residualDeletionReady(t, row), 409, "residual_cleanup_incomplete") + effect := t.one(`SELECT * FROM external_effects WHERE operation_id=$1 AND workspace_id=$2 AND kind='workspace_data_delete' + AND state='succeeded' AND reconciled_epoch=$3`, op.S("id"), row.S("workspaceId"), op.N("controllerEpoch")) + require(effect != nil && effect.O("result").B("removed"), 409, "residual_cleanup_incomplete") + if !appendResidualFact(t, row.S("runId"), "reclaim:"+row.S("reclaimId")+":complete", Object{"kind": "reclaim_complete", "state": "succeeded", "reclaimId": row.S("reclaimId"), "operationId": op.S("id"), "effectId": effect.S("id"), "disposition": "released"}) { + return false + } + t.exec(`UPDATE agent_run_residuals SET reclaim_state='cleared',disposition='released',cleared_at=clock_timestamp(), + reclaim_version=reclaim_version+1 WHERE run_id=$1 AND reclaim_state='reclaiming' AND reclaim_conflicted_at IS NULL`, row.S("runId")) + return true +} + +// resumeResidualReclamation keeps the original authority and operation across process restarts. +// retry_wait is already claimable by the normal Controller path. A historically failed operation +// is never revived here, and neither a failed query nor quiesce refusal creates another operation. +func resumeResidualReclamation(t *transaction, row Object) Object { + if row.S("reclaimMode") != "owned_delete" || row["reclaimConflictedAt"] != nil { + return Object{"residual": row} + } + op := residualBoundOperation(t, row) + require(op != nil, 409, "residual_operation_missing") + checkResidualOperation(t, op) + if op.S("state") == "queued" || op.S("state") == "retry_wait" || op.S("state") == "blocked" || op.S("state") == "running" { + t.queued = append(t.queued, op.S("id")) + } + return Object{"residual": row, "operation": op} +} diff --git a/internal/core/agent_run_residual_reclaim_db_test.go b/internal/core/agent_run_residual_reclaim_db_test.go new file mode 100644 index 00000000..d348e81f --- /dev/null +++ b/internal/core/agent_run_residual_reclaim_db_test.go @@ -0,0 +1,491 @@ +package core + +import ( + "context" + "errors" + "strings" + "testing" +) + +type residualReclaimScene struct { + store *Store + abandonedCreation + node, delivery, work string + deliveryInput, deliveryResult Object +} + +func seedResidualExec(t *testing.T, s *Store, query string, args ...any) { + t.Helper() + if _, err := s.Pool.ExecContext(t.Context(), query, args...); err != nil { + t.Fatal(err) + } +} + +// The ref/history-verification outcome is an explicit fixture. The actual delivery applier, +// receipt, Revision insert and residual fact write run together below; this does not test Git or storage. +func seedResidualScene(t *testing.T, ready bool) residualReclaimScene { + t.Helper() + s := commandStore(t) + scene := residualReclaimScene{store: s, abandonedCreation: seedAbandonedCreation(t, s, true), node: newID(), delivery: "residual-delivery-" + newID(), work: newID()} + seedResidualExec(t, s, `INSERT INTO node_instances(id,workspace_id,sandbox_instance_id,service_subject,connection_state,protocol_version,initialized,node_id,node_incarnation_id) + VALUES($1,$2,$3,'residual-node','connected',1,true,'residual-node','residual-incarnation')`, scene.node, scene.workspace, scene.effect) + seedResidualExec(t, s, `UPDATE sandbox_instances SET observed_state='running' WHERE id=$1`, scene.effect) + if ready { + _, err := submitFact(t, s, scene.abandonedCreation, "residual-ensure", Object{"epoch": 1, "kind": "effect", "effectId": scene.effect, "state": "succeeded", "externalId": "residual-substrate", "result": Object{"sandboxInstanceId": scene.effect, "nodeId": "residual-node"}}) + if err != nil { + t.Fatal(err) + } + _, err = submitFact(t, s, scene.abandonedCreation, "residual-creation", Object{"epoch": 1, "kind": "creation", "outcome": "ready"}) + if err != nil { + t.Fatal(err) + } + } + scene.deliveryInput = Object{"kind": "deliver_revision", "baseCommit": baseCommit, "revisionRef": "refs/ora/revisions/" + scene.run, "deliveryRef": GitDeliveryRefPrefix + scene.work, "historyKey": "revisions/" + scene.run + "/history"} + scene.deliveryResult = Object{"outcome": "revision_unchanged", "node": Object{"nodeId": "residual-node", "nodeIncarnationId": "residual-incarnation"}, "baseCommit": baseCommit, "finalCommit": baseCommit, "revisionRef": scene.deliveryInput.S("revisionRef"), "history": Object{"key": scene.deliveryInput.S("historyKey"), "size": 0, "sha256": strings.Repeat("a", 64)}} + seedResidualExec(t, s, `INSERT INTO execution_work(id,run_id,kind,input,target,execution_id) VALUES($1,$2,'deliver_revision',$3,$4,$5)`, scene.work, scene.run, jsonText(scene.deliveryInput), jsonText(Object{"workspaceId": scene.workspace, "sandboxInstanceId": scene.effect, "runtimeGeneration": 1, "nodeId": "residual-node"}), scene.delivery) + seedResidualExec(t, s, `INSERT INTO node_executions(execution_id,kind,operation_id,work_id,workspace_id,node_id,node_operation_id,input,dispatched_epoch) + VALUES($1,'deliver_revision',$2,$3,$4,'residual-node',$1,$5,1)`, scene.delivery, scene.run, scene.work, scene.workspace, jsonText(scene.deliveryInput)) + if ready { + _, err := s.transact(t.Context(), func(tx *transaction) Object { return seedResidualDelivery(tx, scene, scene.deliveryResult) }) + if err != nil { + t.Fatal(err) + } + seedResidualRuntimeAck(t, scene, true) + } + return scene +} + +func seedResidualDelivery(tx *transaction, scene residualReclaimScene, result Object) Object { + return nodeExecutionResult(tx, &ControlRequest{Action: "clone_takeover", Body: Object{"operationId": scene.run, "executionId": scene.delivery, "sequence": 1, "event": "residual-delivery-terminal", "result": result}, verification: &revisionVerification{input: jsonText(scene.deliveryInput), valid: true}}, true) +} + +func seedResidualRuntimeAck(t *testing.T, scene residualReclaimScene, closed bool) { + t.Helper() + _, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + c := runtimeControl(tx, scene.workspace) + return runtimeControlCommand(tx, &ControlRequest{Action: "runtime_ack", Body: Object{"workspaceId": scene.workspace, "nodeInstanceId": scene.node, "controlEpoch": c.N("controlEpoch"), "controlVersion": c.N("version"), "inputClosed": closed, "unfinishedExecutionIds": []any{}}}) + }) + if err != nil { + t.Fatal(err) + } +} + +func seedResidualClaim(t *testing.T, scene residualReclaimScene) Object { + t.Helper() + if err := scene.store.ReconcileAgentRunResidualsOnce(t.Context()); err != nil { + t.Fatal(err) + } + out, err := scene.store.Control(t.Context(), &ControlRequest{Action: "claim", Body: Object{"epoch": 1}, Service: scene.claims}) + if err != nil { + t.Fatal(err) + } + op := out.O("operation") + if op.S("kind") != "delete_workspace" || op.O("request").S("reclaimId") == "" { + t.Fatalf("no owned residual delete claimed: %v", out) + } + return op +} + +func seedResidualAction(t *testing.T, scene residualReclaimScene, op Object, action string, body Object, effect string) Object { + t.Helper() + if body == nil { + body = Object{} + } + body["epoch"], body["version"] = int64(1), op.N("version") + out, err := scene.store.Control(t.Context(), &ControlRequest{Action: action, OperationID: op.S("id"), EffectID: effect, Body: body, Service: scene.claims}) + if err != nil { + t.Fatalf("residual %s at %s: %v", action, op.S("step"), err) + } + if out.S("id") != "" { + return Object{"operation": out} + } + return out +} + +// Drive the shipped quiesce -> terminate -> cleanup topology, leaving cleanup -> done to callers. +func seedResidualCleanup(t *testing.T, scene residualReclaimScene) Object { + t.Helper() + op, effect := seedResidualCleanupPlan(t, scene) + return seedResidualAction(t, scene, op, "effect_result", Object{"state": "succeeded", "externalId": "residual-cleanup", "result": Object{"removed": true}}, effect.S("id")).O("operation") +} + +func seedResidualCleanupPlan(t *testing.T, scene residualReclaimScene) (Object, Object) { + t.Helper() + op, termination := seedResidualTerminationPlan(t, scene) + op = seedResidualAction(t, scene, op, "effect_result", Object{"state": "succeeded", "externalId": "residual-termination", "result": Object{"terminated": true, "lateEnsureFenced": true}}, termination.S("id")).O("operation") + op = seedResidualAction(t, scene, op, "advance", nil, "").O("operation") + planned := seedResidualAction(t, scene, op, "plan", Object{"kind": "workspace_data_delete", "workspaceId": scene.workspace}, "") + return planned.O("operation"), planned.O("effect") +} + +func seedResidualTerminationPlan(t *testing.T, scene residualReclaimScene) (Object, Object) { + t.Helper() + op := seedResidualClaim(t, scene) + seedResidualRuntimeAck(t, scene, false) + _, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + n := tx.one("SELECT * FROM node_instances WHERE id=$1", scene.node) + w := tx.one("SELECT * FROM workspaces WHERE id=$1", scene.workspace) + return nodeIdle(tx, &ControlRequest{Body: Object{"version": n.N("version"), "operationId": op.S("id"), "admissionEpoch": w.N("admissionEpoch"), "idle": true}}, n, w) + }) + if err != nil { + t.Fatal(err) + } + op = seedResidualAction(t, scene, op, "advance", nil, "").O("operation") + planned := seedResidualAction(t, scene, op, "plan", Object{"kind": "sandbox_terminate", "workspaceId": scene.workspace}, "") + return planned.O("operation"), planned.O("effect") +} + +func TestResidualOwnedCleanupRetriesSameFailedEffectAcrossControllerEpoch(t *testing.T) { + for _, kind := range []string{"sandbox_terminate", "workspace_data_delete"} { + t.Run(kind, func(t *testing.T) { seedResidualRetryCleanupEffect(t, kind) }) + } +} + +func seedResidualRetryCleanupEffect(t *testing.T, kind string) { + t.Helper() + scene := seedResidualScene(t, true) + var op, cleanup Object + success := Object{"removed": true} + if kind == "sandbox_terminate" { + op, cleanup = seedResidualTerminationPlan(t, scene) + success = Object{"terminated": true, "lateEnsureFenced": true} + } else { + op, cleanup = seedResidualCleanupPlan(t, scene) + } + cleanupID := cleanup.S("id") + var reclaimID, terminationID string + if err := scene.store.Pool.QueryRow(`SELECT r.reclaim_id::text,e.id::text FROM agent_run_residuals r JOIN external_effects e ON e.operation_id=r.reclaim_operation_id AND e.kind='sandbox_terminate' WHERE r.run_id=$1`, scene.run).Scan(&reclaimID, &terminationID); err != nil { + t.Fatal(err) + } + failed := Object{"state": "failed", "externalId": "residual-cleanup-retry", "result": Object{"error": "temporary_storage_unavailable"}} + op = seedResidualAction(t, scene, op, "effect_result", failed, cleanupID).O("operation") + deferredState, deferredCode := "retry_wait", "node_unavailable" + if kind == "sandbox_terminate" { + // The existing Controller uses blocked for an unconfirmed termination. A residual + // responsibility must normalize that request to a claimable retry_wait, not strand + // the hidden Workspace behind an unavailable public/operator retry action. + deferredState, deferredCode = "blocked", "termination_unconfirmed" + } + op = seedResidualAction(t, scene, op, "defer", Object{"state": deferredState, "errorCode": deferredCode, "retrySeconds": 1}, "").O("operation") + if op.S("state") != "retry_wait" || op.S("errorCode") != deferredCode { + t.Fatalf("owned cleanup defer is not retryable: %v", op) + } + seedResidualExec(t, scene.store, `UPDATE operations SET retry_at=clock_timestamp()-interval '1 second' WHERE id=$1`, op.S("id")) + _, err := scene.store.Control(t.Context(), &ControlRequest{Action: "lease_release", Body: Object{"epoch": 1}, Service: scene.claims}) + if err != nil { + t.Fatal(err) + } + restarted := seedResidualRebuildStore(scene.store) + lease, err := restarted.Control(t.Context(), &ControlRequest{Action: "lease_acquire", Body: Object{}, Service: scene.claims}) + if err != nil { + t.Fatal(err) + } + epoch := lease.N("epoch") + if epoch != 2 { + t.Fatalf("replacement Controller epoch=%d, want 2", epoch) + } + if err := restarted.ReconcileAgentRunResidualsOnce(t.Context()); err != nil { + t.Fatal(err) + } + claimed, err := restarted.Control(t.Context(), &ControlRequest{Action: "claim", Body: Object{"epoch": epoch}, Service: scene.claims}) + if err != nil { + t.Fatal(err) + } + if current := claimed.O("operation"); current.S("id") != op.S("id") || current.N("controllerEpoch") != epoch { + t.Fatalf("retry replaced its owned operation: %v", claimed) + } else { + op = current + } + control := func(action, effect string, body Object) (Object, error) { + body["epoch"], body["version"] = epoch, op.N("version") + out, err := restarted.Control(t.Context(), &ControlRequest{Action: action, OperationID: op.S("id"), EffectID: effect, Body: body, Service: scene.claims}) + if err == nil { + if current := out.O("operation"); len(current) > 0 { + op = current + } else if out.S("id") != "" { + op = out + } + } + return out, err + } + // Reconcile each original effect under the replacement epoch before reusing its plan. These + // are fresh accepted observations, not mutation of the old effect or invention of a new one. + if kind == "workspace_data_delete" { + if _, err := control("effect_result", terminationID, Object{"state": "succeeded", "externalId": "residual-termination", "result": Object{"terminated": true, "lateEnsureFenced": true}}); err != nil { + t.Fatal(err) + } + } + if _, err := control("effect_result", cleanupID, Object{"state": "failed", "externalId": "residual-cleanup-retry", "result": Object{"error": "temporary_storage_unavailable"}}); err != nil { + t.Fatal(err) + } + planned, err := control("plan", "", Object{"kind": kind, "workspaceId": scene.workspace}) + if err != nil { + t.Fatal(err) + } + if planned.O("effect").S("id") != cleanupID { + t.Fatalf("retry substituted a new cleanup effect: %v", planned) + } + if _, err := control("effect_result", cleanupID, Object{"state": "succeeded", "externalId": "residual-cleanup-retry", "result": success}); err != nil { + t.Fatalf("same-effect failed-to-success retry was rejected: %v", err) + } + if kind == "sandbox_terminate" { + if _, err := control("advance", "", Object{}); err != nil { + t.Fatal(err) + } + dataPlan, err := control("plan", "", Object{"kind": "workspace_data_delete", "workspaceId": scene.workspace}) + if err != nil { + t.Fatal(err) + } + if _, err := control("effect_result", dataPlan.O("effect").S("id"), Object{"state": "succeeded", "externalId": "post-termination-cleanup", "result": Object{"removed": true}}); err != nil { + t.Fatal(err) + } + } + finished, err := control("advance", "", Object{}) + if err != nil { + t.Fatal(err) + } + if finished.S("state") != "succeeded" { + t.Fatalf("retry did not finish cleanup: %v", finished) + } + completedVersion := finished.N("version") + if completedVersion <= 1 { + t.Fatalf("cleanup did not advance its operation version: %v", finished) + } + lateResult := func(submission, message string) (Object, error) { + // The succeeded operation is no longer executable. Its original binding still accepts + // facts through Control with the current lease and a deliberately stale operation version. + return restarted.Control(t.Context(), &ControlRequest{ + Action: "effect_result", OperationID: op.S("id"), EffectID: cleanupID, SubmissionID: submission, + Body: Object{"epoch": epoch, "version": 1, "state": "failed", "externalId": "residual-cleanup-retry", "result": Object{"error": message}}, Service: scene.claims, + }) + } + // A previously accepted failed observation can replay after success, even after clearing. It + // must neither downgrade the effect nor manufacture a conflict or another evidence entry. + for attempt := 0; attempt < 2; attempt++ { + replayed, err := lateResult("cleared-prior-failure-"+kind, "temporary_storage_unavailable") + if err != nil { + t.Fatalf("consistent historical failed observation did not replay through Control: %v", err) + } + if replayed.O("operation").N("version") != completedVersion || replayed.O("effect").S("state") != "succeeded" { + t.Fatalf("cleared fact replay changed canonical state/version: %v", replayed) + } + } + seedResidualAssert(t, scene, `SELECT r.reclaim_id=$2::uuid AND r.reclaim_operation_id=$3::uuid AND r.reclaim_state='cleared' + AND r.disposition='released' AND r.reclaim_conflicted_at IS NULL AND r.responsibility=$5::jsonb + AND e.state='succeeded' AND e.reconciled_epoch=$6 AND e.result=$7::jsonb + AND (SELECT version=$8 AND state='succeeded' FROM operations WHERE id=r.reclaim_operation_id) + AND (SELECT count(*) FROM external_effects WHERE operation_id=r.reclaim_operation_id)=2 + AND (SELECT count(*) FROM operations WHERE workspace_id=r.workspace_id AND kind='delete_workspace')=1 + AND (SELECT string_agg(entry->'fact'->>'state',',' ORDER BY ordinal) FROM jsonb_array_elements(r.reclaim_evidence) WITH ORDINALITY facts(entry,ordinal) + WHERE entry->>'key'='effect:'||$4::text)='failed,succeeded' + FROM agent_run_residuals r JOIN external_effects e ON e.id=$4::uuid WHERE r.run_id=$1`, scene.run, reclaimID, op.S("id"), cleanupID, scene.responsibility, epoch, jsonText(success), completedVersion) + // A changed post-success conclusion remains an A4 contradiction, durably audited even after + // the business/physical cleanup completed. It cannot rewrite the accepted success. + for attempt := 0; attempt < 2; attempt++ { + _, err = lateResult("cleared-conflicting-failure-"+kind, "different_post_success_failure") + assertResidualFactConflict(t, err) + } + seedResidualAssert(t, scene, `SELECT r.reclaim_state='cleared' AND r.reclaim_conflicted_at IS NOT NULL + AND r.disposition='released' AND r.cleared_at IS NOT NULL + AND (SELECT version=$4 AND state='succeeded' FROM operations WHERE id=r.reclaim_operation_id) + AND e.state='succeeded' AND e.result=$3::jsonb + AND (SELECT count(*) FROM jsonb_array_elements(r.reclaim_evidence) entry WHERE entry->>'key'='effect:'||$2::text + AND entry->'fact'->>'kind'='conflict' AND entry->'fact'->'original'->>'state'='succeeded' + AND entry->'fact'->'incoming'->'result'->>'error'='different_post_success_failure')=1 + FROM agent_run_residuals r JOIN external_effects e ON e.id=$2::uuid WHERE r.run_id=$1`, scene.run, cleanupID, jsonText(success), completedVersion) +} + +func seedResidualAssert(t *testing.T, scene residualReclaimScene, query string, args ...any) { + t.Helper() + var ok bool + if err := scene.store.Pool.QueryRowContext(t.Context(), query, args...).Scan(&ok); err != nil { + t.Fatal(err) + } + if !ok { + t.Fatalf("residual invariant failed: %s", query) + } +} + +func TestResidualOwnedDeleteRunsCloudTopologyAndPreservesBusinessTerminal(t *testing.T) { + scene := seedResidualScene(t, true) + op := seedResidualCleanup(t, scene) + out := seedResidualAction(t, scene, op, "advance", nil, "") + if out.O("operation").S("state") != "succeeded" { + t.Fatalf("cleanup did not finish: %v", out) + } + seedResidualAssert(t, scene, `SELECT r.reclaim_state='cleared' AND r.disposition='released' AND r.cleared_at IS NOT NULL + AND r.responsibility=$2::jsonb AND r.reason='workspace_unavailable' AND w.deleted_at IS NOT NULL AND NOT w.admission_open + AND i.phase='done' AND i.status='failed' AND i.failure_reason='workspace_unavailable' + AND (SELECT count(*) FROM operations WHERE workspace_id=w.id AND kind='delete_workspace')=1 + AND (SELECT count(*) FROM external_effects WHERE operation_id=r.reclaim_operation_id AND kind IN ('sandbox_terminate','workspace_data_delete') AND state='succeeded')=2 + FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id JOIN issue_runs i ON i.id=r.run_id WHERE r.run_id=$1`, scene.run, scene.responsibility) +} + +func TestResidualUnknownAndDeliveryFailureNeverAuthorizeDataDeletion(t *testing.T) { + for _, delivery := range []string{"unknown", "failed", "skipped", "timeout"} { + t.Run(delivery, func(t *testing.T) { + scene := seedResidualScene(t, false) + if delivery != "unknown" { + _, err := submitFact(t, scene.store, scene.abandonedCreation, "negative-ready-ensure", Object{"epoch": 1, "kind": "effect", "effectId": scene.effect, "state": "succeeded", "externalId": "negative-substrate", "result": Object{"sandboxInstanceId": scene.effect, "nodeId": "residual-node"}}) + if err != nil { + t.Fatal(err) + } + _, err = scene.store.transact(t.Context(), func(tx *transaction) Object { + return seedResidualDelivery(tx, scene, Object{"outcome": "revision_failed", "reason": "upload_failed", "node": Object{"nodeId": "residual-node", "nodeIncarnationId": "residual-incarnation"}}) + }) + if err != nil { + t.Fatal(err) + } + seedResidualRuntimeAck(t, scene, true) + seedResidualExec(t, scene.store, `UPDATE issue_runs SET result=jsonb_set(COALESCE(result,'{}'),'{deliveryState}',to_jsonb($2::text)) WHERE id=$1`, scene.run, map[string]string{"failed": "failed", "skipped": "skipped", "timeout": "failed"}[delivery]) + } + if delivery == "timeout" { + seedResidualExec(t, scene.store, `UPDATE issue_runs SET workspace_deadline_at=clock_timestamp()-interval '10 days' WHERE id=$1`, scene.run) + } + if err := scene.store.ReconcileAgentRunResidualsOnce(t.Context()); err != nil { + t.Fatal(err) + } + seedResidualAssert(t, scene, `SELECT reclaim_state='reconciling' AND reclaim_id IS NULL AND cleared_at IS NULL + AND responsibility=$2::jsonb AND NOT EXISTS(SELECT 1 FROM operations WHERE workspace_id=$3 AND kind='delete_workspace') + AND NOT EXISTS(SELECT 1 FROM external_effects WHERE workspace_id=$3 AND kind IN ('sandbox_terminate','workspace_data_delete')) + FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.responsibility, scene.workspace) + }) + } +} + +func TestResidualPartialAndNullScopeTargetsRemainAUnion(t *testing.T) { + scene := seedResidualScene(t, true) + extra, missing := newID(), newID() + seedResidualExec(t, scene.store, `INSERT INTO external_effects(id,operation_id,project_id,workspace_id,kind,state,request,reconciled_epoch) + VALUES($1,$2,$3,NULL,'sandbox_ensure','failed','{}',1)`, extra, scene.operation, scene.project) + seedResidualExec(t, scene.store, `UPDATE agent_run_residuals SET reclaim_targets=$2 WHERE run_id=$1`, scene.run, jsonText([]Object{{"kind": "sandbox", "id": missing, "workspaceId": scene.workspace}})) + if err := scene.store.ReconcileAgentRunResidualsOnce(t.Context()); err != nil { + t.Fatal(err) + } + seedResidualAssert(t, scene, `SELECT reclaim_state='reconciling' AND reclaim_id IS NULL + AND reclaim_targets @> jsonb_build_array(jsonb_build_object('kind','sandbox','id',$2::text)) + AND reclaim_targets @> jsonb_build_array(jsonb_build_object('kind','effect','id',$3::text)) + AND reclaim_targets @> jsonb_build_array(jsonb_build_object('kind','sandbox','id',$3::text)) + AND responsibility=$4::jsonb FROM agent_run_residuals WHERE run_id=$1`, scene.run, missing, extra, scene.responsibility) +} + +func TestResidualCleanupRequiresEveryDualTerminationAndRemovedProof(t *testing.T) { + for _, proof := range []string{"terminated_only", "wrong_sandbox", "not_removed", "historical_target"} { + t.Run(proof, func(t *testing.T) { + scene := seedResidualScene(t, true) + op := seedResidualCleanup(t, scene) + switch proof { + case "terminated_only": + seedResidualExec(t, scene.store, `UPDATE external_effects SET result='{"terminated":true}' WHERE operation_id=$1 AND kind='sandbox_terminate'`, op.S("id")) + case "wrong_sandbox": + seedResidualExec(t, scene.store, `UPDATE external_effects SET request=jsonb_set(request,'{sandboxInstanceId}',to_jsonb($2::text)) WHERE operation_id=$1 AND kind='sandbox_terminate'`, op.S("id"), newID()) + case "not_removed": + seedResidualExec(t, scene.store, `UPDATE external_effects SET result='{"removed":false}' WHERE operation_id=$1 AND kind='workspace_data_delete'`, op.S("id")) + case "historical_target": + seedResidualExec(t, scene.store, `UPDATE agent_run_residuals SET reclaim_targets=reclaim_targets||$2::jsonb WHERE run_id=$1`, scene.run, jsonText([]Object{{"kind": "sandbox", "id": newID(), "workspaceId": scene.workspace}})) + } + _, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + return Object{"complete": completeResidualReclamation(tx, tx.one("SELECT * FROM operations WHERE id=$1", op.S("id")))} + }) + if err == nil { + t.Fatal("incomplete cleanup proof cleared the residual") + } + seedResidualAssert(t, scene, `SELECT r.cleared_at IS NULL AND r.reclaim_state='reclaiming' AND w.deleted_at IS NULL FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id WHERE r.run_id=$1`, scene.run) + }) + } +} + +func TestResidualClaimRollbackAndRestartKeepOneBoundOperation(t *testing.T) { + scene := seedResidualScene(t, true) + _, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + authorizeResidualOwnedDelete(tx, scene.run) + reject(409, "fixture_rollback") + return nil + }) + var fault *Fault + if !errors.As(err, &fault) || fault.Code != "fixture_rollback" { + t.Fatalf("rollback boundary: %v", err) + } + seedResidualAssert(t, scene, `SELECT reclaim_state='reconciling' AND reclaim_id IS NULL AND NOT EXISTS(SELECT 1 FROM operations WHERE workspace_id=$2 AND kind='delete_workspace') FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.workspace) + op := seedResidualClaim(t, scene) + seedResidualExec(t, scene.store, `UPDATE operations SET state='retry_wait',retry_at=clock_timestamp()-interval '1 second',error_code='node_unavailable' WHERE id=$1`, op.S("id")) + var authority, targets string + if err := scene.store.Pool.QueryRow(`SELECT reclaim_authorization::text,reclaim_targets::text FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&authority, &targets); err != nil { + t.Fatal(err) + } + // A new process-owned Store value shares the persisted database, not any in-memory claim state. + restarted := seedResidualRebuildStore(scene.store) + if err := restarted.ReconcileAgentRunResidualsOnce(context.Background()); err != nil { + t.Fatal(err) + } + _, err = restarted.transact(t.Context(), func(tx *transaction) Object { return authorizeResidualOwnedDelete(tx, scene.run) }) + if err != nil { + t.Fatal(err) + } + // Run Workspaces are intentionally hidden from public retryOperation. The server resumes + // their due retry_wait operation through the Controller's existing claim path instead. + reclaimed, err := restarted.Control(t.Context(), &ControlRequest{Action: "claim", Body: Object{"epoch": 1}, Service: scene.claims}) + if err != nil { + t.Fatal(err) + } + if retried := reclaimed.O("operation"); retried.S("id") != op.S("id") || retried.S("state") != "running" { + t.Fatalf("restart did not retry the original bound operation: %v", reclaimed) + } + seedResidualAssert(t, scene, `SELECT reclaim_operation_id=$2 AND reclaim_authorization=$3::jsonb AND reclaim_targets=$4::jsonb + AND (SELECT count(*) FROM operations WHERE workspace_id=$5 AND kind='delete_workspace')=1 FROM agent_run_residuals WHERE run_id=$1`, scene.run, op.S("id"), authority, targets, scene.workspace) +} + +func seedResidualRebuildStore(original *Store) *Store { + restarted := &Store{Pool: original.Pool, Signals: NewControlHub()} + BindBusinessHooks(restarted) + return restarted +} + +func TestResidualCompletionRollbackKeepsLedgerAndWorkspaceAtomic(t *testing.T) { + scene := seedResidualScene(t, true) + op := seedResidualCleanup(t, scene) + _, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + advance(tx, &ControlRequest{}, tx.one("SELECT * FROM operations WHERE id=$1", op.S("id"))) + reject(409, "fixture_completion_rollback") + return nil + }) + var fault *Fault + if !errors.As(err, &fault) || fault.Code != "fixture_completion_rollback" { + t.Fatalf("completion rollback boundary: %v", err) + } + seedResidualAssert(t, scene, `SELECT r.reclaim_state='reclaiming' AND r.cleared_at IS NULL AND r.disposition IS NULL + AND w.deleted_at IS NULL AND o.state='running' AND o.step='cleanup' + AND NOT EXISTS(SELECT 1 FROM jsonb_array_elements(r.reclaim_evidence) e WHERE e->'fact'->>'kind'='reclaim_complete') + FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id JOIN operations o ON o.id=r.reclaim_operation_id WHERE r.run_id=$1`, scene.run) +} + +func TestResidualOrdinaryLifecycleGuardsAndUnrelatedD8StayProtected(t *testing.T) { + scene := seedResidualScene(t, true) + if _, err := scene.store.DeleteRunWorkspace(t.Context(), scene.run); err == nil { + t.Fatal("ordinary run delete bypassed the residual") + } + for _, kind := range []string{"stop", "delete_workspace", "delete_project"} { + _, err := scene.store.transact(t.Context(), func(tx *transaction) Object { + w := tx.one("SELECT * FROM workspaces WHERE id=$1", scene.workspace) + if kind == "delete_project" { + for _, item := range tx.list("SELECT * FROM workspaces WHERE project_id=$1", scene.project) { + checkActivities(tx, item) + } + } else { + checkActivities(tx, w) + } + return nil + }) + var fault *Fault + if !errors.As(err, &fault) || fault.Code != "resource_in_use" { + t.Fatalf("%s guard bypass: %v", kind, err) + } + } + // D8 remains on its existing releasing-only scan: this done residual never enters that path. + if err := scene.store.GiveUpStaleWorkspaceReleasesOnce(t.Context()); err != nil { + t.Fatal(err) + } + seedResidualAssert(t, scene, `SELECT i.phase='done' AND i.status='failed' AND i.failure_reason='workspace_unavailable' + AND r.reason='workspace_unavailable' AND r.responsibility=$2::jsonb AND r.cleared_at IS NULL + FROM issue_runs i JOIN agent_run_residuals r ON r.run_id=i.id WHERE i.id=$1`, scene.run, scene.responsibility) +} diff --git a/internal/core/agent_run_residual_reclaim_race_db_test.go b/internal/core/agent_run_residual_reclaim_race_db_test.go new file mode 100644 index 00000000..1cce4f9b --- /dev/null +++ b/internal/core/agent_run_residual_reclaim_race_db_test.go @@ -0,0 +1,392 @@ +package core + +import ( + "context" + "database/sql" + "fmt" + "testing" + "time" +) + +type residualRaceResult struct { + out, observed Object + err error +} + +// Wrap the two real, dedicated-connection transactions with the production transaction context. +// This deliberately mirrors Store.transact's hooks: the callbacks below call the same production +// state functions while the advisory lock is held, never substitutes built only for the race. +func seedResidualTransaction(ctx context.Context, s *Store, raw *sql.Tx) *transaction { + tx := &transaction{tx: raw, ctx: ctx, store: s, pluginExecution: s.PluginExecution, directory: s.Directory, contextBuilder: s.Context, forms: s.Forms, assist: s.Assist, simulator: s.Simulator, objectStore: s.ObjectStore, onThreadEvents: s.OnThreadEvents, onSessionEnded: s.OnSessionEnded, onRunWorkspaceSettled: s.OnRunWorkspaceSettled, onRunWorkspaceDeleted: s.OnRunWorkspaceDeleted, onDeliverySettled: s.OnDeliverySettled} + tx.ctx = context.WithValue(ctx, businessTransactionKey{}, tx) + return tx +} + +func seedResidualRaceApply(tx *transaction, run string, fn func(*transaction) Object) (result residualRaceResult) { + defer func() { + if recovered := recover(); recovered != nil { + switch value := recovered.(type) { + case *Fault: + result.err = value + case databaseFailure: + result.err = value.err + default: + panic(recovered) + } + } + }() + // This statement happens after lock acquisition. A default READ COMMITTED transaction must + // observe A's commit here, even though B began and tried the lock before that commit. + result.observed = tx.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", run) + result.out = fn(tx) + return result +} + +// A executes while holding the production lock and stays uncommitted. B proves contention with +// pg_try_advisory_xact_lock=false on its SAME transaction, signals the channel, then blocks on the +// lock. Only then may A commit/roll back; B subsequently re-reads and calls its production function. +// No sleep, external probe connection or elapsed-time inference substitutes for that ordering. +func seedResidualRace(t *testing.T, scene residualReclaimScene, first, second func(*transaction) Object) (residualRaceResult, residualRaceResult) { + return seedResidualRaceStores(t, scene, scene.store, scene.store, first, second) +} + +func seedResidualRaceStores(t *testing.T, scene residualReclaimScene, storeA, storeB *Store, first, second func(*transaction) Object) (residualRaceResult, residualRaceResult) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + a, err := scene.store.Pool.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer a.Close() + b, err := scene.store.Pool.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer b.Close() + rawA, err := a.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer rawA.Rollback() + rawB, err := b.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + defer rawB.Rollback() + if _, err = rawA.ExecContext(ctx, "SELECT pg_advisory_xact_lock(67420911)"); err != nil { + t.Fatal(err) + } + txA, txB := seedResidualTransaction(ctx, storeA, rawA), seedResidualTransaction(ctx, storeB, rawB) + resultA := seedResidualRaceApply(txA, scene.run, first) + ready, finished := make(chan error, 1), make(chan residualRaceResult, 1) + go func() { + var acquired bool + if err := rawB.QueryRowContext(ctx, "SELECT pg_try_advisory_xact_lock(67420911)").Scan(&acquired); err != nil { + ready <- err + return + } + if acquired { + ready <- fmt.Errorf("B acquired A's still-uncommitted lock") + return + } + ready <- nil + if _, err := rawB.ExecContext(ctx, "SELECT pg_advisory_xact_lock(67420911)"); err != nil { + finished <- residualRaceResult{err: err} + return + } + result := seedResidualRaceApply(txB, scene.run, second) + if result.err != nil { + _ = rawB.Rollback() + } else { + result.err = rawB.Commit() + if result.err == nil && txB.residualFactConflict { + result.err = &Fault{Status: 409, Code: "residual_fact_conflict"} + } + } + finished <- result + }() + select { + case err := <-ready: + if err != nil { + t.Fatal(err) + } + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + if resultA.err != nil { + _ = rawA.Rollback() + } else { + resultA.err = rawA.Commit() + if resultA.err == nil && txA.residualFactConflict { + resultA.err = &Fault{Status: 409, Code: "residual_fact_conflict"} + } + } + var resultB residualRaceResult + select { + case resultB = <-finished: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + return resultA, resultB +} + +// A delayed dispatch record is a staged precondition, not a new runtime invocation. Its real +// terminal is taken over by nodeExecutionResult. Session completion still leaves its independent +// deliver_revision target without verified storage evidence, which must block data deletion. +func seedResidualLateExecution(tx *transaction, scene residualReclaimScene, kind string) Object { + work, execution := newID(), "late-"+kind+"-"+newID() + target := Object{"workspaceId": scene.workspace, "sandboxInstanceId": scene.effect, "runtimeGeneration": 1, "nodeId": "residual-node"} + tx.exec(`INSERT INTO execution_work(id,run_id,kind,input,target,execution_id) VALUES($1,$2,$3,'{}',$4,$5)`, work, scene.run, kind, jsonText(target), execution) + tx.exec(`INSERT INTO node_executions(execution_id,kind,operation_id,work_id,workspace_id,node_id,node_operation_id,input,dispatched_epoch) + VALUES($1,$2,$3,$4,$5,'residual-node',$1,'{}',1)`, execution, kind, scene.run, work, scene.workspace) + result := Object{"outcome": "revision_failed", "reason": "upload_failed", "node": Object{"nodeId": "residual-node", "nodeIncarnationId": "residual-incarnation"}} + if kind == "agent_session" { + result["outcome"], result["reason"] = "agent_session_ended", "agent_failed" + out := nodeExecutionResult(tx, &ControlRequest{Body: Object{"operationId": scene.run, "executionId": execution, "sequence": 1, "event": "late-original-terminal", "result": result}}, true) + pendingWork, pendingExecution := newID(), "pending-rescue-"+newID() + tx.exec(`INSERT INTO execution_work(id,run_id,kind,input,target,execution_id) VALUES($1,$2,'deliver_revision','{}',$3,$4)`, pendingWork, scene.run, jsonText(target), pendingExecution) + tx.exec(`INSERT INTO node_executions(execution_id,kind,operation_id,work_id,workspace_id,node_id,node_operation_id,input,dispatched_epoch) + VALUES($1,'deliver_revision',$2,$3,$4,'residual-node',$1,'{}',1)`, pendingExecution, scene.run, pendingWork, scene.workspace) + return out + } + return nodeExecutionResult(tx, &ControlRequest{Body: Object{"operationId": scene.run, "executionId": execution, "sequence": 1, "event": "late-original-terminal", "result": result}}, true) +} + +func TestResidualRaceCompletionVersusNewUnclosedExecutionTargets(t *testing.T) { + for _, kind := range []string{"agent_session", "deliver_revision"} { + for _, factFirst := range []bool{false, true} { + t.Run(fmt.Sprint(kind, "/fact_first=", factFirst), func(t *testing.T) { + scene := seedResidualScene(t, true) + op := seedResidualCleanup(t, scene) + complete := func(tx *transaction) Object { + return advance(tx, &ControlRequest{}, tx.one("SELECT * FROM operations WHERE id=$1", op.S("id"))) + } + fact := func(tx *transaction) Object { return seedResidualLateExecution(tx, scene, kind) } + first, second := complete, fact + if factFirst { + first, second = fact, complete + } + a, b := seedResidualRace(t, scene, first, second) + if factFirst { + if a.err != nil { + t.Fatal(a.err) + } + if b.err == nil { + t.Fatal("completion ignored the newly committed unclosed delivery target") + } + if b.observed.S("reclaimState") != "reclaiming" { + t.Fatal("completion did not reread the retained responsibility") + } + } else if a.err != nil || b.err != nil { + t.Fatalf("completed responsibility did not retain late original facts: %v / %v", a.err, b.err) + } + seedResidualAssert(t, scene, `SELECT (r.cleared_at IS NOT NULL)=$2 AND (w.deleted_at IS NOT NULL)=$2 + AND r.responsibility=$3::jsonb AND i.phase='done' AND i.status='failed' + AND (SELECT count(*) FROM node_executions WHERE operation_id=r.run_id AND kind=$4 AND result IS NOT NULL)>=1 + AND (SELECT count(*) FROM revisions WHERE run_id=r.run_id)=1 + AND (SELECT count(*) FROM node_executions e WHERE e.workspace_id=r.workspace_id AND e.kind='deliver_revision' + AND NOT EXISTS(SELECT 1 FROM revision_verifications v WHERE v.execution_id=e.execution_id AND v.outcome IN ('delivered','unchanged')))>=1 + AND (SELECT count(*) FROM operations WHERE workspace_id=r.workspace_id AND kind='delete_workspace')=1 + FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id JOIN issue_runs i ON i.id=r.run_id WHERE r.run_id=$1`, scene.run, !factFirst, scene.responsibility, kind) + }) + } + } +} + +func TestResidualRaceRestartRebuildsTwoStoresWithoutNewResponsibility(t *testing.T) { + for _, resumeFirst := range []bool{false, true} { + t.Run(fmt.Sprint("resume_first=", resumeFirst), func(t *testing.T) { + scene := seedResidualScene(t, true) + op := seedResidualCleanup(t, scene) + var reclaim, authority, targets string + if err := scene.store.Pool.QueryRow(`SELECT reclaim_id::text,reclaim_authorization::text,reclaim_targets::text FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&reclaim, &authority, &targets); err != nil { + t.Fatal(err) + } + storeA, storeB := seedResidualRebuildStore(scene.store), seedResidualRebuildStore(scene.store) + resume := func(tx *transaction) Object { + return resumeResidualReclamation(tx, tx.one("SELECT * FROM agent_run_residuals WHERE run_id=$1", scene.run)) + } + reclaimAgain := func(tx *transaction) Object { return authorizeResidualOwnedDelete(tx, scene.run) } + first, second := reclaimAgain, resume + if resumeFirst { + first, second = resume, reclaimAgain + } + a, b := seedResidualRaceStores(t, scene, storeA, storeB, first, second) + if a.err != nil || b.err != nil { + t.Fatalf("restart contenders failed: %v / %v", a.err, b.err) + } + if a.out.O("operation").S("id") != op.S("id") || b.out.O("operation").S("id") != op.S("id") { + t.Fatal("a rebuilt Store substituted a second operation") + } + seedResidualAssert(t, scene, `SELECT reclaim_id=$2 AND reclaim_operation_id=$3 AND reclaim_authorization=$4::jsonb + AND reclaim_targets=$5::jsonb AND responsibility=$6::jsonb AND reclaim_state='reclaiming' AND cleared_at IS NULL + AND (SELECT count(*) FROM operations WHERE workspace_id=$7 AND kind='delete_workspace')=1 + AND (SELECT count(*) FROM external_effects WHERE operation_id=$3)=2 + FROM agent_run_residuals WHERE run_id=$1`, scene.run, reclaim, op.S("id"), authority, targets, scene.responsibility, scene.workspace) + }) + } +} + +func TestResidualRaceAuthorizationVersusLateFactsBothCommitOrders(t *testing.T) { + for _, factFirst := range []bool{false, true} { + t.Run(fmt.Sprint("fact_first=", factFirst), func(t *testing.T) { + scene := seedResidualScene(t, true) + authorize := func(tx *transaction) Object { return authorizeResidualOwnedDelete(tx, scene.run) } + fact := func(tx *transaction) Object { + return provisioningCreationFact(tx, &ControlRequest{OperationID: scene.operation, Body: Object{"outcome": "failed"}}, scene.run) + } + first, second := authorize, fact + if factFirst { + first, second = fact, authorize + } + a, b := seedResidualRace(t, scene, first, second) + if factFirst { + assertResidualFactConflict(t, a.err) + assertResidualFactConflict(t, b.err) + if b.observed["reclaimConflictedAt"] == nil { + t.Fatal("authorization did not reread the committed conflict") + } + } else { + if a.err != nil { + t.Fatal(a.err) + } + assertResidualFactConflict(t, b.err) + if b.observed.S("reclaimMode") != "owned_delete" { + t.Fatal("late fact did not observe the committed authorization") + } + } + seedResidualAssert(t, scene, `SELECT reclaim_conflicted_at IS NOT NULL AND responsibility=$2::jsonb AND cleared_at IS NULL + AND (SELECT count(*) FROM operations WHERE workspace_id=$3 AND kind='delete_workspace')=$4 + AND NOT EXISTS(SELECT 1 FROM external_effects WHERE workspace_id=$3 AND kind='workspace_data_delete') + FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.responsibility, scene.workspace, map[bool]int{false: 1, true: 0}[factFirst]) + }) + } +} + +func TestResidualRaceAutomaticVersusControlledClaimBothCommitOrders(t *testing.T) { + for _, controlledFirst := range []bool{false, true} { + t.Run(fmt.Sprint("controlled_first=", controlledFirst), func(t *testing.T) { + scene := seedResidualScene(t, true) + auto := func(tx *transaction) Object { return authorizeResidualOwnedDelete(tx, scene.run) } + intent := newID() + controlled := func(tx *transaction) Object { + return claimResidualReclamation(tx, scene.run, "controlled", Object{"kind": "controlled_intent", "intentId": intent, "source": "authorized-test-intent"}) + } + first, second := auto, controlled + mode := "owned_delete" + count := 1 + if controlledFirst { + first, second = controlled, auto + mode = "controlled" + count = 0 + } + a, b := seedResidualRace(t, scene, first, second) + if a.err != nil { + t.Fatal(a.err) + } + fault, ok := b.err.(*Fault) + if !ok || fault.Status != 409 || fault.Code != "residual_mode_conflict" { + t.Fatalf("losing claimant did not respect the committed mode: %v", b.err) + } + if b.observed.S("reclaimId") != a.out.O("residual").S("reclaimId") { + t.Fatal("B did not reread A's durable responsibility") + } + seedResidualAssert(t, scene, `SELECT reclaim_state='reclaiming' AND reclaim_mode=$2 AND reclaim_id IS NOT NULL + AND responsibility=$3::jsonb AND cleared_at IS NULL + AND (SELECT count(*) FROM operations WHERE workspace_id=$4 AND kind='delete_workspace')=$5 + AND NOT EXISTS(SELECT 1 FROM external_effects WHERE workspace_id=$4 AND kind IN ('sandbox_terminate','workspace_data_delete')) + FROM agent_run_residuals WHERE run_id=$1`, scene.run, mode, scene.responsibility, scene.workspace, count) + }) + } +} + +func TestResidualRaceCompletionVersusLateDeliveryBothCommitOrders(t *testing.T) { + for _, factFirst := range []bool{false, true} { + t.Run(fmt.Sprint("fact_first=", factFirst), func(t *testing.T) { + scene := seedResidualScene(t, true) + op := seedResidualCleanup(t, scene) + complete := func(tx *transaction) Object { + return advance(tx, &ControlRequest{}, tx.one("SELECT * FROM operations WHERE id=$1", op.S("id"))) + } + late := func(tx *transaction) Object { + return seedResidualDelivery(tx, scene, Object{"outcome": "revision_failed", "reason": "upload_failed", "node": Object{"nodeId": "residual-node", "nodeIncarnationId": "residual-incarnation"}}) + } + first, second := complete, late + if factFirst { + first, second = late, complete + } + a, b := seedResidualRace(t, scene, first, second) + if factFirst { + assertResidualFactConflict(t, a.err) + if b.err == nil || b.observed["reclaimConflictedAt"] == nil { + t.Fatalf("completion ignored committed delivery conflict: %v", b.err) + } + } else { + if a.err != nil { + t.Fatal(a.err) + } + assertResidualFactConflict(t, b.err) + if b.observed.S("reclaimState") != "cleared" { + t.Fatal("late delivery did not reread cleared state") + } + } + seedResidualAssert(t, scene, `SELECT r.reclaim_conflicted_at IS NOT NULL AND r.responsibility=$2::jsonb + AND (r.cleared_at IS NOT NULL)=$3 AND (w.deleted_at IS NOT NULL)=$3 + AND e.result=$4::jsonb AND (SELECT count(*) FROM revisions WHERE run_id=r.run_id)=1 + AND (SELECT count(*) FROM node_event_receipts WHERE execution_id=$5)=1 + FROM agent_run_residuals r JOIN workspaces w ON w.id=r.workspace_id JOIN node_executions e ON e.execution_id=$5 WHERE r.run_id=$1`, scene.run, scene.responsibility, !factFirst, jsonText(scene.deliveryResult), scene.delivery) + }) + } +} + +func TestResidualRaceClearedReplayVersusLateFactBothCommitOrders(t *testing.T) { + for _, conflict := range []bool{false, true} { + for _, factFirst := range []bool{false, true} { + t.Run(fmt.Sprint("conflict=", conflict, "/fact_first=", factFirst), func(t *testing.T) { + scene := seedResidualScene(t, true) + op := seedResidualCleanup(t, scene) + seedResidualAction(t, scene, op, "advance", nil, "") + var before int + if err := scene.store.Pool.QueryRow(`SELECT jsonb_array_length(reclaim_evidence) FROM agent_run_residuals WHERE run_id=$1`, scene.run).Scan(&before); err != nil { + t.Fatal(err) + } + replay := func(tx *transaction) Object { + return Object{"complete": completeResidualReclamation(tx, tx.one("SELECT * FROM operations WHERE id=$1", op.S("id")))} + } + fact := func(tx *transaction) Object { + outcome := "ready" + if conflict { + outcome = "failed" + } + return provisioningCreationFact(tx, &ControlRequest{OperationID: scene.operation, Body: Object{"outcome": outcome}}, scene.run) + } + first, second := replay, fact + if factFirst { + first, second = fact, replay + } + a, b := seedResidualRace(t, scene, first, second) + if !conflict { + if a.err != nil || b.err != nil { + t.Fatalf("cleared replay failed: %v / %v", a.err, b.err) + } + } else if factFirst { + assertResidualFactConflict(t, a.err) + assertResidualFactConflict(t, b.err) + } else { + if a.err != nil { + t.Fatal(a.err) + } + assertResidualFactConflict(t, b.err) + } + if b.observed.S("reclaimState") != "cleared" { + t.Fatal("second transaction lost the cleared identity") + } + seedResidualAssert(t, scene, `SELECT reclaim_state='cleared' AND disposition='released' AND cleared_at IS NOT NULL + AND responsibility=$2::jsonb AND jsonb_array_length(reclaim_evidence)=$3 AND (reclaim_conflicted_at IS NOT NULL)=$4 + AND (SELECT count(*) FROM operations WHERE workspace_id=$5 AND kind='delete_workspace')=1 + FROM agent_run_residuals WHERE run_id=$1`, scene.run, scene.responsibility, before+map[bool]int{false: 0, true: 1}[conflict], conflict, scene.workspace) + }) + } + } +} diff --git a/internal/core/clone.go b/internal/core/clone.go index 79eaa9d7..21465b69 100644 --- a/internal/core/clone.go +++ b/internal/core/clone.go @@ -53,6 +53,9 @@ func recordedSubmission(t *transaction, r *ControlRequest, hash string, apply fu return old.O("response") } out := apply() + if t.residualFactConflict { + out["residualFactConflict"] = true + } t.exec("INSERT INTO control_submissions(submission_id,holder_id,request_hash,response) VALUES($1,$2,$3,$4)", r.SubmissionID, r.Service.Subject, hash, jsonText(out)) return out } @@ -116,7 +119,14 @@ func cloneResult(t *transaction, r *ControlRequest, withReceipt bool) Object { require((outcome == "clone_ready" || outcome == "clone_failed") && result.O("node").S("nodeId") == e.S("nodeId"), 409, "result_conflict") // A checkout of any other commit than the snapshot it was given is a contradiction, not a // fact to record: the baseline would be wrong for every later delivery and resume (contract D3). + // Validate before receipt auditing intentionally: invalid results return 400 without audit, as on the Node path. require(outcome != "clone_ready" || result.S("commit") == e.O("input").S("baseCommit"), 400, "base_commit_mismatch") + if withReceipt && !acceptResidualReceipt(t, e, r.Body.N("sequence"), r.Body.S("event"), "clone_event_receipts") { + return e + } + if !acceptResidualExecutionFact(t, e, result) { + return e + } if previous := e.O("result"); len(previous) > 0 { require(jsonText(previous) == jsonText(result), 409, "result_conflict") } else { diff --git a/internal/core/control.go b/internal/core/control.go index 93ab2d10..a470f792 100644 --- a/internal/core/control.go +++ b/internal/core/control.go @@ -32,7 +32,12 @@ func (s *Store) Control(ctx context.Context, r *ControlRequest) (Object, error) copyRequest.verification = prepared r = ©Request } - out, err := s.transact(ctx, func(t *transaction) Object { + out, err := s.transact(ctx, func(t *transaction) (response Object) { + defer func() { + if t.residualFactConflict && response != nil { + response["residualFactConflict"] = true + } + }() if r.Action == "access" || r.Action == "admit" { require(r.Service.Role == "controller", 403, "service_forbidden") if r.Action == "admit" || r.Body.S("action") == "execute" { @@ -54,6 +59,10 @@ func (s *Store) Control(ctx context.Context, r *ControlRequest) (Object, error) return cloneCommand(t, r) } leaseValid(t, r) + t.residualFactSource = Object{ + "kind": "controller", "holderId": r.Service.Subject, + "epoch": r.Body.N("epoch"), "submissionId": r.SubmissionID, + } refreshRuntimeControls(t) if r.Action == "effect_permit" { return effectPermit(t, r) @@ -87,11 +96,25 @@ func (s *Store) Control(ctx context.Context, r *ControlRequest) (Object, error) // return the recorded response, not fail the version check it already passed. return submitted(t, r, func() Object { return operationCommand(t, r) }) }) + if err == nil && out.B("residualFactConflict") { + return nil, &Fault{Status: 409, Code: "residual_fact_conflict", Params: Object{}} + } return out, err } // operationCommand runs one Effect-level action on the operation the caller claimed. func operationCommand(t *transaction, r *ControlRequest) Object { + if r.Action == "effect_result" && validID(r.OperationID) { + // A completed residual operation has no execution authority left, but its original + // effect receipts still arrive through this existing intake. Do not require a running + // operation or its old version merely to preserve an identical fact or audit a conflict. + completed := t.one(`SELECT o.* FROM operations o JOIN agent_run_residuals ar ON ar.reclaim_operation_id=o.id + WHERE o.id=$1 AND o.kind='delete_workspace' AND o.state='succeeded' + AND ar.reclaim_state='cleared' AND ar.reclaim_mode='owned_delete'`, r.OperationID) + if completed != nil { + return effectResult(t, r, completed) + } + } o := operation(t, r) switch r.Action { case "snapshot": @@ -109,6 +132,12 @@ func operationCommand(t *transaction, r *ControlRequest) Object { require(deferCodes[code], 400, "invalid_error_code") delay := r.Body.N("retrySeconds") require(delay >= 1 && delay <= 3600, 400, "invalid_retry_delay") + if row := residualOperation(t, o); row != nil && row.S("reclaimMode") == "owned_delete" && row.S("reclaimState") == "reclaiming" && row["reclaimConflictedAt"] == nil { + // A3 has no public/operator retry path for this hidden Workspace. Preserve the + // same retryable operation even when the existing Controller requests blocked + // for an unconfirmed termination; plan/permit still require fresh real evidence. + state = "retry_wait" + } if o.S("step") == "clone" && code == "clone_failed" { deferWorkspaceClone(t, o) return t.one("SELECT * FROM operations WHERE id=$1", o.S("id")) @@ -266,6 +295,8 @@ func planEffect(t *transaction, r *ControlRequest, o Object) Object { reconciled(t, o) kind, wid := r.Body.S("kind"), r.Body.S("workspaceId") require(validID(wid), 400, "invalid_effect_scope") + requireNoResidualConflict(t, wid) + checkResidualEffect(t, o, kind, wid, nil) found := false for _, w := range operationWorkspaces(t, o) { if w.S("id") == wid { @@ -314,22 +345,19 @@ func effectResult(t *transaction, r *ControlRequest, o Object) Object { e := t.one("SELECT * FROM external_effects WHERE id=$1 AND operation_id=$2", r.EffectID, o.S("id")) require(e != nil, 404, "not_found") require(e.S("kind") != "plugin_ensure" && e.S("kind") != "plugin_delete", 409, "invalid_step") + residual := residualEffectRow(t, e) state := r.Body.S("state") require(state == "running" || state == "succeeded" || state == "failed" || state == "absent", 400, "invalid_effect_state") result := r.Body.O("result") external := r.Body.S("externalId") if state == "absent" { - require(e.S("state") == "planned", 409, "external_state_conflict") state = "planned" } else { require(external != "" && len(external) <= 200, 400, "external_id_required") - if e.S("externalId") != "" { + if e.S("externalId") != "" && residual == nil { require(external == e.S("externalId"), 409, "external_binding_conflict") } } - if e.S("state") == "succeeded" { - require(state == "succeeded" && jsonText(result) == jsonText(e.O("result")), 409, "external_state_conflict") - } if state == "succeeded" { switch e.S("kind") { case "sandbox_terminate": @@ -348,6 +376,26 @@ func effectResult(t *transaction, r *ControlRequest, o Object) Object { require(result.B("removed"), 400, "invalid_cleanup_evidence") } } + if residual != nil { + fact := Object{ + "kind": "effect", "effectKind": e.S("kind"), "state": state, + "externalId": external, "result": result, + } + if !acceptResidualEffectFact(t, residual, e, fact) { + return Object{"effect": e, "operation": t.one("SELECT * FROM operations WHERE id=$1", o.S("id"))} + } + if residual.S("reclaimState") == "cleared" && residual.S("reclaimOperationId") == o.S("id") { + // Receipt only: neither canonical effects, operation versions nor resource rows may + // advance again after atomic completion. Contradictions already committed their block. + return Object{"effect": e, "operation": o} + } + } + if r.Body.S("state") == "absent" { + require(e.S("state") == "planned", 409, "external_state_conflict") + } + if e.S("state") == "succeeded" { + require(state == "succeeded" && jsonText(result) == jsonText(e.O("result")), 409, "external_state_conflict") + } if state == "failed" && (e.S("kind") == "plugin_ensure" || e.S("kind") == "plugin_delete") { // A failed install/remove surfaces on the space row immediately so the // UI can render the failure while the operation waits for retry. @@ -382,6 +430,7 @@ func completedEffect(t *transaction, o Object, kind, wid string) Object { func advance(t *transaction, r *ControlRequest, o Object) Object { reconciled(t, o) + checkResidualOperation(t, o) next := "" wid := o.S("workspaceId") switch o.S("step") { @@ -412,7 +461,7 @@ func advance(t *transaction, r *ControlRequest, o Object) Object { require(!repositoryOperationsUnfinished(t, o.S("projectId")), 409, "repository_operation_unfinished") } for _, w := range operationWorkspaces(t, o) { - checkActivities(t, w) + checkResidualCleanupActivities(t, w, o) live := t.one("SELECT id FROM sandbox_instances WHERE workspace_id=$1 AND terminated_at IS NULL", w.S("id")) if live != nil { require(t.one("SELECT id FROM node_instances WHERE sandbox_instance_id=$1 AND ended_at IS NULL AND idle_admission_epoch=$2 AND connection_state='connected' AND last_seen_at>clock_timestamp()-interval '30 seconds'", live.S("id"), w.N("admissionEpoch")) != nil, 409, "idle_unconfirmed") @@ -424,6 +473,9 @@ func advance(t *transaction, r *ControlRequest, o Object) Object { live := t.one("SELECT id FROM sandbox_instances WHERE workspace_id=$1 AND terminated_at IS NULL", w.S("id")) if live != nil { completedEffect(t, o, "sandbox_terminate", w.S("id")) + if row := residualOperation(t, o); row != nil { + require(residualSandboxTerminated(t, row, live.S("id")), 409, "termination_unconfirmed") + } t.exec("UPDATE node_instances SET ended_at=now(),connection_state='ended',version=version+1 WHERE sandbox_instance_id=$1 AND ended_at IS NULL", live.S("id")) t.exec("UPDATE sandbox_instances SET observed_state='terminated',terminated_at=now() WHERE id=$1", live.S("id")) } @@ -440,6 +492,9 @@ func advance(t *transaction, r *ControlRequest, o Object) Object { next = "cleanup" } case "cleanup": + if !completeResidualReclamation(t, o) { + return t.one("SELECT * FROM operations WHERE id=$1", o.S("id")) + } // Each Workspace's data was deleted by its own effect, planned only after its sandbox ended. for _, w := range operationWorkspaces(t, o) { completedEffect(t, o, "workspace_data_delete", w.S("id")) diff --git a/internal/core/migrations/0057_agent_run_residual_reclamation.sql b/internal/core/migrations/0057_agent_run_residual_reclamation.sql new file mode 100644 index 00000000..a01605e3 --- /dev/null +++ b/internal/core/migrations/0057_agent_run_residual_reclamation.sql @@ -0,0 +1,71 @@ +-- The residual row remains the sole reclamation authority. New defaults initialize legacy D4/D5 +-- rows without rewriting their reason, responsibility, timestamps, or business settlement. +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid='agent_run_residuals'::regclass AND attname='reclaim_state' AND NOT attisdropped + ) AND EXISTS (SELECT 1 FROM agent_run_residuals WHERE cleared_at IS NOT NULL) THEN + RAISE EXCEPTION 'residual reclamation migration cannot initialize legacy rows with non-null cleared_at; explicit compatibility handling is required (A2)'; + END IF; +END $$; + +ALTER TABLE agent_run_residuals + ADD COLUMN IF NOT EXISTS reclaim_state text NOT NULL DEFAULT 'reconciling', + ADD COLUMN IF NOT EXISTS disposition text, + ADD COLUMN IF NOT EXISTS reclaim_id uuid, + ADD COLUMN IF NOT EXISTS reclaim_mode text, + ADD COLUMN IF NOT EXISTS reclaim_operation_id uuid, + ADD COLUMN IF NOT EXISTS reclaim_authorization jsonb NOT NULL DEFAULT '{}', + ADD COLUMN IF NOT EXISTS reclaim_targets jsonb NOT NULL DEFAULT '[]', + ADD COLUMN IF NOT EXISTS reclaim_evidence jsonb NOT NULL DEFAULT '[]', + ADD COLUMN IF NOT EXISTS reclaim_version bigint NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS reclaim_checked_at timestamptz, + ADD COLUMN IF NOT EXISTS reclaim_conflicted_at timestamptz; + +-- Named constraints make direct DDL replay safe too. They describe valid row shapes, not the +-- history of state transitions; conditional writes in the owning transaction enforce that history. +DO $reclaim_constraints$ +DECLARE + item record; +BEGIN + FOR item IN SELECT * FROM (VALUES + ('agent_run_residuals_reclaim_state_check', + $$CHECK (reclaim_state IN ('reconciling','reclaiming','cleared'))$$), + ('agent_run_residuals_disposition_check', + $$CHECK (disposition IS NULL OR disposition IN ('released','terminated_data_retained','externally_confirmed'))$$), + ('agent_run_residuals_reclaim_mode_check', + $$CHECK (reclaim_mode IS NULL OR reclaim_mode IN ('owned_delete','controlled'))$$), + ('agent_run_residuals_reclaim_json_check', + $$CHECK (jsonb_typeof(reclaim_authorization)='object' + AND jsonb_typeof(reclaim_targets)='array' AND jsonb_typeof(reclaim_evidence)='array')$$), + ('agent_run_residuals_reclaim_version_check', $$CHECK (reclaim_version>=0)$$), + ('agent_run_residuals_reclaim_responsibility_check', + $$CHECK ( + (reclaim_state='reconciling' AND reclaim_id IS NULL AND reclaim_mode IS NULL + AND reclaim_operation_id IS NULL AND reclaim_authorization='{}'::jsonb) + OR (reclaim_state IN ('reclaiming','cleared') AND reclaim_id IS NOT NULL + AND reclaim_mode IS NOT NULL AND reclaim_authorization<>'{}'::jsonb + AND (reclaim_mode<>'owned_delete' OR reclaim_operation_id IS NOT NULL)))$$), + ('agent_run_residuals_reclaim_completion_check', + $$CHECK ( + (reclaim_state='cleared' AND cleared_at IS NOT NULL AND disposition IS NOT NULL) + OR (reclaim_state<>'cleared' AND cleared_at IS NULL AND disposition IS NULL))$$), + ('agent_run_residuals_reclaim_operation_fk', + $$FOREIGN KEY (reclaim_operation_id) REFERENCES operations(id)$$) + ) AS definitions(name, definition) + LOOP + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid='agent_run_residuals'::regclass AND conname=item.name) THEN + EXECUTE format('ALTER TABLE agent_run_residuals ADD CONSTRAINT %I %s', item.name, item.definition); + END IF; + END LOOP; +END $reclaim_constraints$; + +CREATE UNIQUE INDEX IF NOT EXISTS agent_run_residuals_reclaim_id_unique + ON agent_run_residuals(reclaim_id) WHERE reclaim_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS agent_run_residuals_reclaim_operation_unique + ON agent_run_residuals(reclaim_operation_id) WHERE reclaim_operation_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS agent_run_residuals_reclaim_scan + ON agent_run_residuals(reclaim_checked_at NULLS FIRST, created_at, run_id) + WHERE reclaim_state IN ('reconciling','reclaiming'); diff --git a/internal/core/migrations/README.en.md b/internal/core/migrations/README.en.md index fce2c0b6..06fd662f 100644 --- a/internal/core/migrations/README.en.md +++ b/internal/core/migrations/README.en.md @@ -6,7 +6,7 @@ SQL is embedded in the binary through `embed.FS` in `core/store.go`. `Store.Migr ## File index -There are currently 58 SQL files. +There are currently 59 SQL files. | SQL file | Schema change | |---|---| @@ -68,6 +68,7 @@ There are currently 58 SQL files. | [0054_git_delivery_refs.sql](0054_git_delivery_refs.sql) | Revisions delivered over Git | | [0055_workflow_run_workspace_release.sql](0055_workflow_run_workspace_release.sql) | Workspace release facts for WorkflowRuns | | [0056_workflow_invocation_reference.sql](0056_workflow_invocation_reference.sql) | Invocation reference from workflow IssueRuns to WorkflowRuns | +| [0057_agent_run_residual_reclamation.sql](0057_agent_run_residual_reclamation.sql) | Residual reclamation responsibility, appended evidence and disposition | 0050 copies no personal connection or key and backfills no historical run. Clearing configuration retains its revision; run options are stored separately from shared run input. @@ -87,6 +88,7 @@ There are currently 58 SQL files. - 0054 is the switch to Revisions delivered over Git and migrates no data: `revisions` gains `delivery_ref`, and `revisions_bundle_shape` is replaced by `revisions_shape` (with a NULL `delivery_ref` a row keeps a legacy bundle shape; with one it must be a well-formed `refs/ora/deliveries/` and carry no bundle column); the partial unique index `revisions_delivery_ref` lets one delivery ref belong to one Revision only. Existing bundle rows are never rewritten, and it is retry-safe. See [Revisions](../../../docs/agent-runs/revisions.en.md); the constraints are covered by `TestRevisionShapeKeepsGitAndBundleRowsApart`. - 0055 adds the forward-only `workspace_release_state` (`pending`/`releasing`/`released`/`abandoned`/`not_required`), `workspace_release_error` and `workspace_released_at` to `workflow_runs`, and backfills by upgrade-time state: non-terminal rows stay `pending`; terminal rows with no Workspace become `not_required`, those whose Workspace is already soft-deleted become `released` (using that row's `deleted_at`), and the rest become `releasing`. Two CHECK constraints require a release instant for `released` and a reason for `abandoned`. Only `released` means Cloud observed the Workspace deleted; `abandoned` keeps the Workspace row and writes no release instant. Semantics are in [workflows](../../../docs/workflows/workflows.en.md#scheduling); coverage is `TestMigration0055WorkflowRunWorkspaceReleaseAppliesFreshAndUpgrades` and `internal/core/workflow_run_release_db_test.go`. - 0056 first settles every unlinked non-terminal historical workflow IssueRun (including running and deferred) as failed with `workflow_execution_unavailable_before_invocation_linkage`, preserving its input and outcome without inventing an execution. It then adds the `(workflow_invocation_ref, tenant_id)` → `workflow_runs(id, tenant_id)` composite foreign key (targeting the `workflow_run_tenant_identity` unique constraint 0044 already added, without duplicating it), a unique reference, and a CHECK that non-terminal workflow rows carry a reference while other executors carry none, all unconditionally validated. Historical terminal workflow rows may keep a NULL reference. Fresh, seeded upgrade, repeated application and constraint validation coverage is in `integration/workflow_invocation_migration_test.go`. +- 0057 only adds residual-ledger columns: defaults initialize existing D4/D5 rows as `reconciling`, without backfilling targets or evidence, rewriting old `reason`, `responsibility` or business settlement, or backfilling D8. Legacy rows with non-null `cleared_at` cause an explicit migration error and rollback pending compatibility handling (A2). Reentrant DDL preserves progressed rows. The schema accepts `released`, `terminated_data_retained` and `externally_confirmed`; this step's automatic path implements only `released`, leaving the other dispositions to later controlled entry points. The authorization transaction checks the operation's tenant and Workspace relationships; a UUID foreign key alone does not establish resource scope. See [agent_run_residual_reclaim_migration_test.go](../../../integration/agent_run_residual_reclaim_migration_test.go) for fresh, upgrade, DDL replay and constraint coverage. - New migrations must verify fresh databases, upgrades from the preceding schema, repeated application and key constraints. See [migration_upgrade_path_test.go](../../../integration/migration_upgrade_path_test.go) and domain upgrade tests in integration. See [Persistence](../../../docs/core/persistence.en.md) for ownership/transactions, and [Workflows](../../../docs/workflows/workflows.en.md) / [External Git line](../../../docs/repositories/external-git.en.md) for runtime semantics. diff --git a/internal/core/migrations/README.md b/internal/core/migrations/README.md index f72c42f4..873dc1c0 100644 --- a/internal/core/migrations/README.md +++ b/internal/core/migrations/README.md @@ -6,7 +6,7 @@ SQL 通过 `core/store.go` 的 `embed.FS` 编入二进制。`Store.Migrate` 在 ## 文件索引 -当前共 58 个 SQL 文件。 +当前共 59 个 SQL 文件。 | SQL 文件 | schema 变化 | |---|---| @@ -68,6 +68,7 @@ SQL 通过 `core/store.go` 的 `embed.FS` 编入二进制。`Store.Migrate` 在 | [0054_git_delivery_refs.sql](0054_git_delivery_refs.sql) | 经 Git 交付的 Revision | | [0055_workflow_run_workspace_release.sql](0055_workflow_run_workspace_release.sql) | WorkflowRun 的 Workspace 释放事实 | | [0056_workflow_invocation_reference.sql](0056_workflow_invocation_reference.sql) | workflow IssueRun 到 WorkflowRun 的调用引用 | +| [0057_agent_run_residual_reclamation.sql](0057_agent_run_residual_reclamation.sql) | 残留回收责任、追加证据和处置状态 | 0050 不复制个人连接或 Key,不回填历史运行;清除配置保留 revision,运行选项独立于共享运行输入保存。 @@ -87,6 +88,7 @@ SQL 通过 `core/store.go` 的 `embed.FS` 编入二进制。`Store.Migrate` 在 - 0054 是 Revision 改为经 Git 交付的切换,不迁移数据:`revisions` 增加 `delivery_ref`,`revisions_bundle_shape` 换成 `revisions_shape`(`delivery_ref` 为 NULL 时仍是原有的 bundle 形状,非 NULL 时必须是格式正确的 `refs/ora/deliveries/` 且不带任何 bundle 列),部分唯一索引 `revisions_delivery_ref` 让一个交付 ref 只属于一个 Revision;既有 bundle 行不改写,可重复执行。语义见 [Revision](../../../docs/agent-runs/revisions.md),约束覆盖见 `TestRevisionShapeKeepsGitAndBundleRowsApart`。 - 0055 给 `workflow_runs` 增加单向的 `workspace_release_state`(`pending`/`releasing`/`released`/`abandoned`/`not_required`)、`workspace_release_error` 和 `workspace_released_at`,并按升级时的状态回填:非终态行保持 `pending`;终态行没有 Workspace 的记 `not_required`,Workspace 已软删除的记 `released`(释放时刻取该行的 `deleted_at`),其余记 `releasing`。两条 CHECK 保证 `released` 必有释放时刻、`abandoned` 必有原因。只有 `released` 表示 Cloud 观察到 Workspace 已被删除;`abandoned` 保留 Workspace 行、不写释放时刻。语义见[工作流](../../../docs/workflows/workflows.md#调度),覆盖见 `TestMigration0055WorkflowRunWorkspaceReleaseAppliesFreshAndUpgrades` 与 `internal/core/workflow_run_release_db_test.go`。 - 0056 先把没有引用的历史非终态 workflow IssueRun(含 running、deferred)结算为 failed(`workflow_execution_unavailable_before_invocation_linkage`),保留其输入和结果、不虚构执行;再加 `(workflow_invocation_ref, tenant_id)` → `workflow_runs(id, tenant_id)` 复合外键(引用 0044 已建的 `workflow_run_tenant_identity` 唯一约束,不重复建)、唯一引用,以及「非终态 workflow 行必须有引用、其他执行器不得有引用」的 CHECK,无条件 VALIDATE。历史终态 workflow 行可保留 NULL 引用。空库、带历史数据升级、重复应用与约束验证状态见 `integration/workflow_invocation_migration_test.go`。 +- 0057 只给残留账本增列:存量 D4/D5 行通过默认值初始化为 `reconciling`,不回填目标或证据,不改旧 `reason`、`responsibility` 或业务结算,也不回填 D8。历史行若已有非空 `cleared_at`,迁移明确报错并回滚,等待显式兼容处理(A2)。DDL 可重入,不会重置已推进条目。数据库接受 `released`、`terminated_data_retained`、`externally_confirmed` 三种处置;本步自动路径只实现 `released`,其余处置留给后续受控入口。operation 的租户和 Workspace 关联由授权事务比对,UUID 外键本身不证明资源范围。空库、升级、DDL 重放和约束覆盖见 [agent_run_residual_reclaim_migration_test.go](../../../integration/agent_run_residual_reclaim_migration_test.go)。 - 新增迁移须验证空库、前一 schema 升级、重复应用与关键约束。覆盖见 [migration_upgrade_path_test.go](../../../integration/migration_upgrade_path_test.go) 和 integration 的领域升级测试。 数据归属与事务规则见 [持久化](../../../docs/core/persistence.md),运行语义见 [工作流](../../../docs/workflows/workflows.md) 和 [外部 Git 线](../../../docs/repositories/external-git.md)。 diff --git a/internal/core/node.go b/internal/core/node.go index d186817e..882eb803 100644 --- a/internal/core/node.go +++ b/internal/core/node.go @@ -117,7 +117,7 @@ func nodeStatus(t *transaction, r *ControlRequest, n, w Object) { // A business-abandoned execution still owns residual resources after it reconnects; // connection evidence must not reopen their admission or imply physical cleanup. t.exec(`UPDATE workspaces SET observed_state='ready',admission_open=NOT EXISTS( - SELECT 1 FROM agent_run_residuals WHERE workspace_id=$1 AND cleared_at IS NULL), + SELECT 1 FROM agent_run_residuals WHERE workspace_id=$1 AND (cleared_at IS NULL OR reclaim_conflicted_at IS NOT NULL)), version=version+1 WHERE id=$1`, w.S("id")) } } @@ -141,7 +141,7 @@ func nodeIdle(t *transaction, r *ControlRequest, n, w Object) Object { restoreAdmission(t, o) return Object{"accepted": false, "errorCode": "resource_in_use"} } - checkActivities(t, w) + checkResidualCleanupActivities(t, w, o) c := runtimeControl(t, w.S("id")) require(c.S("state") == "maintenance" && c.S("maintenanceOperationId") == o.S("id") && c.B("bindingConfirmed") && c.S("boundSandboxId") == n.S("sandboxInstanceId"), 409, "runtime_input_closure_unconfirmed") require(n.B("initialized") && n.S("connectionState") == "connected", 409, "idle_unconfirmed") @@ -150,6 +150,17 @@ func nodeIdle(t *transaction, r *ControlRequest, n, w Object) Object { } func restoreAdmission(t *transaction, o Object) { + if row := residualOperation(t, o); row != nil { + checkResidualOperation(t, o) + // The original authority survives refusal. A new binding acknowledgment and idle report + // are required before this SAME operation can retry; admission never reopens. + t.exec(`UPDATE operations SET state='retry_wait',error_code='resource_in_use',retry_at=clock_timestamp(), + version=version+1,updated_at=clock_timestamp() WHERE id=$1`, o.S("id")) + t.exec(`UPDATE runtime_controls SET binding_confirmed=false,input_closed=false,control_epoch=control_epoch+1, + version=version+1,updated_at=clock_timestamp() WHERE workspace_id=$1`, row.S("workspaceId")) + t.exec("UPDATE node_instances SET idle_admission_epoch=NULL WHERE workspace_id=$1", row.S("workspaceId")) + return + } for wid, v := range o.O("request").O("previous") { raw, ok := v.(map[string]any) require(ok, 500, "internal_error") diff --git a/internal/core/node_execution.go b/internal/core/node_execution.go index 0db824a6..09c95af4 100644 --- a/internal/core/node_execution.go +++ b/internal/core/node_execution.go @@ -115,6 +115,13 @@ func nodeExecutionResult(t *transaction, r *ControlRequest, withReceipt bool) Ob e := t.one("SELECT * FROM node_executions WHERE execution_id=$1 AND operation_id=$2", execution, operation) require(e != nil, 404, "not_found") require(result.O("node").S("nodeId") == e.S("nodeId"), 409, "result_conflict") + validateNodeResult(e, result) + if withReceipt && !acceptResidualReceipt(t, e, r.Body.N("sequence"), r.Body.S("event"), "node_event_receipts") { + return e + } + if !acceptResidualExecutionFact(t, e, result) { + return e + } fresh := len(e.O("result")) == 0 if !fresh { require(jsonText(e.O("result")) == jsonText(result), 409, "result_conflict") diff --git a/internal/core/public.go b/internal/core/public.go index 8fae195b..38f94fd7 100644 --- a/internal/core/public.go +++ b/internal/core/public.go @@ -204,6 +204,7 @@ func (s *Store) Public(ctx context.Context, r *PublicRequest) (Object, int, erro require(r.Method == "DELETE", 405, "method_not_allowed") requireSpaceRole(m, "admin") version(p, r.Body.N("version")) + require(t.one("SELECT run_id FROM agent_run_residuals WHERE project_id=$1 AND reclaim_conflicted_at IS NOT NULL", p.S("id")) == nil, 409, "residual_fact_conflict") idleProject(t, p.S("id")) require(p.S("lifecycle") == "active", 409, "resource_unavailable") ws := t.list("SELECT * FROM workspaces WHERE project_id=$1 AND deleted_at IS NULL ORDER BY id", p.S("id")) @@ -716,6 +717,7 @@ func newOperation(t *transaction, r *PublicRequest, uid, pid, wid, kind, step, h } func checkActivities(t *transaction, w Object) { + requireNoResidualConflict(t, w.S("id")) require(t.one("SELECT run_id FROM agent_run_residuals WHERE workspace_id=$1 AND cleared_at IS NULL", w.S("id")) == nil, 409, "resource_in_use") require(t.one("SELECT id FROM execution_tickets WHERE workspace_id=$1 AND state='active' AND terminated_by_force_stop_id IS NULL", w.S("id")) == nil, 409, "resource_in_use") require(t.one("SELECT execution_id FROM clone_executions WHERE workspace_id=$1 AND result IS NULL AND terminated_by_force_stop_id IS NULL", w.S("id")) == nil, 409, "resource_in_use") diff --git a/internal/core/revision.go b/internal/core/revision.go index d95a016b..e75790dc 100644 --- a/internal/core/revision.go +++ b/internal/core/revision.go @@ -33,6 +33,10 @@ func (s *Store) prepareRevision(ctx context.Context, r *ControlRequest) (*revisi e, err := s.transact(ctx, func(t *transaction) Object { require(r.Service != nil && r.Service.Role == "controller", 403, "service_forbidden") leaseValid(t, r) + t.residualFactSource = Object{ + "kind": "controller", "holderId": r.Service.Subject, + "epoch": r.Body.N("epoch"), "submissionId": r.SubmissionID, + } require(len(r.SubmissionID) <= 200, 400, "invalid_submission") if old := t.one("SELECT * FROM control_submissions WHERE submission_id=$1", r.SubmissionID); old != nil { require(old.S("requestHash") == requestHash(r.Action, r.Service.Subject, r.Body), 409, "submission_conflict") @@ -41,6 +45,11 @@ func (s *Store) prepareRevision(ctx context.Context, r *ControlRequest) (*revisi e := t.one("SELECT * FROM node_executions WHERE execution_id=$1 AND operation_id=$2 AND kind='deliver_revision'", r.Body.S("executionId"), r.Body.S("operationId")) require(e != nil, 404, "not_found") if len(e.O("result")) > 0 { + require(r.Body.O("result").O("node").S("nodeId") == e.S("nodeId"), 409, "result_conflict") + validateNodeResult(e, r.Body.O("result")) + if !acceptResidualExecutionFact(t, e, r.Body.O("result")) { + return Object{"residualFactConflict": true} + } require(jsonText(e.O("result")) == jsonText(r.Body.O("result")), 409, "result_conflict") return nil } @@ -54,6 +63,9 @@ func (s *Store) prepareRevision(ctx context.Context, r *ControlRequest) (*revisi if err != nil || e == nil { return nil, err } + if e.B("residualFactConflict") { + return nil, &Fault{Status: 409, Code: "residual_fact_conflict", Params: Object{}} + } result := r.Body.O("result") verified := &revisionVerification{input: jsonText(e.O("input")), valid: true} if s.GitServer == nil || s.GitServer.Refs == nil { @@ -104,6 +116,10 @@ func validStoredObject(object Object) bool { // liveDelivery fences cancellation, deletion, replaced Nodes and closed runtime bindings again // after external verification. A stale result must not release a different Workspace generation. func liveDelivery(t *transaction, e Object) { + if residualExecutionIdentity(t, e) != nil { + // This is evidence about the original execution, not permission to execute it again. + return + } require(e["terminatedByForceStopId"] == nil, 409, "dispatch_conflict") w := t.one("SELECT * FROM workspaces WHERE id=$1 AND deleted_at IS NULL", e.S("workspaceId")) require(w != nil && w.S("issueRunId") == e.S("operationId") && w.S("observedState") == "ready" && w.B("admissionOpen"), 409, "dispatch_conflict") diff --git a/internal/core/run_workspace.go b/internal/core/run_workspace.go index 40e19348..943ed1db 100644 --- a/internal/core/run_workspace.go +++ b/internal/core/run_workspace.go @@ -115,6 +115,8 @@ func deleteWorkflowRunWorkspace(t *transaction, runID string) Object { func deleteRunWorkspace(t *transaction, runID string) Object { w := t.one("SELECT * FROM workspaces WHERE issue_run_id=$1", runID) require(w != nil, 404, "not_found") + requireNoResidualConflict(t, w.S("id")) + require(t.one("SELECT run_id FROM agent_run_residuals WHERE workspace_id=$1 AND cleared_at IS NULL", w.S("id")) == nil, 409, "resource_in_use") if w["deletedAt"] != nil { return Object{"busy": false, "workspace": w} } @@ -124,7 +126,6 @@ func deleteRunWorkspace(t *transaction, runID string) Object { if t.one("SELECT id FROM operations WHERE project_id=$1 AND state IN ('queued','running','retry_wait','blocked')", w.S("projectId")) != nil { return Object{"busy": true} } - require(t.one("SELECT run_id FROM agent_run_residuals WHERE workspace_id=$1 AND cleared_at IS NULL", w.S("id")) == nil, 409, "resource_in_use") closeAdmission(t, w, "deleted") actor := w.S("creatorUserId") // The request carries the pre-declaration Workspace row, exactly as the public delete path does diff --git a/internal/core/runtime_control.go b/internal/core/runtime_control.go index 21307023..0cc0473c 100644 --- a/internal/core/runtime_control.go +++ b/internal/core/runtime_control.go @@ -34,7 +34,7 @@ func reconcileRuntimeControls(t *transaction) { rows := t.list(`SELECT c.* FROM runtime_controls c JOIN workspaces w ON w.id=c.workspace_id WHERE c.state IN ('draining','reconciling') AND NOT EXISTS(SELECT 1 FROM runtime_force_stops f WHERE f.workspace_id=w.id AND f.state<>'succeeded') - AND NOT EXISTS(SELECT 1 FROM agent_run_residuals ar WHERE ar.workspace_id=w.id AND ar.cleared_at IS NULL) + AND NOT EXISTS(SELECT 1 FROM agent_run_residuals ar WHERE ar.workspace_id=w.id AND (ar.cleared_at IS NULL OR ar.reclaim_conflicted_at IS NOT NULL)) AND (c.input_closed OR (w.observed_state IN ('stopped','deleted') AND NOT EXISTS(SELECT 1 FROM sandbox_instances s WHERE s.workspace_id=w.id AND s.terminated_at IS NULL))) AND NOT EXISTS(SELECT 1 FROM execution_tickets a WHERE a.workspace_id=w.id AND a.state='active' AND a.terminated_by_force_stop_id IS NULL) diff --git a/internal/core/runtime_control_delivery.go b/internal/core/runtime_control_delivery.go index 1753cb4b..9dea87c3 100644 --- a/internal/core/runtime_control_delivery.go +++ b/internal/core/runtime_control_delivery.go @@ -1,5 +1,7 @@ package core +import "fmt" + // runtimeControlCommand is a finite, lease-fenced Controller contract. The Controller may // acknowledge Node evidence, but cannot assign a holder, epoch or target chosen by a user. func runtimeControlCommand(t *transaction, r *ControlRequest) Object { @@ -16,6 +18,15 @@ func runtimeControlCommand(t *transaction, r *ControlRequest) Object { v, ok := id.(string) require(ok && v != "" && len(v) <= 256, 400, "invalid_runtime_evidence") } + if row := residualForWorkspace(t, c.S("workspaceId")); row != nil { + sandbox := t.one("SELECT generation FROM sandbox_instances WHERE id=$1", n.S("sandboxInstanceId")) + appendResidualFact(t, row.S("runId"), "runtime:"+n.S("id")+":"+fmt.Sprint(c.N("controlEpoch"))+":"+fmt.Sprint(c.N("version")), Object{ + "kind": "runtime_closed", "state": "observed", "workspaceId": c.S("workspaceId"), + "sandboxId": n.S("sandboxInstanceId"), "generation": sandbox.N("generation"), + "controlEpoch": c.N("controlEpoch"), "controlVersion": c.N("version"), + "inputClosed": closed, "unfinishedExecutionIds": unfinished, + }) + } if closed && (c.S("state") == "held" || c.S("state") == "acquiring") { // Node expiry is conservative. Observed closed input cannot be reopened by a late renew, // even if the PostgreSQL deadline has not elapsed yet. diff --git a/internal/core/runtime_force_stop.go b/internal/core/runtime_force_stop.go index 3c4458a6..7627d765 100644 --- a/internal/core/runtime_force_stop.go +++ b/internal/core/runtime_force_stop.go @@ -102,6 +102,9 @@ func effectPermit(t *transaction, r *ControlRequest) Object { } else { e := t.one("SELECT e.*,o.state AS operation_state,o.controller_epoch,w.tenant_id,w.runtime_generation FROM external_effects e JOIN operations o ON o.id=e.operation_id JOIN workspaces w ON w.id=e.workspace_id WHERE e.id=$1", r.EffectID) require(e != nil && e.S("operationState") == "running" && e.N("controllerEpoch") == r.Body.N("epoch"), 409, "stale_effect") + op := t.one("SELECT * FROM operations WHERE id=$1", e.S("operationId")) + requireNoResidualConflict(t, e.S("workspaceId")) + checkResidualEffect(t, op, e.S("kind"), e.S("workspaceId"), e) wid, tid, generation = e.S("workspaceId"), e.S("tenantId"), e.N("runtimeGeneration") requireNoForceStop(t, wid) c := runtimeControl(t, wid) diff --git a/internal/core/store.go b/internal/core/store.go index cc3b3a33..044b02c1 100644 --- a/internal/core/store.go +++ b/internal/core/store.go @@ -252,6 +252,8 @@ type transaction struct { workSignals []string commandSignals []string events []SpaceEvent + residualFactConflict bool + residualFactSource Object } func (t *transaction) exec(q string, args ...any) {