From 7834594f7dfafd90fe4ba8dd84165eb20d41593d Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 9 Sep 2026 16:37:44 -0500 Subject: [PATCH 01/10] fix: retain modeled wait-path obligations --- js/tests/replica-command-runtime.test.mjs | 319 +++++++++++++++++++--- src/graphql/projection_delta/tests.rs | 169 ++++++++++++ src/microsvc/service/causal.rs | 52 +--- 3 files changed, 457 insertions(+), 83 deletions(-) diff --git a/js/tests/replica-command-runtime.test.mjs b/js/tests/replica-command-runtime.test.mjs index 74e11de4..a17865ff 100644 --- a/js/tests/replica-command-runtime.test.mjs +++ b/js/tests/replica-command-runtime.test.mjs @@ -12,6 +12,7 @@ import { ReplicaCommandRuntimeError } from '../dist/replica/command-runtime.js'; import { + createDistributedReplica, prepareReplicaCommand, replicaRecordKey, ReplicaCommandContractError @@ -21,6 +22,11 @@ import { COMMAND_STATE, commandReceipt } from './fixtures/command-protocol.mjs'; +import { + TodosArtifact, + TodoModel, + todoFrame +} from './fixtures/adapter-conformance.mjs'; const HASH_A = `sha256:${'a'.repeat(64)}`; const HASH_B = `sha256:${'b'.repeat(64)}`; @@ -46,6 +52,24 @@ const Todo = Object.freeze({ id: 'Todos', identityFields: Object.freeze(['id']) }); + +// Reuse the generated query/live selection while matching the command +// runtime's authoritative protocol scope. This keeps the regression on the +// public replica and generated-command paths rather than a hand-built cache. +const CommandTodos = Object.freeze({ + ...TodosArtifact, + id: 'query:command-runtime-todos', + protocol: Object.freeze({ + ...TodosArtifact.protocol, + schemaHash: HASH_B, + surface: SURFACE, + operation: 'query:command-runtime-todos' + }), + live: Object.freeze({ + ...TodosArtifact.live, + id: 'live:command-runtime-todos' + }) +}); const Audit = Object.freeze({ id: 'Audits', identityFields: Object.freeze(['id']) @@ -88,14 +112,15 @@ function scope(value, model = Todo.id) { }); } -function projection(operation = 'upsert') { +function projection(operation = 'upsert', model = Todo) { const event = Object.freeze({ id: 'event-1', name: 'todo.changed', version: 1 }); - const previewScope = scope(input(['id'])); + const previewScope = scope(input(['id']), model.id); const targetScope = scope( Object.freeze({ kind: 'constant', value: Object.freeze({ type: 'string', value: 'target' }) - }) + }), + model.id ); let mutation; if (operation === 'upsert') { @@ -130,7 +155,7 @@ function projection(operation = 'upsert') { mutation = Object.freeze({ op: 'invalidate_model', partition: unit, - model: Todo.id + model: model.id }); } else { mutation = Object.freeze({ @@ -144,40 +169,40 @@ function projection(operation = 'upsert') { ? Object.freeze({ kind: 'relationship', relationship: 'related', - source_model: Todo.id, + source_model: model.id, source_key: Object.freeze(['id']), - target_model: Todo.id, + target_model: model.id, target_key: Object.freeze(['id']), link: operation === 'link', unlink: operation === 'unlink' }) - : operation === 'invalidate_model' - ? Object.freeze({ kind: 'model', model: Todo.id }) - : operation === 'invalidate_relationship' - ? Object.freeze({ - kind: 'relationship', - relationship: 'related', - source_model: Todo.id, - source_key: Object.freeze(['id']), - target_model: Todo.id, - target_key: Object.freeze(['id']), - link: false, - unlink: false - }) - : Object.freeze({ - kind: 'record', - model: Todo.id, - key: Object.freeze(['id']), - fields: Object.freeze( - operation === 'delete' ? [] : ['title'] - ), - replace: Object.freeze( - operation === 'upsert' ? ['title'] : [] - ), - upsert: operation === 'upsert', - patch: operation === 'patch', - delete: operation === 'delete' - }); + : operation === 'invalidate_model' + ? Object.freeze({ kind: 'model', model: model.id }) + : operation === 'invalidate_relationship' + ? Object.freeze({ + kind: 'relationship', + relationship: 'related', + source_model: model.id, + source_key: Object.freeze(['id']), + target_model: model.id, + target_key: Object.freeze(['id']), + link: false, + unlink: false + }) + : Object.freeze({ + kind: 'record', + model: model.id, + key: Object.freeze(['id']), + fields: Object.freeze( + operation === 'delete' ? [] : ['title'] + ), + replace: Object.freeze( + operation === 'upsert' ? ['title'] : [] + ), + upsert: operation === 'upsert', + patch: operation === 'patch', + delete: operation === 'delete' + }); return Object.freeze({ version: 2, deltaWireVersion: 1, @@ -225,6 +250,7 @@ function projection(operation = 'upsert') { function artifact(options = {}) { const operation = options.operation ?? 'upsert'; + const model = options.model ?? Todo; return Object.freeze({ version: 2, name: options.name ?? `todo.${operation}`, @@ -243,7 +269,9 @@ function artifact(options = {}) { input: Object.freeze({ kind: 'object', definition: TodoInput }), output: Object.freeze({ kind: 'object', definition: ResultOutput }), consistency: options.consistency ?? COMMAND_CONSISTENCY.EVENTUAL, - ...(options.modeled === false ? {} : { projection: projection(operation) }), + ...(options.modeled === false + ? {} + : { projection: projection(operation, model) }), ...(options.directProjection === undefined ? {} : { directProjection: options.directProjection }), @@ -251,16 +279,16 @@ function artifact(options = {}) { version: 1, required: options.revalidate ?? false, dependencies: Object.freeze(['todos']), - models: Object.freeze([Todo.id]), + models: Object.freeze([model.id]), relationships: Object.freeze( operation === 'link' || operation === 'unlink' || operation === 'invalidate_relationship' ? [ Object.freeze({ - sourceModel: Todo.id, + sourceModel: model.id, field: 'related', - targetModel: Todo.id + targetModel: model.id }) ] : [] @@ -323,10 +351,11 @@ function modeledArtifactWithAuditArm() { function deltaMutation(request, options = {}) { const operation = options.operation ?? 'upsert'; + const model = options.model ?? Todo; const actualScope = scope({ type: 'string', value: request.variables.input.id - }); + }, model.id); if (operation === 'upsert') { return { op: 'upsert', @@ -366,11 +395,11 @@ function deltaMutation(request, options = {}) { op: operation, relationship: 'related', source: actualScope, - target: scope({ type: 'string', value: 'target' }) + target: scope({ type: 'string', value: 'target' }, model.id) }; } if (operation === 'invalidate_model') { - return { op: 'invalidate_model', partition: unit, model: Todo.id }; + return { op: 'invalidate_model', partition: unit, model: model.id }; } return { op: 'invalidate_relationship', @@ -397,7 +426,7 @@ function commandMetadata(request, options = {}) { { length: options.obligations ?? 1 }, (_, index) => ({ projectionRef: 0, - model: options.obligationModel ?? Todo.id, + model: options.obligationModel ?? options.model?.id ?? Todo.id, scopeToken: token('projection-obligation', index + 3) }) ); @@ -659,6 +688,214 @@ function tick() { return new Promise((resolve) => setTimeout(resolve, 0)); } +function commandFrame(artifactValue, rows, options = {}) { + const frame = todoFrame(artifactValue, rows, { + cacheScope: CACHE_SCOPE, + authorizationGeneration: 'auth-1', + position: options.position ?? '1', + source: options.source ?? 'query', + mode: options.mode ?? 'resumable', + reset: options.reset ?? false, + errors: options.errors + }); + const snapshot = frame.extensions.distributed.snapshot; + snapshot.observations = options.observations ?? []; + if (options.recordRevision !== undefined || options.incarnation !== undefined) { + snapshot.records = snapshot.records.map((record) => ({ + ...record, + ...(options.recordRevision === undefined + ? {} + : { revision: options.recordRevision }), + ...(options.incarnation === undefined + ? {} + : { incarnation: options.incarnation }) + })); + } + return frame; +} + +function obligationObservation(receipt, overrides = {}) { + const expectation = receipt.metadata.expects[0]; + assert.ok(expectation); + return { + causationId: receipt.metadata.causationId, + projection: expectation.projection, + model: expectation.model, + scopeToken: expectation.scopeToken, + ...overrides + }; +} + +test('generated command runtime retains an accepted projection until a matching live observation', async () => { + let liveObserver; + const replica = createDistributedReplica({ + transport: { + fetch() { + return Promise.reject(new Error('unexpected query fetch')); + }, + subscribe(_request, observer) { + liveObserver = observer; + return () => undefined; + } + } + }); + replica.writeResult( + CommandTodos, + {}, + commandFrame(CommandTodos, [ + { id: 'todo-1', title: 'base', status: 'open' } + ]), + 'network' + ); + const watch = replica.watch(CommandTodos, {}, { live: true }); + assert.ok(liveObserver); + const runtime = createReplicaCommandRuntime( + replica, + { + dispatch(request) { + return Promise.resolve( + envelope(request, { + model: TodoModel, + actualTitle: 'server projection' + }) + ); + } + }, + { change: artifact({ model: TodoModel }) } + ); + const receipt = await runtime.commands.change( + { id: 'todo-1', title: 'optimistic' }, + { commandId: COMMAND_A } + ); + assert.equal(receipt.metadata.expects[0].model, TodoModel.id); + assert.equal(watch.get().data.todos[0].title, 'server projection'); + const assertOverlayRetained = () => { + assert.equal(watch.get().data.todos[0].title, 'server projection'); + assert.throws(() => replica.createOptimisticLayer(COMMAND_A, () => undefined)); + }; + + // A complete base frame without causal evidence must not retire the layer; + // the accepted projection remains the visible overlay. + liveObserver.next( + commandFrame( + CommandTodos, + [{ id: 'todo-1', title: 'base update', status: 'open' }], + { source: 'live', position: '2' } + ) + ); + assertOverlayRetained(); + + // Wrong-scope evidence is ignored even though the frame is otherwise fresh. + liveObserver.next( + commandFrame( + CommandTodos, + [{ id: 'todo-1', title: 'wrong observation', status: 'open' }], + { + source: 'live', + position: '3', + observations: [ + obligationObservation(receipt, { + scopeToken: token('projection-obligation', 99) + }) + ] + } + ) + ); + assertOverlayRetained(); + + // Causation identity is part of the proof; a different command cannot + // retire this command's accepted projection. + liveObserver.next( + commandFrame( + CommandTodos, + [{ id: 'todo-1', title: 'wrong causation', status: 'open' }], + { + source: 'live', + position: '4', + observations: [ + obligationObservation(receipt, { + causationId: 'cause:other-command' + }) + ] + } + ) + ); + assertOverlayRetained(); + + // Projection identity is equally strict; evidence from another projector + // is not an observation of this obligation. + liveObserver.next( + commandFrame( + CommandTodos, + [{ id: 'todo-1', title: 'wrong projection', status: 'open' }], + { + source: 'live', + position: '5', + observations: [ + obligationObservation(receipt, { + projection: 'other-projector' + }) + ] + } + ) + ); + assertOverlayRetained(); + + // A frame with a lower record revision but no causal observation remains + // unable to retire the layer. This intentionally tests only missing proof; + // it does not establish whether a matching observation is subject to record + // revision gating. + liveObserver.next( + commandFrame( + CommandTodos, + [{ id: 'todo-1', title: 'insufficient revision', status: 'open' }], + { + source: 'live', + position: '4', + recordRevision: '1' + } + ) + ); + assertOverlayRetained(); + + // Likewise, an incarnation marker without a causal observation is not proof. + // This does not assert a revision or incarnation fence on a matching proof. + liveObserver.next( + commandFrame( + CommandTodos, + [{ id: 'todo-1', title: 'stale incarnation', status: 'open' }], + { + source: 'live', + position: '5', + recordRevision: '5', + incarnation: '0' + } + ) + ); + assertOverlayRetained(); + + // Exact causal observation from the active comparable live frame retires the + // accepted layer. Its proof is the protocol identity tuple; record-clock + // reconciliation is a separate cache concern. + liveObserver.next( + commandFrame( + CommandTodos, + [{ id: 'todo-1', title: 'confirmed', status: 'open' }], + { + source: 'live', + position: '6', + recordRevision: '6', + observations: [obligationObservation(receipt)] + } + ) + ); + assert.equal(watch.get().data.todos[0].title, 'confirmed'); + assert.equal(replica.inspectRecord(TodoModel, 'todo-1').revision, '6'); + assert.equal((await receipt.projected).state, 'atomic'); + runtime.dispose(); + watch.destroy(); +}); + test('artifact v1 is rejected at the public boundary', () => { assert.throws( () => diff --git a/src/graphql/projection_delta/tests.rs b/src/graphql/projection_delta/tests.rs index 132ed426..c55253b8 100644 --- a/src/graphql/projection_delta/tests.rs +++ b/src/graphql/projection_delta/tests.rs @@ -1030,6 +1030,175 @@ fn zero_occurrence_metadata_is_classified_from_the_current_causal_command_contra assert_eq!(draining_command["expects"], serde_json::json!([])); } +#[cfg(feature = "graphql")] +fn wait_path_fixture() -> ( + crate::graphql::protocol::ProtocolResponseAccumulator, + crate::command::TypedCommandContract, + crate::OutboxMessage, +) { + use std::sync::Arc; + + use super::runtime::{ProtocolProjectionProgramRegistry, ProtocolProjectionRequestSeed}; + use crate::graphql::protocol::{ + DistributedEnvelopeV1, ProtocolResponseAccumulator, ProtocolTokenCodec, + ProtocolTokenPurpose, + }; + + let fixture = modeled_fixture( + ProjectionBindingState::Active, + ProjectionExecutionClass::Causal, + ); + let registry = + Arc::new(ProtocolProjectionProgramRegistry::try_from_surface(&fixture.surface).unwrap()); + let now_unix_ms = std::time::SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + let seed = ProtocolProjectionRequestSeed::new( + selected_export(&fixture.surface), + registry, + crate::command_ledger::PrincipalPartitionId::new("wait-path-principal").unwrap(), + "wait-path-generation", + Vec::new(), + now_unix_ms, + ) + .unwrap(); + let codec = ProtocolTokenCodec::new([0x70; 32]); + let cache_scope = codec + .issue(ProtocolTokenPurpose::CacheScope, &("wait-path", "cache")) + .unwrap(); + let accumulator = ProtocolResponseAccumulator::new( + DistributedEnvelopeV1::new("sha256:wait-path", "wait-path-auth", cache_scope, None), + codec, + ); + accumulator.bind_projection_request(seed).unwrap(); + let occurrence = state_occurrence(13, "todo-wait-path", "wait-path"); + let event = crate::OutboxMessage::from_domain_event_occurrence(&occurrence).unwrap(); + let contract = crate::command::typed_command::< + ModeledCommandInput, + crate::command::Eventual, + >(TEST_COMMAND_NAME) + .roles(["delta-user"]) + .emits(crate::command::__command_projection_events([Ok( + event_descriptor(), + )])) + .into_contract(); + (accumulator, contract, event) +} + +#[test] +#[cfg(feature = "graphql")] +fn wait_path_sealing_retains_event_obligations_and_status_stays_pending_without_observation() { + use crate::command::CommandConsistency; + use crate::command_ledger::CommandLedgerState; + use crate::microsvc::{ + CausalCommandPublicState, CausalCommandReceiptSource, CausalDispatchResult, + }; + + let (protocol, contract, event) = wait_path_fixture(); + let result = CausalDispatchResult { + payload: json!({"accepted": true}), + receipt: CausalCommandReceiptSource { + command_id: "wait-path-command".into(), + command_name: String::new(), + causation_id: String::new(), + consistency: CommandConsistency::Eventual, + state: CommandLedgerState::Succeeded, + outcome: json!({"accepted": true}), + obligations: Vec::new(), + projection_metadata: None, + direct_projection: None, + }, + projection_events: vec![event], + }; + + let result = result + .seal_wait_path_protocol(&protocol, &contract, Duration::from_secs(60)) + .unwrap(); + let metadata = result + .receipt + .projection_metadata + .as_ref() + .expect("an emitted modeled event must retain its exact metadata"); + assert_eq!(metadata.obligations.len(), 1); + assert_eq!(metadata.obligations[0].projection_ref, 0); + assert_eq!(metadata.obligations[0].model, "TodoView"); + assert!(metadata.obligations[0] + .scope_token + .as_str() + .starts_with("v1.projection-obligation.")); + assert_eq!(result.receipt.state, CommandLedgerState::Succeeded); + + let status = result.public_status(); + assert_eq!(status.state, CausalCommandPublicState::Succeeded); + assert!(status.evidence.is_empty()); + assert_eq!( + status + .projection_metadata + .as_ref() + .expect("status must carry modeled metadata") + .obligations + .len(), + 1 + ); + + protocol.record_status(&status).unwrap(); + let command = serde_json::to_value(protocol.snapshot().unwrap()).unwrap()["command"].clone(); + assert_eq!(command["state"], "succeeded"); + assert_eq!(command["expects"].as_array().unwrap().len(), 1); + assert!(command.get("observations").is_none()); +} + +#[test] +#[cfg(feature = "graphql")] +fn wait_path_sealing_keeps_atomic_and_unselected_commands_without_metadata() { + use crate::command::CommandConsistency; + use crate::command_ledger::CommandLedgerState; + use crate::microsvc::{CausalCommandReceiptSource, CausalDispatchResult}; + + let (protocol, contract, event) = wait_path_fixture(); + let result_for = |contract: crate::command::TypedCommandContract, + events: Vec| + -> CausalDispatchResult { + CausalDispatchResult { + payload: json!({"accepted": true}), + receipt: CausalCommandReceiptSource { + command_id: "wait-path-bypass".into(), + command_name: String::new(), + causation_id: String::new(), + consistency: CommandConsistency::Eventual, + state: CommandLedgerState::Succeeded, + outcome: json!({"accepted": true}), + obligations: Vec::new(), + projection_metadata: None, + direct_projection: None, + }, + projection_events: events, + } + .seal_wait_path_protocol(&protocol, &contract, Duration::from_secs(60)) + .unwrap() + }; + + let mut atomic_contract = contract.clone(); + atomic_contract.consistency = CommandConsistency::Atomic; + let atomic = result_for(atomic_contract, vec![event.clone()]); + assert_eq!(atomic.receipt.consistency, CommandConsistency::Atomic); + assert!(atomic.receipt.projection_metadata.is_none()); + assert_eq!(atomic.receipt.state, CommandLedgerState::Succeeded); + + let mut unselected_contract = contract.clone(); + unselected_contract.projections.selectors.clear(); + let unselected = result_for(unselected_contract, vec![event.clone()]); + assert_eq!(unselected.receipt.consistency, CommandConsistency::Eventual); + assert!(unselected.receipt.projection_metadata.is_none()); + assert_eq!(unselected.receipt.state, CommandLedgerState::Succeeded); + + let no_events = result_for(contract, Vec::new()); + assert_eq!(no_events.receipt.consistency, CommandConsistency::Eventual); + assert!(no_events.receipt.projection_metadata.is_none()); + assert_eq!(no_events.receipt.state, CommandLedgerState::Succeeded); +} + #[test] #[cfg(feature = "graphql")] fn opaque_fallback_hint_does_not_hide_empty_draining_modeled_work() { diff --git a/src/microsvc/service/causal.rs b/src/microsvc/service/causal.rs index fa4e3ec6..2cefd280 100644 --- a/src/microsvc/service/causal.rs +++ b/src/microsvc/service/causal.rs @@ -311,30 +311,12 @@ impl CausalDispatchResult { .map_err(|error| { CausalDispatchError::Internal(format!("wait-path projection metadata: {error}")) })?; - // Cell wait-path has no GraphQL command-ledger observations. Keep the - // modeled delta so the replica can apply it, but drop expects so - // `projected` does not wait on live/status observations this process - // cannot emit. Preserve the delta's own recovery disposition: a fully - // resolved actual delta is sufficient local authority and must not turn - // every successful cell command into a full-query revalidation. - let metadata = if metadata.obligations.is_empty() { - metadata - } else { - let revalidate = metadata.revalidate; - crate::graphql::protocol::CommandProjectionMetadataV1::try_new( - metadata.issued_at_unix_ms, - metadata.expires_at_unix_ms, - metadata.delta, - metadata.lifecycle_proofs, - Vec::new(), - revalidate, - ) - .map_err(|error| { - CausalDispatchError::Internal(format!( - "wait-path projection metadata without ledger observations: {error}" - )) - })? - }; + // The cell has durably committed the domain event, but the modeled + // read-model projector is still asynchronous. Preserve the exact + // event-derived obligations so the client can retire its accepted + // optimistic layer only after a matching live/read observation. The + // wait-path has no command-ledger observation rows of its own; that + // affects status evidence below, not the modeled obligation contract. self.receipt.state = CommandLedgerState::Succeeded; self.receipt.projection_metadata = Some(metadata); Ok(self) @@ -356,24 +338,10 @@ impl CausalDispatchResult { CommandLedgerState::ProjectionFailed => CausalCommandPublicState::ProjectionFailed, CommandLedgerState::Expired => CausalCommandPublicState::Expired, }; - let evidence = self - .receipt - .projection_metadata - .as_ref() - .map(|metadata| { - metadata - .obligations - .iter() - .enumerate() - .map(|(index, _)| CausalCommandProjectionEvidence { - obligation_index: index, - state: CausalProjectionEvidenceState::Observed, - incarnation: None, - revision: None, - }) - .collect() - }) - .unwrap_or_default(); + // A wait-path receipt proves the cell command commit, not asynchronous + // projector application. Keep modeled obligations in the metadata, but + // never claim them observed without a read/live proof. + let evidence = Vec::new(); CausalCommandPublicStatus { state, command_id: self.receipt.command_id.clone(), From 1515691a0ebeb890955cf8a0eb858e38cf6ee9cb Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 9 Sep 2026 19:19:20 -0500 Subject: [PATCH 02/10] fix: bind causal observations to authored program --- migrations/inventory.json | 12 + .../0007_projection_program_identity.sql | 5 + .../0007_projection_program_identity.sql | 5 + src/command/direct_projection.rs | 4 + src/command_ledger/tests.rs | 2 + src/graphql/projection_delta/runtime.rs | 8 + src/graphql/projection_delta/tests.rs | 287 +++++++++++ src/graphql/protocol/tests.rs | 2 + src/graphql/query_protocol.rs | 477 ++++++++++++++++-- src/graphql/surface/tests.rs | 99 +++- .../projection_protocol/direct_projection.rs | 6 + .../projection_protocol/rebuild.rs | 1 + .../projection_protocol/state.rs | 1 + .../projection_protocol/state_impl.rs | 1 + .../projection_protocol/store_impl.rs | 14 + .../projection_protocol/tests.rs | 1 + src/microsvc/projector/runtime.rs | 3 +- src/projection/catalog.rs | 87 ++++ src/projection_protocol/store/identity.rs | 13 + src/projection_protocol/store/mod.rs | 1 + src/projection_protocol/store/query.rs | 7 + src/projection_protocol/store/replay.rs | 26 + src/projection_protocol/store/tests.rs | 2 + src/projection_protocol/workspace.rs | 21 +- src/sqlx_repo/projection_protocol/helpers.rs | 14 + src/sqlx_repo/projection_protocol/mod.rs | 1 + src/sqlx_repo/projection_protocol/reads.rs | 22 +- src/sqlx_repo/projection_protocol/rebuild.rs | 1 + .../projection_protocol/store_impl.rs | 6 + src/sqlx_repo/projection_protocol/tests.rs | 138 ++++- src/sqlx_repo/projection_protocol/writes.rs | 38 +- src/sqlx_repo/repo/backend.rs | 18 +- 32 files changed, 1254 insertions(+), 69 deletions(-) create mode 100644 migrations/postgres/0007_projection_program_identity.sql create mode 100644 migrations/sqlite/0007_projection_program_identity.sql diff --git a/migrations/inventory.json b/migrations/inventory.json index ccea0395..9e2e9add 100644 --- a/migrations/inventory.json +++ b/migrations/inventory.json @@ -72,6 +72,18 @@ "path": "migrations/postgres/0006_gateway_dependency_versions.sql", "sha256": "c156bc51dddecd49b1beef7bb4069557b024c19cb1271f0ec9cc0efe71939471" } + }, + { + "version": 7, + "description": "projection program identity", + "sqlite": { + "path": "migrations/sqlite/0007_projection_program_identity.sql", + "sha256": "cb247b632ce6b1dbab2bf3d15a25532334432ef58d20755c5491b009bd31a320" + }, + "postgres": { + "path": "migrations/postgres/0007_projection_program_identity.sql", + "sha256": "ad7da74a11637a522279a6bc061d2c79db1f807f869368cf31735a089dfe0204" + } } ] } diff --git a/migrations/postgres/0007_projection_program_identity.sql b/migrations/postgres/0007_projection_program_identity.sql new file mode 100644 index 00000000..e730271e --- /dev/null +++ b/migrations/postgres/0007_projection_program_identity.sql @@ -0,0 +1,5 @@ +-- Preserve the semantic program that authored each causal proof row. +-- NULL is intentional for history written before program identities existed; +-- readers must treat it as unversioned rather than infer the active program. +ALTER TABLE projection_changes ADD COLUMN program_id text; +ALTER TABLE projection_observations ADD COLUMN program_id text; diff --git a/migrations/sqlite/0007_projection_program_identity.sql b/migrations/sqlite/0007_projection_program_identity.sql new file mode 100644 index 00000000..27d53972 --- /dev/null +++ b/migrations/sqlite/0007_projection_program_identity.sql @@ -0,0 +1,5 @@ +-- Preserve the semantic program that authored each causal proof row. +-- NULL is intentional for history written before program identities existed; +-- readers must treat it as unversioned rather than infer the active program. +ALTER TABLE projection_changes ADD COLUMN program_id TEXT; +ALTER TABLE projection_observations ADD COLUMN program_id TEXT; diff --git a/src/command/direct_projection.rs b/src/command/direct_projection.rs index e454b66f..ddc18f55 100644 --- a/src/command/direct_projection.rs +++ b/src/command/direct_projection.rs @@ -392,6 +392,10 @@ impl ResolvedDirectProjectionTarget { ) .map_err(|error| ProjectionProtocolError::InvalidBatch(error.to_string()))?; let ownership = ProjectionModelOwnership::new(self.model, self.table)?; + let ownership = self + .modeled_program_id + .map(|program_id| ownership.clone().with_program_id(program_id)) + .unwrap_or(ownership); SameTransactionProjectionBatch::single_upsert( self.codec.topology().clone(), self.partition, diff --git a/src/command_ledger/tests.rs b/src/command_ledger/tests.rs index a3ccd1bd..8bbb1151 100644 --- a/src/command_ledger/tests.rs +++ b/src/command_ledger/tests.rs @@ -136,6 +136,7 @@ fn direct_projection_evidence(marker: &str) -> SameTransactionProjectionEvidence scope: Some(scope.clone()), revision: Some(revision.clone()), failure_id: None, + program_id: None, }; let observation = ProjectionObservation { causation_id: format!("cause:{marker}"), @@ -143,6 +144,7 @@ fn direct_projection_evidence(marker: &str) -> SameTransactionProjectionEvidence revision: Some(revision), scope, change: cursor, + program_id: None, }; SameTransactionProjectionEvidence { records: vec![record], diff --git a/src/graphql/projection_delta/runtime.rs b/src/graphql/projection_delta/runtime.rs index 3c271f7b..c8001d3b 100644 --- a/src/graphql/projection_delta/runtime.rs +++ b/src/graphql/projection_delta/runtime.rs @@ -259,6 +259,14 @@ impl ProtocolProjectionRequestSeed { if candidate.causation_id != causation_id || candidate.scope.topology() != entry.codec.topology() || candidate.scope.model() != obligation.model + // A modeled observation is proof for the exact semantic + // program that authored it. Physical projector names are + // shared across deployments and cannot establish this + // identity after an upgrade or cold restart. Legacy + // observations have no program ID and remain readable, + // but cannot satisfy a modeled obligation. + || candidate.program_id.map(|id| id.to_string()) + != Some(identity.program_id.clone()) || !entry .binding .outputs() diff --git a/src/graphql/projection_delta/tests.rs b/src/graphql/projection_delta/tests.rs index c55253b8..bfd49d0f 100644 --- a/src/graphql/projection_delta/tests.rs +++ b/src/graphql/projection_delta/tests.rs @@ -1149,6 +1149,293 @@ fn wait_path_sealing_retains_event_obligations_and_status_stays_pending_without_ assert!(command.get("observations").is_none()); } +#[test] +#[cfg(feature = "graphql")] +fn wait_path_observation_proof_requires_the_persisted_semantic_program_identity() { + use std::sync::Arc; + + use crate::command::CommandConsistency; + use crate::command_ledger::CommandLedgerState; + use crate::graphql::protocol::{ProtocolTokenCodec, ProtocolTokenPurpose}; + use crate::microsvc::{ + CausalCommandProjectionEvidence, CausalCommandPublicState, CausalCommandPublicStatus, + CausalCommandReceiptSource, CausalDispatchResult, CausalProjectionEvidenceState, + }; + use crate::projection_protocol::{ + ProjectionCausationEvidenceBatch, ProjectionChangeCursor, ProjectionEpoch, + ProjectionObservation, ProjectionObservationKind, ProjectionScopeCodec, + ProjectorTopologyId, RecordRevision, + }; + use crate::ProjectionProgramId; + + let (protocol, contract, event) = wait_path_fixture(); + let result = CausalDispatchResult { + payload: json!({"accepted": true}), + receipt: CausalCommandReceiptSource { + command_id: "wait-path-identity-command".into(), + command_name: String::new(), + causation_id: String::new(), + consistency: CommandConsistency::Eventual, + state: CommandLedgerState::Succeeded, + outcome: json!({"accepted": true}), + obligations: Vec::new(), + projection_metadata: None, + direct_projection: None, + }, + projection_events: vec![event], + } + .seal_wait_path_protocol(&protocol, &contract, Duration::from_secs(60)) + .unwrap(); + let metadata = result + .receipt + .projection_metadata + .clone() + .expect("the real wait-path event produces modeled metadata"); + let semantic_program = ProjectionProgramId::parse(&metadata.delta.projections[0].program_id) + .expect("metadata carries the canonical semantic program identity"); + let wrong_program = ProjectionProgramId::parse( + "pp1:sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + ) + .unwrap(); + let topology = ProjectorTopologyId::new(1, "projection-delta-test", [0x44; 32]).unwrap(); + let todo_schema = todos(); + let user_schema = users(); + let observation_codec = ProjectionScopeCodec::with_models( + topology.clone(), + [("TodoView", &todo_schema), ("UserView", &user_schema)], + ) + .unwrap(); + let scope = observation_codec + .encode_row_scope( + "projection-delta-test", + "TodoView", + Some(&json!("tenant-a")), + &crate::table::RowKey::new([( + "todo_id", + crate::table::RowValue::String("todo-wait-path".into()), + )]), + ) + .unwrap(); + let revision = RecordRevision::new(scope.clone(), 1, 1).unwrap(); + let change = ProjectionChangeCursor::new( + topology, + scope.projection_partition().clone(), + ProjectionEpoch::new("projection-delta-v1").unwrap(), + 1, + ) + .unwrap(); + let candidate = ProjectionObservation { + causation_id: result.causation_id().into(), + kind: ProjectionObservationKind::Record, + revision: Some(revision), + scope: scope.clone(), + change, + program_id: Some(semantic_program), + }; + let disposition = protocol + .modeled_projection_evidence_topologies(TEST_COMMAND_NAME, result.causation_id(), &metadata) + .unwrap() + .disposition; + let evidence = protocol + .modeled_projection_evidence( + TEST_COMMAND_NAME, + result.causation_id(), + &metadata, + &ProjectionCausationEvidenceBatch { + observations: vec![candidate.clone()], + terminal_failure_topologies: Vec::new(), + }, + disposition, + ) + .unwrap(); + assert!(matches!( + evidence.as_slice(), + [super::runtime::ModeledProjectionEvidence::Observed(observation)] + if observation.program_id == Some(semantic_program) + )); + assert_ne!( + candidate.scope.topology().name(), + semantic_program.to_string(), + "the physical projector name and semantic author identity are distinct" + ); + + let mut mismatched_program = candidate.clone(); + mismatched_program.program_id = Some(wrong_program); + let pending_for = |observation: ProjectionObservation| { + protocol + .modeled_projection_evidence( + TEST_COMMAND_NAME, + result.causation_id(), + &metadata, + &ProjectionCausationEvidenceBatch { + observations: vec![observation], + terminal_failure_topologies: Vec::new(), + }, + disposition, + ) + .unwrap() + }; + assert!(matches!( + pending_for(mismatched_program).as_slice(), + [super::runtime::ModeledProjectionEvidence::Pending] + )); + let mut unversioned = candidate.clone(); + unversioned.program_id = None; + assert!(matches!( + pending_for(unversioned).as_slice(), + [super::runtime::ModeledProjectionEvidence::Pending] + )); + let mut wrong_causation = candidate.clone(); + wrong_causation.causation_id = FOREIGN_CAUSATION_ID.into(); + assert!(matches!( + pending_for(wrong_causation).as_slice(), + [super::runtime::ModeledProjectionEvidence::Pending] + )); + let wrong_scope = observation_codec + .encode_row_scope( + "projection-delta-test", + "UserView", + Some(&json!("tenant-a")), + &crate::table::RowKey::new([( + "user_id", + crate::table::RowValue::String("owner-secret".into()), + )]), + ) + .unwrap(); + let mut wrong_model = candidate.clone(); + wrong_model.scope = wrong_scope; + wrong_model.revision = None; + assert!(matches!( + pending_for(wrong_model).as_slice(), + [super::runtime::ModeledProjectionEvidence::Pending] + )); + + // A cold restart with the same mounted program can still prove the + // persisted observation. Replacing only the program behavior while keeping + // the physical projector name/topology does not: the semantic ID is the + // durable authoring boundary, not a current-name lookup. + let restarted_fixture = modeled_fixture_deployment( + ProjectionBindingState::Active, + ProjectionExecutionClass::Causal, + ProjectionMutationKind::Upsert, + false, + "delta-service", + "projection-delta-test", + [0x44; 32], + ); + let restart_codec = ProtocolTokenCodec::new([0x70; 32]); + let restart_cache = restart_codec + .issue(ProtocolTokenPurpose::CacheScope, &("wait-path", "cache")) + .unwrap(); + let restart_seed = super::runtime::ProtocolProjectionRequestSeed::new( + selected_export(&restarted_fixture.surface), + Arc::new( + super::runtime::ProtocolProjectionProgramRegistry::try_from_surface( + &restarted_fixture.surface, + ) + .unwrap(), + ), + crate::command_ledger::PrincipalPartitionId::new("wait-path-principal").unwrap(), + "wait-path-generation", + Vec::new(), + 1, + ) + .unwrap(); + let restart_disposition = restart_seed + .modeled_evidence_topologies( + &restart_codec, + &restart_cache, + TEST_COMMAND_NAME, + result.causation_id(), + &metadata, + ) + .unwrap() + .disposition; + assert!(matches!( + restart_seed + .modeled_evidence( + &restart_codec, + &restart_cache, + TEST_COMMAND_NAME, + result.causation_id(), + &metadata, + &ProjectionCausationEvidenceBatch { + observations: vec![candidate.clone()], + terminal_failure_topologies: Vec::new(), + }, + restart_disposition, + ) + .unwrap() + .as_slice(), + [super::runtime::ModeledProjectionEvidence::Observed(_)] + )); + let changed_fixture = modeled_fixture_deployment( + ProjectionBindingState::Active, + ProjectionExecutionClass::Causal, + ProjectionMutationKind::Patch, + false, + "delta-service", + "projection-delta-test", + [0x44; 32], + ); + assert_ne!( + restarted_fixture.program.id().unwrap(), + changed_fixture.program.id().unwrap(), + "the cold-restarted replacement must have a distinct semantic identity" + ); + let changed_seed = super::runtime::ProtocolProjectionRequestSeed::new( + selected_export(&changed_fixture.surface), + Arc::new( + super::runtime::ProtocolProjectionProgramRegistry::try_from_surface( + &changed_fixture.surface, + ) + .unwrap(), + ), + crate::command_ledger::PrincipalPartitionId::new("wait-path-principal").unwrap(), + "wait-path-generation", + Vec::new(), + 1, + ) + .unwrap(); + let changed_result = changed_seed.modeled_evidence_topologies( + &restart_codec, + &restart_cache, + TEST_COMMAND_NAME, + result.causation_id(), + &metadata, + ); + assert!( + changed_result.is_err(), + "changed program identity cannot reuse old metadata" + ); + + let status = CausalCommandPublicStatus { + state: CausalCommandPublicState::Succeeded, + command_id: result.command_id().into(), + command_name: Some(TEST_COMMAND_NAME.into()), + causation_id: Some(result.causation_id().into()), + consistency: Some(CommandConsistency::Eventual), + outcome: Some(json!({"accepted": true})), + obligations: Vec::new(), + projection_metadata: Some(metadata), + projection_revalidate: false, + evidence: vec![CausalCommandProjectionEvidence { + obligation_index: 0, + state: CausalProjectionEvidenceState::Observed, + incarnation: Some(1), + revision: Some(1), + }], + direct_projection: None, + }; + protocol.record_status(&status).unwrap(); + let command = serde_json::to_value(protocol.snapshot().unwrap()).unwrap()["command"].clone(); + assert_eq!(command["observations"].as_array().unwrap().len(), 1); + assert_eq!( + command["observations"][0]["causationId"], + result.causation_id() + ); +} + #[test] #[cfg(feature = "graphql")] fn wait_path_sealing_keeps_atomic_and_unselected_commands_without_metadata() { diff --git a/src/graphql/protocol/tests.rs b/src/graphql/protocol/tests.rs index 7ad481d6..6888bb3e 100644 --- a/src/graphql/protocol/tests.rs +++ b/src/graphql/protocol/tests.rs @@ -159,6 +159,7 @@ fn direct_projected_receipt() -> CausalCommandReceiptSource { scope: Some(scope.clone()), revision: Some(revision.clone()), failure_id: None, + program_id: None, }], observations: vec![ProjectionObservation { causation_id: receipt.causation_id.clone(), @@ -166,6 +167,7 @@ fn direct_projected_receipt() -> CausalCommandReceiptSource { revision: Some(revision), scope, change, + program_id: None, }], }); receipt diff --git a/src/graphql/query_protocol.rs b/src/graphql/query_protocol.rs index bb31a5e4..a07b49df 100644 --- a/src/graphql/query_protocol.rs +++ b/src/graphql/query_protocol.rs @@ -46,6 +46,13 @@ pub(crate) struct QueryProjectorRuntime { pub(crate) change_epoch: Option, models: BTreeSet, dependencies: BTreeSet, + /// Public causal-observation identity for each output model. Modeled + /// projections use their versioned program ID; legacy projections retain + /// the physical owner name as their public identity. + public_projection_by_model: BTreeMap, + /// Modeled changes must carry their authoring identity on the durable + /// change row. Legacy rows may derive the historical physical identity. + requires_semantic_identity: bool, } impl QueryProjectorRuntime { @@ -53,6 +60,12 @@ impl QueryProjectorRuntime { self.static_partition.is_some() && self.change_epoch.is_some() } + pub(crate) fn public_projection_for_model(&self, model: &str) -> Option<&str> { + self.public_projection_by_model + .get(model) + .map(String::as_str) + } + /// A partition-wide change cursor is safe to expose only when every model /// sharing that projector partition is visible without row filtering on /// this exact authorization surface. Otherwise positions, causations, and @@ -247,6 +260,13 @@ impl QueryProtocolRuntime { .get(model) .map(|owner| owner.static_partition.is_some()) } + + #[cfg(test)] + pub(crate) fn public_projection_for_model(&self, model: &str) -> Option<&str> { + self.model_owners + .get(model) + .and_then(|owner| owner.public_projection_for_model(model)) + } } fn compile_legacy_query_projector( @@ -324,6 +344,12 @@ fn compile_legacy_query_projector( change_epoch, models: projector.models.iter().cloned().collect(), dependencies: projector.dependencies.iter().cloned().collect(), + public_projection_by_model: projector + .models + .iter() + .map(|model| (model.clone(), projector.name.clone())) + .collect(), + requires_semantic_identity: false, })) } @@ -368,6 +394,7 @@ fn compile_modeled_query_projector( validate_modeled_partition_binding(projector, first_program, first_binding)?; let mut models = BTreeSet::new(); + let mut public_projection_by_model = BTreeMap::new(); for modeled in &active { let (program, binding) = modeled.raw().ok_or_else(|| { format!( @@ -405,7 +432,18 @@ fn compile_modeled_query_projector( projector.name )); } - models.extend(modeled.output_models().iter().cloned()); + for model in modeled.output_models() { + models.insert(model.clone()); + if public_projection_by_model + .insert(model.clone(), modeled.program_id().to_string()) + .is_some() + { + return Err(format!( + "query protocol modeled owner `{}` has ambiguous active program identity for model `{model}`", + projector.name + )); + } + } } if models.is_empty() { return Err(format!( @@ -475,6 +513,8 @@ fn compile_modeled_query_projector( change_epoch: Some(change_epoch), models, dependencies, + public_projection_by_model, + requires_semantic_identity: true, }))) } @@ -519,7 +559,10 @@ struct PreparedQueryEvidence { const MAX_PROTOCOL_EVIDENCE_ITEMS: usize = 4_096; struct PreparedLiveChange { + /// Physical owner used for change-log routing and opaque scope tokens. projection: String, + /// Versioned semantic identity used in public causal observations. + public_projection: Option, change: crate::projection_protocol::ProjectionChange, } @@ -1084,8 +1127,28 @@ where break; } replayed_changes.extend(changes.into_iter().map(|change| { + // Modeled identity is part of the persisted change + // evidence. Never relabel an old/null row from the + // currently active program after a cold restart. + let public_projection = change + .program_id + .map(|program_id| program_id.to_string()) + .or_else(|| { + if projector.requires_semantic_identity { + None + } else { + change + .scope + .as_ref() + .and_then(|scope| { + projector.public_projection_for_model(scope.model()) + }) + .map(str::to_owned) + } + }); PreparedLiveChange { projection: projector.name.clone(), + public_projection, change, } })); @@ -1269,24 +1332,33 @@ fn wire_query_snapshot( live_record_fences.insert(scope_key, (clock, wire_record)); } } - let observation = super::protocol::DistributedProjectionObservation { - causation_id: change.causation_id.clone(), - projection: live.projection.clone(), - model: scope.model().to_string(), - scope_token: accumulator - .issue_projection_obligation_scope( - &change.causation_id, - &live.projection, - scope.model(), - crate::projection_protocol::ProjectionObservationKind::Record, - scope, - ) - .map_err(|error| { - ProjectionProtocolError::InvalidBatch(error.to_string()) - })?, - }; - if observation_tokens.insert(observation.scope_token.as_str().to_string()) { - observations.push(observation); + // A retained pre-identity change still contributes its + // authoritative row/revision. It cannot, however, mint a + // modeled causal observation because its authoring program is + // unknown. Keep the readable data and omit only that proof. + if let Some(projection) = live.public_projection.clone() { + let observation = super::protocol::DistributedProjectionObservation { + causation_id: change.causation_id.clone(), + projection, + model: scope.model().to_string(), + scope_token: accumulator + .issue_projection_obligation_scope( + &change.causation_id, + &live.projection, + scope.model(), + crate::projection_protocol::ProjectionObservationKind::Record, + scope, + ) + .map_err(|error| { + ProjectionProtocolError::InvalidBatch(error.to_string()) + })?, + }; + if observation_tokens.insert(( + observation.projection.clone(), + observation.scope_token.as_str().to_string(), + )) { + observations.push(observation); + } } } crate::projection_protocol::ProjectionChangeKind::Observation => { @@ -1300,24 +1372,31 @@ fn wire_query_snapshot( "live projection observation omitted its kind".into(), ) })?; - let observation = super::protocol::DistributedProjectionObservation { - causation_id: change.causation_id.clone(), - projection: live.projection.clone(), - model: scope.model().to_string(), - scope_token: accumulator - .issue_projection_obligation_scope( - &change.causation_id, - &live.projection, - scope.model(), - kind, - scope, - ) - .map_err(|error| { - ProjectionProtocolError::InvalidBatch(error.to_string()) - })?, - }; - if observation_tokens.insert(observation.scope_token.as_str().to_string()) { - observations.push(observation); + // As with record changes, legacy/unversioned modeled history + // remains queryable but cannot be advertised as causal proof. + if let Some(projection) = live.public_projection.clone() { + let observation = super::protocol::DistributedProjectionObservation { + causation_id: change.causation_id.clone(), + projection, + model: scope.model().to_string(), + scope_token: accumulator + .issue_projection_obligation_scope( + &change.causation_id, + &live.projection, + scope.model(), + kind, + scope, + ) + .map_err(|error| { + ProjectionProtocolError::InvalidBatch(error.to_string()) + })?, + }; + if observation_tokens.insert(( + observation.projection.clone(), + observation.scope_token.as_str().to_string(), + )) { + observations.push(observation); + } } } crate::projection_protocol::ProjectionChangeKind::Checkpoint @@ -1375,4 +1454,328 @@ mod tests { assert!(query_index_budget_allows(MAX_PROTOCOL_EVIDENCE_ITEMS)); assert!(!query_index_budget_allows(MAX_PROTOCOL_EVIDENCE_ITEMS + 1)); } + + #[cfg(feature = "sqlite")] + #[tokio::test] + async fn wire_snapshot_preserves_persisted_modeled_identity_and_receipt_scope() { + use std::collections::{BTreeMap, BTreeSet}; + + use crate::graphql::protocol::{ProtocolTokenCodec, ProtocolTokenPurpose}; + use crate::projection_protocol::{ + ProjectionChangeRead, ProjectionCommitBatch, ProjectionEpoch, ProjectionInputCursor, + ProjectionInputFingerprint, ProjectionModelOwnership, ProjectionMutationKind, + ProjectionObservationKind, ProjectionObservationRequest, ProjectionObservationTarget, + ProjectionProtocolStore, ProjectionRecordExpectation, ProjectionRecordMutation, + ProjectionSource, TrustedProjectionInput, + }; + use crate::sqlite_repo::SqliteRepository; + use crate::table::{ + ColumnType, ExpectedVersion, PrimaryKey, RowKey, RowValue, RowValues, RowWriteMode, + TableColumn, TableKind, TableMutation, TableRowMutation, TableSchema, + TableSchemaRegistry, + }; + + let topology = ProjectorTopologyId::new(1, "query-wire-modeled", [0x4a; 32]).unwrap(); + static SCHEMA: std::sync::LazyLock = + std::sync::LazyLock::new(|| TableSchema { + model_name: "WireView".into(), + table_name: "wire_views".into(), + columns: vec![ + TableColumn { + primary_key: true, + ..TableColumn::new("id", "id", ColumnType::Text) + }, + TableColumn::new("title", "title", ColumnType::Text), + ], + primary_key: PrimaryKey::new(["id"]), + version_column: Some(crate::table::DEFAULT_TABLE_VERSION_COLUMN.into()), + foreign_keys: Vec::new(), + indexes: Vec::new(), + relationships: Vec::new(), + kind: TableKind::ReadModel, + }); + let schema = &*SCHEMA; + let mut table_registry = TableSchemaRegistry::new(); + table_registry.register_schema(schema.clone()).unwrap(); + let repository = SqliteRepository::connect_and_migrate("sqlite::memory:") + .await + .unwrap(); + repository + .bootstrap_table_schema_for_dev(&table_registry) + .await + .unwrap(); + let program_id = + crate::ProjectionProgramId::parse(&format!("pp1:sha256:{}", "4".repeat(64))).unwrap(); + let ownership = ProjectionModelOwnership::new("WireView", "wire_views") + .unwrap() + .with_program_id(program_id); + repository + .register_projection_models(&topology, std::slice::from_ref(&ownership)) + .await + .unwrap(); + + let codec = Arc::new( + ProjectionScopeCodec::with_models(topology.clone(), [("WireView", schema)]).unwrap(), + ); + let partition = codec.encode_partition(None).unwrap(); + let key = RowKey::new([("id", RowValue::String("wire-1".into()))]); + let scope = codec + .encode_row_scope_in_partition("WireView", partition.clone(), &key) + .unwrap(); + let mut values = RowValues::new(); + values.insert("id", RowValue::String("wire-1".into())); + values.insert("title", RowValue::String("persisted".into())); + let mutation = TableMutation::UpsertRow(TableRowMutation { + schema: &schema, + key: key.clone(), + values, + expected_version: ExpectedVersion::Any, + mode: RowWriteMode::Upsert, + }); + let change_epoch = ProjectionEpoch::new("query-wire-modeled-v1").unwrap(); + let input = TrustedProjectionInput::mint( + ProjectionInputCursor::new( + topology.clone(), + partition.clone(), + ProjectionSource::new("wire-source", b"wire-partition".to_vec()).unwrap(), + ProjectionEpoch::new("wire-source-v1").unwrap(), + 1, + ) + .unwrap(), + ProjectionInputFingerprint::from_canonical_bytes(b"wire-input"), + "wire-message-1", + "wire-cause-1", + crate::projection_protocol::ProjectionGeneration::initial(), + true, + ) + .unwrap(); + let commit = repository + .commit_projection(ProjectionCommitBatch { + input, + change_epoch: change_epoch.clone(), + ownership: vec![ownership], + mutations: vec![ProjectionRecordMutation::new( + scope.clone(), + mutation, + ProjectionRecordExpectation::Missing, + ProjectionMutationKind::Upsert, + ) + .unwrap()], + observations: vec![ProjectionObservationRequest { + kind: ProjectionObservationKind::Record, + target: ProjectionObservationTarget::StagedRecord(scope.clone()), + }], + }) + .await + .unwrap(); + let stored_change = match repository + .projection_changes(&topology, &partition, None, 16) + .await + .unwrap() + { + ProjectionChangeRead::Changes { changes, .. } => changes + .into_iter() + .find(|change| { + change.kind == crate::projection_protocol::ProjectionChangeKind::RecordUpsert + }) + .expect("stored record change"), + other => panic!("unexpected projection change read: {other:?}"), + }; + assert_eq!(stored_change.program_id, Some(program_id)); + assert!(commit + .changes + .iter() + .any(|change| change.program_id == Some(program_id))); + + let cache_scope = ProtocolTokenCodec::new([0x4b; 32]) + .issue(ProtocolTokenPurpose::CacheScope, &("wire", "cache")) + .unwrap(); + let token_codec = ProtocolTokenCodec::new([0x4b; 32]); + let accumulator = ProtocolResponseAccumulator::new( + super::super::protocol::DistributedEnvelopeV1::new( + "sha256:wire-schema", + "wire-auth", + cache_scope, + None, + ), + token_codec, + ); + accumulator + .bind_query_snapshot_scope(&serde_json::json!({ + "sql": "SELECT wire_views", + "tables": ["wire_views"] + })) + .unwrap(); + + let runtime = Arc::new(QueryProjectorRuntime { + name: "wire-physical-projector".into(), + codec: Arc::clone(&codec), + static_partition: Some(partition.clone()), + change_epoch: Some(change_epoch.clone()), + models: BTreeSet::from(["WireView".into()]), + dependencies: BTreeSet::from(["wire_views".into()]), + public_projection_by_model: BTreeMap::from([( + "WireView".into(), + program_id.to_string(), + )]), + requires_semantic_identity: true, + }); + let snapshot_scope = accumulator.query_snapshot_scope().unwrap(); + let initial_cursor = accumulator + .issue_live_resume_position( + &runtime.name, + &snapshot_scope, + runtime.codec.topology(), + &partition, + &change_epoch, + 0, + ) + .unwrap(); + let partition_snapshot = ProjectionPartitionSnapshot { + head: Some(stored_change.cursor.clone()), + compacted_through: 0, + }; + let prepared_live = PreparedQueryEvidence { + records_complete: true, + records: Vec::new(), + indexes: QueryIndexPlan { + comparable: true, + projectors: vec![Arc::clone(&runtime)], + }, + }; + let mut connection = repository.pool().acquire().await.unwrap(); + let live_metadata = wire_live_metadata::( + &mut *connection, + &accumulator, + &prepared_live, + std::slice::from_ref(&partition_snapshot), + RequestedLiveResume::Cursors(vec![initial_cursor.clone()]), + ) + .await + .unwrap(); + assert_eq!(live_metadata.metadata.mode, DistributedLiveMode::Resumable); + assert!(!live_metadata.metadata.reset); + assert_eq!(live_metadata.changes.len(), 1); + assert_eq!( + live_metadata.changes[0].public_projection.as_deref(), + Some(program_id.to_string().as_str()) + ); + let live_changes = live_metadata.changes; + drop(connection); + let record = repository + .projection_record(&scope) + .await + .unwrap() + .expect("stored record metadata"); + let prepared = PreparedQueryEvidence { + records_complete: true, + records: vec![PreparedRecordProbe { + request: ProjectionLiveRecordRequest::new(&codec, "WireView", key).unwrap(), + paths: vec![vec!["wire_views".into(), "0".into()]], + }], + indexes: QueryIndexPlan { + comparable: true, + projectors: vec![Arc::clone(&runtime)], + }, + }; + let snapshot = wire_query_snapshot( + &accumulator, + prepared, + vec![Some(record)], + vec![partition_snapshot.clone()], + live_changes, + ) + .unwrap(); + assert_eq!( + snapshot.observations.len(), + 1, + "modeled proof is on the wire" + ); + let observation = &snapshot.observations[0]; + assert_eq!(observation.causation_id, "wire-cause-1"); + assert_eq!(observation.projection, program_id.to_string()); + assert_eq!(observation.model, "WireView"); + + // A sealed command receipt uses the same physical scope material and + // semantic public label. The opaque token must be byte-for-byte equal + // to the live observation, while the physical owner stays private. + let receipt_scope = accumulator + .issue_projection_obligation_scope( + "wire-cause-1", + &runtime.name, + "WireView", + ProjectionObservationKind::Record, + &scope, + ) + .unwrap(); + assert_eq!(observation.scope_token, receipt_scope); + + // Nulling the durable identity models pre-migration history: the row + // and record fence remain readable, but the wire must not relabel it + // as the currently active semantic program. + sqlx::query("UPDATE projection_changes SET program_id = NULL WHERE change_position = ?") + .bind(stored_change.cursor.position() as i64) + .execute(repository.pool()) + .await + .unwrap(); + let legacy_change = match repository + .projection_changes(&topology, &partition, None, 16) + .await + .unwrap() + { + ProjectionChangeRead::Changes { changes, .. } => changes + .into_iter() + .find(|change| { + change.kind == crate::projection_protocol::ProjectionChangeKind::RecordUpsert + }) + .expect("stored legacy record change"), + other => panic!("unexpected legacy projection change read: {other:?}"), + }; + assert!(legacy_change.program_id.is_none()); + let mut connection = repository.pool().acquire().await.unwrap(); + let legacy_live = wire_live_metadata::( + &mut *connection, + &accumulator, + &prepared_live, + std::slice::from_ref(&partition_snapshot), + RequestedLiveResume::Cursors(vec![initial_cursor]), + ) + .await + .unwrap(); + assert_eq!(legacy_live.changes.len(), 1); + assert!(legacy_live.changes[0].public_projection.is_none()); + let legacy_changes = legacy_live.changes; + drop(connection); + let legacy_snapshot = wire_query_snapshot( + &accumulator, + PreparedQueryEvidence { + records_complete: true, + records: vec![PreparedRecordProbe { + request: ProjectionLiveRecordRequest::new( + &codec, + "WireView", + RowKey::new([("id", RowValue::String("wire-1".into()))]), + ) + .unwrap(), + paths: vec![vec!["wire_views".into(), "0".into()]], + }], + indexes: QueryIndexPlan { + comparable: true, + projectors: vec![runtime], + }, + }, + vec![Some( + repository + .projection_record(&scope) + .await + .unwrap() + .expect("legacy row metadata remains readable"), + )], + vec![partition_snapshot], + legacy_changes, + ) + .unwrap(); + assert_eq!(legacy_snapshot.observations, Vec::new()); + assert_eq!(legacy_snapshot.records.len(), 1); + } } diff --git a/src/graphql/surface/tests.rs b/src/graphql/surface/tests.rs index 5506b20c..1fa560d3 100644 --- a/src/graphql/surface/tests.rs +++ b/src/graphql/surface/tests.rs @@ -109,6 +109,16 @@ fn modeled_direct_projection( epoch: &str, schema: TableSchema, options: DirectModeledBinding<'_>, +) -> SurfaceModeledProjection { + modeled_projection_with_outputs(owner, program_name, epoch, vec![schema], options) +} + +fn modeled_projection_with_outputs( + owner: &str, + program_name: &str, + epoch: &str, + schemas: Vec, + options: DirectModeledBinding<'_>, ) -> SurfaceModeledProjection { use crate::projection::catalog::{ProjectionBindingActivation, ProjectionCatalog}; use crate::projection::placement::{ @@ -126,8 +136,6 @@ fn modeled_direct_projection( DOMAIN_EVENT_BODY_CODEC, DOMAIN_EVENT_BODY_CODEC_VERSION, }; - let model_name = schema.model_name.clone(); - let table_name = schema.table_name.clone(); let selector = ProjectionEventSelector::try_new( 1, format!("{program_name}.changed"), @@ -141,18 +149,26 @@ fn modeled_direct_projection( DOMAIN_EVENT_BODY_CODEC_VERSION, ) .unwrap(); - let value = ProjectionExpression::constant(ProjectionValue::string("row-1")); - let operation = ProjectionOperation::try_new( - format!("{program_name}-upsert"), - 0, - ProjectionMutationKind::Upsert, - ProjectionTarget::try_new(&model_name, &table_name).unwrap(), - vec![ProjectionKeyField::try_new(0, "id", value.clone()).unwrap()], - vec![ProjectionField::try_new(0, "id", ProjectionAssignment::Set(value)).unwrap()], - Vec::new(), - Vec::new(), - ) - .unwrap(); + assert!(!schemas.is_empty(), "modeled projection requires an output"); + let operations = schemas + .iter() + .enumerate() + .map(|(index, schema)| { + let value = + ProjectionExpression::constant(ProjectionValue::string(format!("row-{index}"))); + ProjectionOperation::try_new( + format!("{program_name}-upsert-{index}"), + index.try_into().unwrap(), + ProjectionMutationKind::Upsert, + ProjectionTarget::try_new(&schema.model_name, &schema.table_name).unwrap(), + vec![ProjectionKeyField::try_new(0, "id", value.clone()).unwrap()], + vec![ProjectionField::try_new(0, "id", ProjectionAssignment::Set(value)).unwrap()], + Vec::new(), + Vec::new(), + ) + .unwrap() + }) + .collect::>(); let partition = if options.dynamic_partition { ProjectionPartition::Expression(ProjectionExpression::constant(ProjectionValue::string( "tenant-1", @@ -164,10 +180,7 @@ fn modeled_direct_projection( program_name, 1, partition, - vec![ - ProjectionArm::try_new(format!("{program_name}-arm"), selector, vec![operation]) - .unwrap(), - ], + vec![ProjectionArm::try_new(format!("{program_name}-arm"), selector, operations).unwrap()], ) .unwrap(); let descriptor = TestProjectionDescriptor(program.clone()); @@ -180,7 +193,13 @@ fn modeled_direct_projection( let source = ProjectionSourceBinding::try_new("test-domain", "ordered-domain-events", 1).unwrap(); let owner = ProjectionOwner::try_new(owner).unwrap(); - let outputs = vec![ProjectionOutput::try_new(model_name, table_name, schema).unwrap()]; + let outputs = schemas + .into_iter() + .map(|schema| { + ProjectionOutput::try_new(schema.model_name.clone(), schema.table_name.clone(), schema) + .unwrap() + }) + .collect::>(); let binding = if options.eventual { ProjectionBinding::from_eventual_program( &program, @@ -636,6 +655,7 @@ fn query_protocol_uses_exact_modeled_physical_topology() { ..DirectModeledBinding::active("exact-physical-topology") }, ); + let semantic_program_id = modeled.program_id().to_string(); let surface = build_surface(&[model], &SurfaceOptions::sqlite()) .unwrap() .with_projection_owners([SurfaceDirectProjection::new("exact-modeled-owner") @@ -653,6 +673,11 @@ fn query_protocol_uses_exact_modeled_physical_topology() { runtime.model_has_static_partition("ExactTopology"), Some(true) ); + assert_eq!( + runtime.public_projection_for_model("ExactTopology"), + Some(semantic_program_id.as_str()), + "causal observations use the active semantic program identity" + ); } #[cfg(feature = "graphql")] @@ -764,6 +789,42 @@ fn query_protocol_merges_compatible_active_models_without_inventing_static_parti } } +#[cfg(feature = "graphql")] +#[test] +fn query_protocol_preserves_one_semantic_identity_for_each_fanout_model() { + let first = direct_model("FirstFanout", "first_fanout"); + let second = direct_model("SecondFanout", "second_fanout"); + let modeled = modeled_projection_with_outputs( + "fanout-owner", + "fanout-program", + "fanout-v1", + vec![first.clone(), second.clone()], + DirectModeledBinding { + eventual: true, + ..DirectModeledBinding::active("fanout-physical-owner") + }, + ); + let semantic_program_id = modeled.program_id().to_string(); + let surface = build_surface(&[first, second], &SurfaceOptions::sqlite()) + .unwrap() + .with_projectors([SurfaceProjector::new("fanout-owner").modeled(modeled)]) + .unwrap(); + + let runtime = crate::graphql::query_protocol::QueryProtocolRuntime::compile(&surface).unwrap(); + for model in ["FirstFanout", "SecondFanout"] { + assert_eq!( + runtime.public_projection_for_model(model), + Some(semantic_program_id.as_str()), + "each fan-out model must retain the program identity that authored it" + ); + } + assert_eq!( + runtime.public_projection_for_model("UnknownModel"), + None, + "unknown models cannot mint a causal proof identity" + ); +} + #[test] fn selected_surfaces_reject_command_and_projector_reattachment() { let full = build_surface(&[orders()], &SurfaceOptions::sqlite()).unwrap(); diff --git a/src/in_memory_repo/projection_protocol/direct_projection.rs b/src/in_memory_repo/projection_protocol/direct_projection.rs index da90e0e8..2293d461 100644 --- a/src/in_memory_repo/projection_protocol/direct_projection.rs +++ b/src/in_memory_repo/projection_protocol/direct_projection.rs @@ -70,6 +70,11 @@ pub(in crate::in_memory_repo) fn stage_same_transaction_projection( scope: Some(mutation.scope.clone()), revision: Some(revision.clone()), failure_id: None, + program_id: batch + .ownership + .iter() + .find(|ownership| ownership.model == mutation.scope.model()) + .and_then(|ownership| ownership.program_id), }, )?; let metadata = ProjectionRecordMetadata { @@ -111,6 +116,7 @@ pub(in crate::in_memory_repo) fn stage_same_transaction_projection( revision: Some(revision), scope: mutation.scope.clone(), change: change.cursor.clone(), + program_id: change.program_id, }; protocol .observations diff --git a/src/in_memory_repo/projection_protocol/rebuild.rs b/src/in_memory_repo/projection_protocol/rebuild.rs index cde839ea..f7795e1e 100644 --- a/src/in_memory_repo/projection_protocol/rebuild.rs +++ b/src/in_memory_repo/projection_protocol/rebuild.rs @@ -91,6 +91,7 @@ impl InMemoryRepository { scope: Some(row.scope.clone()), revision: Some(revision.clone()), failure_id: None, + program_id: None, }, )?; let record = ProjectionRecordMetadata { diff --git a/src/in_memory_repo/projection_protocol/state.rs b/src/in_memory_repo/projection_protocol/state.rs index 2bf0f5cb..2ce23b50 100644 --- a/src/in_memory_repo/projection_protocol/state.rs +++ b/src/in_memory_repo/projection_protocol/state.rs @@ -228,6 +228,7 @@ pub(super) struct PendingChange { pub(super) scope: Option, pub(super) revision: Option, pub(super) failure_id: Option, + pub(super) program_id: Option, } impl PartitionState { diff --git a/src/in_memory_repo/projection_protocol/state_impl.rs b/src/in_memory_repo/projection_protocol/state_impl.rs index fbd37abc..383e8046 100644 --- a/src/in_memory_repo/projection_protocol/state_impl.rs +++ b/src/in_memory_repo/projection_protocol/state_impl.rs @@ -683,6 +683,7 @@ impl InMemoryProjectionProtocolState { scope: pending.scope, revision: pending.revision, failure_id: pending.failure_id, + program_id: pending.program_id, }; partition.change_head = position; partition.changes.insert(position, change.clone()); diff --git a/src/in_memory_repo/projection_protocol/store_impl.rs b/src/in_memory_repo/projection_protocol/store_impl.rs index 655ef9bf..be530e46 100644 --- a/src/in_memory_repo/projection_protocol/store_impl.rs +++ b/src/in_memory_repo/projection_protocol/store_impl.rs @@ -216,6 +216,11 @@ impl ProjectionProtocolStore for InMemoryRepository { scope: Some(mutation.scope.clone()), revision: Some(revision.clone()), failure_id: None, + program_id: batch + .ownership + .iter() + .find(|ownership| ownership.model == mutation.scope.model()) + .and_then(|ownership| ownership.program_id), }, )?; let metadata = ProjectionRecordMetadata { @@ -298,6 +303,11 @@ impl ProjectionProtocolStore for InMemoryRepository { if staged_protocol.observations.contains_key(&observation_key) { continue; } + let program_id = batch + .ownership + .iter() + .find(|ownership| ownership.model == scope.model()) + .and_then(|ownership| ownership.program_id); let change_cursor = match staged_change { Some(cursor) => cursor, None => { @@ -310,6 +320,7 @@ impl ProjectionProtocolStore for InMemoryRepository { scope: Some(scope.clone()), revision: revision.clone(), failure_id: None, + program_id, }, )?; let cursor = change.cursor.clone(); @@ -323,6 +334,7 @@ impl ProjectionProtocolStore for InMemoryRepository { revision, scope, change: change_cursor, + program_id, }; staged_protocol .observations @@ -339,6 +351,7 @@ impl ProjectionProtocolStore for InMemoryRepository { scope: None, revision: None, failure_id: None, + program_id: None, }, )?); } @@ -514,6 +527,7 @@ impl ProjectionProtocolStore for InMemoryRepository { scope: None, revision: None, failure_id: Some(batch.failure_id.clone()), + program_id: None, }, )?; let failure = ProjectionFailure { diff --git a/src/in_memory_repo/projection_protocol/tests.rs b/src/in_memory_repo/projection_protocol/tests.rs index e2f28c16..01754921 100644 --- a/src/in_memory_repo/projection_protocol/tests.rs +++ b/src/in_memory_repo/projection_protocol/tests.rs @@ -1794,6 +1794,7 @@ async fn lengthening_retention_never_restores_a_compacted_prefix() { scope: None, revision: None, failure_id: None, + program_id: None, }], } ); diff --git a/src/microsvc/projector/runtime.rs b/src/microsvc/projector/runtime.rs index c0b3f67f..26b39124 100644 --- a/src/microsvc/projector/runtime.rs +++ b/src/microsvc/projector/runtime.rs @@ -297,11 +297,12 @@ where } } let failure_input = trusted.clone(); - let workspace = ProjectionWorkspace::new( + let workspace = ProjectionWorkspace::new_with_program_id( self.compiled.codec(), partition_value, trusted, self.change_epoch.clone(), + Some(self.executor.program_id), )?; let (context, workspace) = CausalProjectorContext::new(message, D::Store::clone(store), workspace); diff --git a/src/projection/catalog.rs b/src/projection/catalog.rs index a3ffa248..e47437be 100644 --- a/src/projection/catalog.rs +++ b/src/projection/catalog.rs @@ -1808,6 +1808,93 @@ mod tests { )); } + #[test] + fn semantic_program_replacement_requires_epoch_rotation_before_replay() { + let previous_program = program_with_kind_and_partition( + "project_todos", + FINGERPRINT_A, + "Todos", + "todos", + ProjectionMutationKind::Upsert, + ProjectionPartition::Unit, + ); + let replacement_program = program_with_kind_and_partition( + "project_todos", + FINGERPRINT_A, + "Todos", + "todos", + ProjectionMutationKind::Patch, + ProjectionPartition::Unit, + ); + assert_ne!( + previous_program.id().unwrap(), + replacement_program.id().unwrap(), + "different projection behavior must have different semantic identities" + ); + let previous_binding = eventual( + &previous_program, + "todo-reads-v1", + ProjectionExecutionClass::Causal, + ); + let replacement_binding = eventual( + &replacement_program, + "todo-reads-v2", + ProjectionExecutionClass::Causal, + ); + assert_eq!( + previous_binding.physical_topology(), + replacement_binding.physical_topology(), + "the physical owner may remain stable while the program changes" + ); + + let previous_catalog = ProjectionCatalog::try_new(vec![previous_binding.clone()]).unwrap(); + let previous_active = previous_catalog + .activate( + vec![activation( + &previous_binding, + "todos-rebuild-1", + ProjectionBindingState::Active, + Some(ProjectionExecutorRoute::remote("projector-v1").unwrap()), + )], + None, + ) + .unwrap(); + let replacement_catalog = + ProjectionCatalog::try_new(vec![replacement_binding.clone()]).unwrap(); + let same_epoch = replacement_catalog + .activate( + vec![activation( + &replacement_binding, + "todos-rebuild-1", + ProjectionBindingState::Active, + Some(ProjectionExecutorRoute::remote("projector-v2").unwrap()), + )], + Some((&previous_catalog, &previous_active)), + ) + .unwrap_err(); + assert!(matches!( + same_epoch, + ProjectionCatalogError::SameEpochTakeover { .. } + )); + + let rotated = replacement_catalog + .activate( + vec![activation( + &replacement_binding, + "todos-rebuild-2", + ProjectionBindingState::Active, + Some(ProjectionExecutorRoute::remote("projector-v2").unwrap()), + )], + Some((&previous_catalog, &previous_active)), + ) + .unwrap(); + assert_eq!( + rotated.bindings()[0].program_id(), + replacement_program.id().unwrap() + ); + assert_eq!(rotated.bindings()[0].epoch().as_str(), "todos-rebuild-2"); + } + #[test] fn new_epoch_allows_rollout_then_draining_stops_new_obligations() { let program = program("project_todos", FINGERPRINT_A); diff --git a/src/projection_protocol/store/identity.rs b/src/projection_protocol/store/identity.rs index 14d55028..4f2aef32 100644 --- a/src/projection_protocol/store/identity.rs +++ b/src/projection_protocol/store/identity.rs @@ -191,6 +191,13 @@ pub(crate) enum ProjectionInputDisposition { pub(crate) struct ProjectionModelOwnership { pub(crate) model: String, pub(crate) table: String, + /// Semantic author identity for a commit carrying this ownership. + /// + /// Bootstrap declarations leave this unset. A modeled projector attaches + /// its immutable program identity when sealing a commit; the adapter + /// persists that identity on each change/observation rather than changing + /// an existing ownership row retroactively. + pub(crate) program_id: Option, } impl ProjectionModelOwnership { @@ -201,8 +208,14 @@ impl ProjectionModelOwnership { Ok(Self { model: bounded_name("projection model", model, 255)?, table: bounded_name("projection table", table, 255)?, + program_id: None, }) } + + pub(crate) fn with_program_id(mut self, program_id: ProjectionProgramId) -> Self { + self.program_id = Some(program_id); + self + } } /// Record state required before a staged mutation may apply. diff --git a/src/projection_protocol/store/mod.rs b/src/projection_protocol/store/mod.rs index 0ab5d9fb..f99901a1 100644 --- a/src/projection_protocol/store/mod.rs +++ b/src/projection_protocol/store/mod.rs @@ -31,6 +31,7 @@ use crate::repository::{InboxReceipt, RepositoryError}; use crate::table::{ RowKey, RowValues, TableMutation, TableSchema, TableStoreError, TableWritePlan, }; +use crate::ProjectionProgramId; mod backend_helpers; mod commit; diff --git a/src/projection_protocol/store/query.rs b/src/projection_protocol/store/query.rs index 03520749..86132e37 100644 --- a/src/projection_protocol/store/query.rs +++ b/src/projection_protocol/store/query.rs @@ -686,6 +686,10 @@ pub struct ProjectionObservation { /// Canonical dependency scope when no record revision exists. pub scope: ProjectionRecordScope, pub change: ProjectionChangeCursor, + /// Semantic program identity recorded by the projector that authored this + /// observation. `None` is legacy/unversioned evidence and cannot satisfy a + /// modeled causal obligation. + pub program_id: Option, } /// Durable terminal failure for one exact input and repair generation. @@ -753,6 +757,9 @@ pub struct ProjectionChange { pub scope: Option, pub revision: Option, pub failure_id: Option, + /// Semantic program identity recorded at commit time. It is deliberately + /// independent from the currently active projector binding. + pub program_id: Option, } /// Result and exact evidence produced by an asynchronous projection commit. diff --git a/src/projection_protocol/store/replay.rs b/src/projection_protocol/store/replay.rs index df6a257e..e04a8d17 100644 --- a/src/projection_protocol/store/replay.rs +++ b/src/projection_protocol/store/replay.rs @@ -162,6 +162,8 @@ struct ReplayChange { scope: Option, revision: Option, failure_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + program_id: Option, } impl From<&ProjectionChange> for ReplayChange { @@ -176,6 +178,7 @@ impl From<&ProjectionChange> for ReplayChange { scope: change.scope.as_ref().map(ReplayScope::from), revision: change.revision.as_ref().map(ReplayRevision::from), failure_id: change.failure_id.clone(), + program_id: change.program_id.map(|program_id| program_id.to_string()), } } } @@ -188,6 +191,8 @@ struct ReplayObservation { revision: Option, scope: ReplayScope, change: ReplayCursor, + #[serde(default, skip_serializing_if = "Option::is_none")] + program_id: Option, } impl From<&ProjectionObservation> for ReplayObservation { @@ -198,6 +203,9 @@ impl From<&ProjectionObservation> for ReplayObservation { revision: observation.revision.as_ref().map(ReplayRevision::from), scope: ReplayScope::from(&observation.scope), change: ReplayCursor::from(&observation.change), + program_id: observation + .program_id + .map(|program_id| program_id.to_string()), } } } @@ -316,6 +324,12 @@ impl ReplayChange { .map_err(|error| replay_validation_error("change failure ID", error)) }) .transpose()?, + program_id: self + .program_id + .as_deref() + .map(ProjectionProgramId::parse) + .transpose() + .map_err(|error| replay_validation_error("change program ID", error))?, }) } } @@ -336,6 +350,12 @@ impl ReplayObservation { .transpose()?, scope: self.scope.into_scope()?, change: self.change.into_cursor()?, + program_id: self + .program_id + .as_deref() + .map(ProjectionProgramId::parse) + .transpose() + .map_err(|error| replay_validation_error("observation program ID", error))?, }) } } @@ -468,6 +488,12 @@ fn validate_same_transaction_replay_evidence( "direct projection evidence observation does not match its record change".into(), ); } + if observation.program_id != change.program_id { + return Err( + "direct projection evidence observation and change program identities do not match" + .into(), + ); + } Ok(()) } diff --git a/src/projection_protocol/store/tests.rs b/src/projection_protocol/store/tests.rs index 0c7d6685..5ba7b1f2 100644 --- a/src/projection_protocol/store/tests.rs +++ b/src/projection_protocol/store/tests.rs @@ -55,6 +55,7 @@ fn same_transaction_evidence() -> SameTransactionProjectionEvidence { scope: Some(scope.clone()), revision: Some(revision.clone()), failure_id: None, + program_id: None, }], observations: vec![ProjectionObservation { causation_id: "cause-1".into(), @@ -62,6 +63,7 @@ fn same_transaction_evidence() -> SameTransactionProjectionEvidence { revision: Some(revision), scope, change, + program_id: None, }], } } diff --git a/src/projection_protocol/workspace.rs b/src/projection_protocol/workspace.rs index e6bdb9c2..fe7645af 100644 --- a/src/projection_protocol/workspace.rs +++ b/src/projection_protocol/workspace.rs @@ -21,6 +21,7 @@ use crate::table::{ DeleteTableRowMutation, ExpectedVersion, PatchMode, PatchTableRowMutation, RowKey, RowPatch, RowWriteMode, TableMutation, TableRowMutation, TableSchema, }; +use crate::ProjectionProgramId; /// Typed, commit-less workspace passed to a causal projector handler. /// @@ -33,6 +34,7 @@ pub struct ProjectionWorkspace { partition_value: Option, input: TrustedProjectionInput, change_epoch: ProjectionEpoch, + program_id: Option, ownership: BTreeMap, mutations: Vec, observations: Vec, @@ -45,6 +47,16 @@ impl ProjectionWorkspace { partition_value: Option, input: TrustedProjectionInput, change_epoch: ProjectionEpoch, + ) -> Result { + Self::new_with_program_id(codec, partition_value, input, change_epoch, None) + } + + pub(crate) fn new_with_program_id( + codec: Arc, + partition_value: Option, + input: TrustedProjectionInput, + change_epoch: ProjectionEpoch, + program_id: Option, ) -> Result { if codec.topology() != input.cursor.topology() { return Err(ProjectionProtocolError::ScopeMismatch { @@ -64,6 +76,7 @@ impl ProjectionWorkspace { partition_value, input, change_epoch, + program_id, ownership: BTreeMap::new(), mutations: Vec::new(), observations: Vec::new(), @@ -367,7 +380,13 @@ impl ProjectionWorkspace { ) -> Result, ProjectionProtocolError> { self.ownership .iter() - .map(|(model, table)| ProjectionModelOwnership::new(model.clone(), table.clone())) + .map(|(model, table)| { + let ownership = ProjectionModelOwnership::new(model.clone(), table.clone())?; + Ok(self + .program_id + .map(|program_id| ownership.clone().with_program_id(program_id)) + .unwrap_or(ownership)) + }) .collect() } diff --git a/src/sqlx_repo/projection_protocol/helpers.rs b/src/sqlx_repo/projection_protocol/helpers.rs index 175aafde..bfd7d1e0 100644 --- a/src/sqlx_repo/projection_protocol/helpers.rs +++ b/src/sqlx_repo/projection_protocol/helpers.rs @@ -104,3 +104,17 @@ pub(super) fn decode_observation_kind( ProjectionObservationKind::from_storage_str(value) .ok_or_else(|| corrupt_storage(format!("unknown projection observation kind `{value}`"))) } + +pub(super) fn decode_program_id( + value: Option, +) -> Result, ProjectionProtocolError> { + value + .map(|value| { + crate::ProjectionProgramId::parse(&value).map_err(|error| { + corrupt_storage(format!( + "invalid stored projection program identity: {error}" + )) + }) + }) + .transpose() +} diff --git a/src/sqlx_repo/projection_protocol/mod.rs b/src/sqlx_repo/projection_protocol/mod.rs index d27b7c6b..cf125660 100644 --- a/src/sqlx_repo/projection_protocol/mod.rs +++ b/src/sqlx_repo/projection_protocol/mod.rs @@ -51,6 +51,7 @@ use crate::table::{ validate_row_values, RowKey, RowValues, TableMutation, TableSchema, TableStoreError, TableWritePlan, }; +use crate::ProjectionProgramId; mod helpers; mod identity; diff --git a/src/sqlx_repo/projection_protocol/reads.rs b/src/sqlx_repo/projection_protocol/reads.rs index 1dfac6dd..7b31bfd0 100644 --- a/src/sqlx_repo/projection_protocol/reads.rs +++ b/src/sqlx_repo/projection_protocol/reads.rs @@ -176,6 +176,10 @@ where let failure_id: Option = row .try_get("failure_id") .map_err(|error| protocol_storage_error::("decode change failure ID", error))?; + let program_id = + decode_program_id(row.try_get("program_id").map_err(|error| { + protocol_storage_error::("decode change program identity", error) + })?)?; let scope = match (model_name, key_bytes, key_hash) { (Some(model), Some(bytes), Some(hash)) => { @@ -253,6 +257,7 @@ where scope, revision, failure_id, + program_id, }) } @@ -365,6 +370,11 @@ where Some(staged.scope.clone()), Some(revision.clone()), None, + batch + .ownership + .iter() + .find(|ownership| ownership.model == staged.scope.model()) + .and_then(|ownership| ownership.program_id), )?; let metadata = ProjectionRecordMetadata { source_snapshot: None, @@ -398,6 +408,11 @@ where scope: staged.scope.clone(), revision: Some(revision), change: change.cursor.clone(), + program_id: batch + .ownership + .iter() + .find(|ownership| ownership.model == staged.scope.model()) + .and_then(|ownership| ownership.program_id), }; apply_read_model_write_plan_in_tx(tx, write_plan).await?; @@ -1317,7 +1332,8 @@ where observation.scope_kind AS evidence_scope_kind, \ observation.canonical_key_bytes, observation.canonical_key_hash, \ observation.incarnation, observation.revision, observation.change_epoch, \ - observation.change_position, partition.topology_bytes AS evidence_topology_bytes, \ + observation.change_position, observation.program_id, \ + partition.topology_bytes AS evidence_topology_bytes, \ partition.partition_bytes AS evidence_partition_bytes, \ partition.change_epoch AS evidence_partition_epoch, \ partition.change_head AS evidence_partition_head \ @@ -1624,7 +1640,7 @@ where observation.causation_id, observation.model_name, observation.scope_kind, \ observation.canonical_key_bytes, observation.canonical_key_hash, \ observation.incarnation, observation.revision, observation.change_epoch, \ - observation.change_position, partition.topology_bytes, \ + observation.change_position, observation.program_id, partition.topology_bytes, \ partition.partition_bytes, partition.change_epoch AS partition_change_epoch, \ partition.change_head AS partition_change_head \ FROM projection_observations observation \ @@ -2258,7 +2274,7 @@ where let mut builder = QueryBuilder::::new( "SELECT change_epoch, change_position, change_kind, causation_id, model_name, \ scope_kind, canonical_key_bytes, canonical_key_hash, incarnation, revision, \ - failure_id FROM projection_changes WHERE topology_hash = ", + failure_id, program_id FROM projection_changes WHERE topology_hash = ", ); builder.push_bind(topology_hash.as_slice()); builder.push(" AND partition_hash = "); diff --git a/src/sqlx_repo/projection_protocol/rebuild.rs b/src/sqlx_repo/projection_protocol/rebuild.rs index 1f48a7ad..0c82dca3 100644 --- a/src/sqlx_repo/projection_protocol/rebuild.rs +++ b/src/sqlx_repo/projection_protocol/rebuild.rs @@ -135,6 +135,7 @@ where Some(row.scope.clone()), Some(revision.clone()), None, + None, )?; let record = ProjectionRecordMetadata { revision, diff --git a/src/sqlx_repo/projection_protocol/store_impl.rs b/src/sqlx_repo/projection_protocol/store_impl.rs index 5e60a00f..353378e5 100644 --- a/src/sqlx_repo/projection_protocol/store_impl.rs +++ b/src/sqlx_repo/projection_protocol/store_impl.rs @@ -442,6 +442,7 @@ where Some(mutation.scope.clone()), Some(revision.clone()), None, + program_id_for_model(&batch.ownership, mutation.scope.model()), )?; let metadata = ProjectionRecordMetadata { revision, @@ -530,17 +531,20 @@ where Some(scope.clone()), revision.clone(), None, + program_id_for_model(&batch.ownership, scope.model()), )?; let cursor = change.cursor.clone(); changes.push(change); cursor }; + let program_id = program_id_for_model(&batch.ownership, scope.model()); observations.push(ProjectionObservation { causation_id: batch.input.causation_id.clone(), kind: request.kind, revision, scope, change: change_cursor, + program_id, }); } @@ -555,6 +559,7 @@ where None, None, None, + None, )?); } let final_change = changes @@ -722,6 +727,7 @@ where None, None, Some(batch.failure_id.clone()), + None, )?; insert_change_in_tx(&mut tx, &change).await?; insert_failure_in_tx(&mut tx, &batch, &change.cursor).await?; diff --git a/src/sqlx_repo/projection_protocol/tests.rs b/src/sqlx_repo/projection_protocol/tests.rs index 53b908b4..f91a3a8b 100644 --- a/src/sqlx_repo/projection_protocol/tests.rs +++ b/src/sqlx_repo/projection_protocol/tests.rs @@ -17,7 +17,8 @@ mod tests { use crate::projection_protocol::{ ProjectionCheckpointProbe, ProjectionExecutionSnapshotBatchRequest, ProjectionGraphSnapshotRequest, ProjectionObservationRequest, - ProjectionQuerySnapshotRequest, ProjectionRecordMutation, ProjectionScopeCodec, + ProjectionLiveRecordRequest, ProjectionQuerySnapshotRequest, ProjectionRecordMutation, + ProjectionScopeCodec, }; use crate::repository::{CommitBatch, ReadModelWritePlanStore, TransactionalCommit}; use crate::table::{ @@ -364,6 +365,12 @@ mod tests { ProjectionModelOwnership::new("SqlTodoView", "sql_todo_views").unwrap() } + fn semantic_program_id(fill: char) -> crate::ProjectionProgramId { + let hex = std::iter::repeat_n(fill, 64).collect::(); + crate::ProjectionProgramId::parse(&format!("pp1:sha256:{hex}")) + .expect("test semantic program ID is canonical") + } + #[derive(Clone, Copy)] struct ProjectionScenario; @@ -467,6 +474,33 @@ mod tests { (repository, database_path) } + async fn reopen_wal_repository(path: &Path) -> SqlxRepository { + let pool = SqlitePoolOptions::new() + .max_connections(5) + .connect_with( + SqliteConnectOptions::new() + .filename(path) + .create_if_missing(false) + .journal_mode(SqliteJournalMode::Wal), + ) + .await + .unwrap(); + let repository = SqlxRepository::::new(pool) + .with_projection_change_retention(ProjectionChangeRetention::new(16).unwrap()); + repository.migrate().await.unwrap(); + let mut registry = TableSchemaRegistry::new(); + registry.register_schema(schema().clone()).unwrap(); + repository + .bootstrap_table_schema_for_dev(®istry) + .await + .unwrap(); + repository + .register_projection_models(&topology(), &[ownership()]) + .await + .unwrap(); + repository + } + async fn remove_wal_database(repository: SqlxRepository, path: &Path) { repository.pool().close().await; for candidate in [ @@ -1017,6 +1051,108 @@ mod tests { assert_ne!(old_scope, new_scope); } + #[tokio::test] + async fn sqlite_modeled_projection_identity_is_durable_across_restart_and_null_history_stays_readable() + { + let (repository, database_path) = wal_repository_with_retention(16).await; + let program_a = semantic_program_id('a'); + let scope = record_scope(); + let result = repository + .commit_projection(ProjectionCommitBatch { + input: input( + 1, + b"semantic-identity-a", + "semantic-identity-message-a", + "semantic-identity-cause-a", + ProjectionGeneration::initial(), + ), + change_epoch: change_epoch(), + ownership: vec![ownership().with_program_id(program_a)], + mutations: vec![mutation( + ProjectionRecordExpectation::Missing, + ProjectionMutationKind::Upsert, + )], + observations: vec![ProjectionObservationRequest { + kind: ProjectionObservationKind::Record, + target: ProjectionObservationTarget::StagedRecord(scope.clone()), + }], + }) + .await + .unwrap(); + assert!(result + .changes + .iter() + .all(|change| change.program_id == Some(program_a))); + + let selected = + ProjectionCausationEvidenceRequest::new("semantic-identity-cause-a", vec![topology()]) + .unwrap(); + let evidence = repository + .projection_causation_evidence(&selected) + .await + .unwrap(); + assert_eq!(evidence.observations.len(), 1); + assert_eq!(evidence.observations[0].program_id, Some(program_a)); + + repository.pool().close().await; + let reopened = reopen_wal_repository(&database_path).await; + let changes = match reopened + .projection_changes(&topology(), &partition(), None, 100) + .await + .unwrap() + { + ProjectionChangeRead::Changes { changes, .. } => changes, + other => panic!("restarted repository must retain projection changes: {other:?}"), + }; + assert_eq!(changes.len(), 1, "the staged observation shares the record change"); + assert!(changes + .iter() + .all(|change| change.program_id == Some(program_a))); + let evidence = reopened + .projection_causation_evidence(&selected) + .await + .unwrap(); + assert_eq!(evidence.observations.len(), 1); + assert_eq!(evidence.observations[0].program_id, Some(program_a)); + + // Rows written before semantic identities existed remain useful after + // migration, but their null identity cannot mint modeled proof. + sqlx::query( + "UPDATE projection_changes SET program_id = NULL WHERE causation_id = ?", + ) + .bind("semantic-identity-cause-a") + .execute(reopened.pool()) + .await + .unwrap(); + sqlx::query( + "UPDATE projection_observations SET program_id = NULL WHERE causation_id = ?", + ) + .bind("semantic-identity-cause-a") + .execute(reopened.pool()) + .await + .unwrap(); + let null_history = reopened + .projection_causation_evidence(&selected) + .await + .unwrap(); + assert_eq!(null_history.observations.len(), 1); + assert_eq!(null_history.observations[0].program_id, None); + let readable = reopened + .projection_live_record_batch( + &ProjectionLiveRecordBatchRequest::new(vec![ + ProjectionLiveRecordRequest::new(&scope_codec(), "SqlTodoView", record_key()) + .unwrap(), + ]) + .unwrap(), + ) + .await + .unwrap(); + assert!(readable.records[0].is_some(), "unversioned rows remain readable"); + assert_eq!(readable.records[0].as_ref().unwrap().revision.scope(), &scope); + + remove_wal_database(reopened, &database_path).await; + } + #[tokio::test] async fn sqlite_receipts_source_fences_and_raw_write_fence_are_exact() { let repository = repository().await; diff --git a/src/sqlx_repo/projection_protocol/writes.rs b/src/sqlx_repo/projection_protocol/writes.rs index 1711222c..fd5472a6 100644 --- a/src/sqlx_repo/projection_protocol/writes.rs +++ b/src/sqlx_repo/projection_protocol/writes.rs @@ -1,5 +1,15 @@ use super::*; +pub(super) fn program_id_for_model( + ownership: &[ProjectionModelOwnership], + model: &str, +) -> Option { + ownership + .iter() + .find(|declaration| declaration.model == model) + .and_then(|declaration| declaration.program_id) +} + pub(super) async fn ensure_partition_ownership_in_tx( tx: &mut Transaction<'_, DB>, topology: &ProjectorTopologyId, @@ -387,6 +397,7 @@ pub(super) fn allocate_change( scope: Option, revision: Option, failure_id: Option, + program_id: Option, ) -> Result { state.change_head = checked_next(state.change_head, "projection change")?; Ok(ProjectionChange { @@ -402,6 +413,7 @@ pub(super) fn allocate_change( scope, revision, failure_id, + program_id, }) } @@ -425,7 +437,7 @@ where "INSERT INTO projection_changes \ (topology_hash, partition_hash, change_epoch, change_position, change_kind, \ causation_id, model_name, scope_kind, canonical_key_bytes, canonical_key_hash, \ - incarnation, revision, failure_id) VALUES (", + incarnation, revision, failure_id, program_id) VALUES (", ); builder.push_bind(topology_hash.as_slice()); builder.push(", "); @@ -487,6 +499,13 @@ where } else { builder.push("NULL"); } + builder.push(", "); + let program_id_value = change.program_id.map(|program_id| program_id.to_string()); + if let Some(program_id) = program_id_value.as_deref() { + builder.push_bind(program_id); + } else { + builder.push("NULL"); + } builder.push(")"); builder .build() @@ -659,6 +678,9 @@ where })?, "observation change position", )?; + let program_id = decode_program_id(row.try_get("program_id").map_err(|error| { + protocol_storage_error::("decode observation program identity", error) + })?)?; Ok(ProjectionObservation { causation_id: causation_id.to_string(), kind, @@ -670,6 +692,7 @@ where change_epoch, change_position, )?, + program_id, }) } @@ -696,7 +719,7 @@ where let key_hash = scope.key_digest(); let mut builder = QueryBuilder::::new( "SELECT canonical_key_bytes, canonical_key_hash, incarnation, revision, \ - change_epoch, change_position FROM projection_observations WHERE topology_hash = ", + change_epoch, change_position, program_id FROM projection_observations WHERE topology_hash = ", ); builder.push_bind(topology_hash.as_slice()); builder.push(" AND partition_hash = "); @@ -740,7 +763,7 @@ where "INSERT INTO projection_observations \ (topology_hash, partition_hash, causation_id, model_name, scope_kind, \ canonical_key_bytes, canonical_key_hash, incarnation, revision, \ - change_epoch, change_position) VALUES (", + change_epoch, change_position, program_id) VALUES (", ); builder.push_bind(topology_hash.as_slice()); builder.push(", "); @@ -780,6 +803,15 @@ where observation.change.position(), "observation change position", )?); + builder.push(", "); + let program_id_value = observation + .program_id + .map(|program_id| program_id.to_string()); + if let Some(program_id) = program_id_value.as_deref() { + builder.push_bind(program_id); + } else { + builder.push("NULL"); + } builder.push(")"); builder .build() diff --git a/src/sqlx_repo/repo/backend.rs b/src/sqlx_repo/repo/backend.rs index 79f9f113..3bb2ce53 100644 --- a/src/sqlx_repo/repo/backend.rs +++ b/src/sqlx_repo/repo/backend.rs @@ -47,7 +47,7 @@ mod tests { .iter() .map(|migration| migration.sql) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7]); assert_eq!( descriptions, vec![ @@ -56,7 +56,8 @@ mod tests { "projection protocol", "command ledger atomic state", "projection source snapshots", - "gateway dependency versions" + "gateway dependency versions", + "projection program identity" ] ); assert_eq!( @@ -86,6 +87,10 @@ mod tests { env!("CARGO_MANIFEST_DIR"), "/migrations/sqlite/0006_gateway_dependency_versions.sql" )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/migrations/sqlite/0007_projection_program_identity.sql" + )), ] ); } @@ -105,7 +110,7 @@ mod tests { .iter() .map(|migration| migration.sql) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7]); assert_eq!( descriptions, vec![ @@ -114,7 +119,8 @@ mod tests { "projection protocol", "command ledger atomic state", "projection source snapshots", - "gateway dependency versions" + "gateway dependency versions", + "projection program identity" ] ); assert_eq!( @@ -144,6 +150,10 @@ mod tests { env!("CARGO_MANIFEST_DIR"), "/migrations/postgres/0006_gateway_dependency_versions.sql" )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/migrations/postgres/0007_projection_program_identity.sql" + )), ] ); } From 4a796001097e0daad2b03670c43cb7c635b23ea3 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 9 Sep 2026 19:24:42 -0500 Subject: [PATCH 03/10] test: track projection identity migration --- tests/postgres_repository/main.rs | 2 +- tests/sqlite_repository/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/postgres_repository/main.rs b/tests/postgres_repository/main.rs index 33bd83f5..db25e06b 100644 --- a/tests/postgres_repository/main.rs +++ b/tests/postgres_repository/main.rs @@ -234,7 +234,7 @@ async fn projected_command_ledger_rows_upgrade_to_atomic_and_preserve_checks() { .fetch_one(repo.pool()) .await .unwrap(); - assert_eq!(latest_version, 6); + assert_eq!(latest_version, 7); let invalid_service = sqlx::query( r#" diff --git a/tests/sqlite_repository/main.rs b/tests/sqlite_repository/main.rs index 55e7fd3c..2a98aff4 100644 --- a/tests/sqlite_repository/main.rs +++ b/tests/sqlite_repository/main.rs @@ -201,7 +201,7 @@ async fn projected_command_ledger_rows_upgrade_to_atomic_without_schema_drift() .fetch_one(repo.pool()) .await .unwrap(); - assert_eq!(latest_version, 6); + assert_eq!(latest_version, 7); let created_at_type: String = sqlx::query_scalar( "SELECT typeof(created_at) FROM command_ledger WHERE service_id = 'service'", From 4d08f12730eddf09b3052a203841edd286bcc7fd Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 9 Sep 2026 19:26:25 -0500 Subject: [PATCH 04/10] test: include projection identity migration --- distributed_cli/src/contracts/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distributed_cli/src/contracts/tests.rs b/distributed_cli/src/contracts/tests.rs index 11b4bb37..0f0052be 100644 --- a/distributed_cli/src/contracts/tests.rs +++ b/distributed_cli/src/contracts/tests.rs @@ -1259,7 +1259,7 @@ fn migration_inventory_is_deterministic_and_preserves_runtime_order() { .iter() .map(|migration| migration.version) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7]); assert_eq!( inventory.canonical_bytes().expect("canonical inventory"), inventory From 0b511bb8d0f02ec9d0c1026e32eed3fe67f28ed1 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 9 Sep 2026 20:44:14 -0500 Subject: [PATCH 05/10] fix: evaluate proofs for succeeded causal receipts --- src/microsvc/service/causal.rs | 20 +++ src/microsvc/service/tests.rs | 237 +++++++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+) diff --git a/src/microsvc/service/causal.rs b/src/microsvc/service/causal.rs index 2cefd280..aa5a705a 100644 --- a/src/microsvc/service/causal.rs +++ b/src/microsvc/service/causal.rs @@ -846,6 +846,26 @@ where == crate::graphql::projection_delta::runtime::ModeledProjectionStatusDisposition::Revalidate }); let (state, evidence) = match receipt.state { + CommandLedgerState::Succeeded + if receipt + .projection_metadata + .as_ref() + .is_some_and(|metadata| !metadata.obligations.is_empty()) + || !receipt.obligations.is_empty() => + { + let (_, evidence) = evaluate_pending_projection_evidence( + repository, + &receipt, + protocol, + modeled_plan.as_ref(), + ) + .await?; + // A succeeded ledger receipt proves the command commit. + // Retained projection obligations are only evidence detail; + // their asynchronous proof must not rewrite that public + // command state. + (CausalCommandPublicState::Succeeded, evidence) + } CommandLedgerState::Succeeded => (CausalCommandPublicState::Succeeded, Vec::new()), CommandLedgerState::Atomic => ( CausalCommandPublicState::Atomic, diff --git a/src/microsvc/service/tests.rs b/src/microsvc/service/tests.rs index 089f86c7..dc2671b9 100644 --- a/src/microsvc/service/tests.rs +++ b/src/microsvc/service/tests.rs @@ -2680,6 +2680,243 @@ async fn graphql_terminal_replay_revalidates_after_active_projection_starts_drai assert_eq!(handler_calls.load(Ordering::SeqCst), 2); } +#[cfg(all(feature = "graphql", feature = "sqlite"))] +#[tokio::test] +async fn graphql_succeeded_status_evaluates_retained_projection_evidence() { + let repository = crate::SqliteRepository::connect_and_migrate("sqlite::memory:") + .await + .expect("framework migrations should apply"); + let mut table_registry = crate::table::TableSchemaRegistry::new(); + table_registry + .register_schema( + ::schema().clone(), + ) + .unwrap(); + repository + .bootstrap_table_schema_for_dev(&table_registry) + .await + .unwrap(); + let projector = modeled_lifecycle_projector( + crate::projection::placement::ProjectionBindingState::Active, + "causal-succeeded-status", + "causal-succeeded-status-topology", + 0x7a, + ); + let service = Service::new().named("causal-succeeded-status").routes( + Routes::new() + .with_repo(repository.clone().aggregate::()) + .with_read_model_store(repository.clone()) + .typed_command( + typed_command::>("causal.lifecycle") + .roles(["user"]) + .emits(crate::events![CausalLifecycleRecorded]), + ) + .handle( + |context: &CausalCommandContext<'_, CausalDispatcherAggregate>, + input: CausalTestInput| { + let result = (|| { + let mut checkout = context.create(); + checkout.record_lifecycle(input.id.clone(), input.label)?; + context + .publish_events() + .commit(checkout)? + .eventual(TypedOutput { id: input.id }) + })(); + async move { result } + }, + ) + .consume_projection(projector.clone()), + ); + let engine = crate::graphql::GraphqlEngine::builder(&repository) + .protocol_token_key(TEST_PROTOCOL_TOKEN_KEY) + .model::( + crate::graphql::ModelPermissions::new() + .grant("user", crate::graphql::read().all_columns()), + ) + .service(&service) + .client_projectors([projector]) + .build() + .expect("active modeled projection should compile"); + let service = Arc::new( + service + .try_with_graphql(engine) + .expect("compiled service should bind"), + ); + let command_id = causal_test_command_id(); + let mutation = format!( + "mutation {{ causal_lifecycle(commandId: \"{command_id}\", input: {{ id: \"todo-succeeded-status\", label: \"active\" }}) {{ id }} }}" + ); + let session = session_with_role("user"); + let principal = causal_test_principal(); + let response = service + .graphql_engine() + .unwrap() + .execute( + &session, + async_graphql::Request::new(&mutation) + .data(command_host(&service)) + .data(principal.clone()), + ) + .await; + assert!(response.errors.is_empty(), "{response:?}"); + let envelope = serde_json::to_value( + response + .extensions + .get("distributed") + .expect("modeled command should carry the protocol envelope"), + ) + .unwrap(); + assert_eq!(envelope["command"]["state"], "succeeded_pending_projection"); + assert_eq!(envelope["command"]["expects"].as_array().unwrap().len(), 1); + let causation_id = envelope["command"]["causationId"] + .as_str() + .expect("command envelope should carry its causation") + .to_string(); + + // The wait-path cell commits a terminal succeeded receipt while retaining + // modeled obligations. Before the projector writes proof, status remains + // publicly succeeded but cannot claim an observation. + let changed = sqlx::query( + "UPDATE command_ledger SET state = 'succeeded' \ + WHERE service_id = ? AND command_id = ?", + ) + .bind("causal-succeeded-status") + .bind(&command_id) + .execute(repository.pool()) + .await + .unwrap(); + assert_eq!(changed.rows_affected(), 1); + + let status_query = + format!("query {{ commandStatus(commandId: \"{command_id}\") {{ state }} }}"); + let before_projection = service + .graphql_engine() + .unwrap() + .execute( + &session, + async_graphql::Request::new(&status_query) + .data(command_host(&service)) + .data(principal.clone()), + ) + .await; + assert!(before_projection.errors.is_empty(), "{before_projection:?}"); + assert_eq!( + before_projection.data.into_json().unwrap(), + json!({"commandStatus": {"state": "succeeded"}}) + ); + let before_envelope = serde_json::to_value( + before_projection + .extensions + .get("distributed") + .expect("status should carry its protocol envelope"), + ) + .unwrap(); + assert_eq!(before_envelope["command"]["state"], "succeeded"); + assert_eq!( + before_envelope["command"]["expects"] + .as_array() + .unwrap() + .len(), + 1 + ); + assert!(before_envelope["command"].get("observations").is_none()); + + let pending = repository + .outbox_store() + .pending(10) + .await + .expect("the committed domain event should be pending"); + assert_eq!(pending.len(), 1); + let ordered = crate::bus::OrderedDelivery::new( + crate::projection_protocol::ProjectionSource::new( + "test-ordered-events", + b"causal-succeeded-status".to_vec(), + ) + .unwrap(), + crate::projection_protocol::ProjectionEpoch::new("test-ordered-events-v1").unwrap(), + 1, + true, + ) + .unwrap(); + service + .dispatch_ordered_message(&Message::from(pending[0].clone()), Some(&ordered)) + .await + .expect("the real modeled projector should commit its evidence"); + + let after_projection = service + .graphql_engine() + .unwrap() + .execute( + &session, + async_graphql::Request::new(&status_query) + .data(command_host(&service)) + .data(principal.clone()), + ) + .await; + assert!(after_projection.errors.is_empty(), "{after_projection:?}"); + let after_envelope = serde_json::to_value( + after_projection + .extensions + .get("distributed") + .expect("status should carry its protocol envelope"), + ) + .unwrap(); + assert_eq!(after_envelope["command"]["state"], "succeeded"); + assert_eq!( + after_envelope["command"]["observations"] + .as_array() + .expect("matching durable proof should be exposed") + .len(), + 1 + ); + assert_eq!( + after_envelope["command"]["observations"][0]["causationId"], + causation_id + ); + + // A proof authored by a different semantic program is not an observation + // for this command, even when its physical topology and scope are equal. + let wrong_program = + "pp1:sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + let changed = + sqlx::query("UPDATE projection_observations SET program_id = ? WHERE causation_id = ?") + .bind(wrong_program) + .bind(&causation_id) + .execute(repository.pool()) + .await + .unwrap(); + assert_eq!(changed.rows_affected(), 1); + + let wrong_projection = service + .graphql_engine() + .unwrap() + .execute( + &session, + async_graphql::Request::new(&status_query) + .data(command_host(&service)) + .data(principal), + ) + .await; + assert!(wrong_projection.errors.is_empty(), "{wrong_projection:?}"); + let wrong_envelope = serde_json::to_value( + wrong_projection + .extensions + .get("distributed") + .expect("status should carry its protocol envelope"), + ) + .unwrap(); + assert_eq!(wrong_envelope["command"]["state"], "succeeded"); + assert!(wrong_envelope["command"].get("observations").is_none()); + assert_eq!( + wrong_envelope["command"]["expects"] + .as_array() + .unwrap() + .len(), + 1, + "mismatched proof must leave the exact obligation pending" + ); +} + #[cfg(all(feature = "graphql", feature = "sqlite"))] #[tokio::test] async fn engine_rejects_incompatible_direct_owner_before_typed_command_binding() { From a1176e4ce3dbfaa8bc73f3117e00640efb5dcfca Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 9 Sep 2026 23:15:51 -0500 Subject: [PATCH 06/10] feat: add external command ledger bindings --- README.md | 13 + distributed_cli/src/contracts/tests.rs | 2 +- migrations/inventory.json | 12 + .../0008_external_command_binding.sql | 4 + .../sqlite/0008_external_command_binding.sql | 4 + src/command_ledger/mod.rs | 3 +- src/command_ledger/record.rs | 62 ++- src/command_ledger/reservation.rs | 240 ++++++++++ src/command_ledger/tests.rs | 440 ++++++++++++++++++ src/command_ledger/traits.rs | 11 +- src/in_memory_repo/repository.rs | 27 +- src/microsvc/cell_host/sql_store.rs | 10 +- src/microsvc/cell_host/store.rs | 10 +- src/microsvc/service/tests.rs | 10 +- src/postgres_repo/mod.rs | 2 +- src/queued_repo/repository.rs | 10 +- src/repository/migrations.rs | 2 +- src/repository/sql/ledger.rs | 79 +++- src/repository/sqlite_codec.rs | 2 +- src/sqlx_repo/repo/backend.rs | 18 +- src/sqlx_repo/repo/commit.rs | 18 + src/sqlx_repo/repo/mod.rs | 2 +- tests/postgres_repository/main.rs | 2 +- tests/sqlite_repository/main.rs | 2 +- 24 files changed, 964 insertions(+), 21 deletions(-) create mode 100644 migrations/postgres/0008_external_command_binding.sql create mode 100644 migrations/sqlite/0008_external_command_binding.sql diff --git a/README.md b/README.md index bbb49214..9ea07ee8 100644 --- a/README.md +++ b/README.md @@ -739,6 +739,19 @@ can deliver the same event ID again; consumers must remain idempotent. Event history belongs in the event store, and command retry results belong in command receipts. Neither depends on retaining delivered outbox rows. +Commands dispatched to another aggregate cell use the same command ledger, but +with an explicit external dispatch binding. The binding records the logical +route kind and shard alongside the command identity; it is immutable for that +reservation and survives lease reclaim and terminal replay. External completion +updates only the ledger after the returned attempt fence and binding match. It +does not append local events, write the outbox, or treat the cell's commit as +proof that an asynchronous read-model projection has completed. A local causal +commit cannot complete an externally bound reservation, and an external +completion cannot complete a local reservation. Retained cell-only receipts +without a gateway binding remain explicit unknowns until a matching gateway +reservation and trusted completion protocol can be established; they are never +retrofitted into a different route or silently replayed. + In v5, `OutboxStore::complete` and `complete_many` remove delivered rows instead of retaining `Published` records. Repeating settlement of a removed row returns `NotFound`; stale claims on existing rows still return `InvalidState`. diff --git a/distributed_cli/src/contracts/tests.rs b/distributed_cli/src/contracts/tests.rs index 0f0052be..1c05512d 100644 --- a/distributed_cli/src/contracts/tests.rs +++ b/distributed_cli/src/contracts/tests.rs @@ -1259,7 +1259,7 @@ fn migration_inventory_is_deterministic_and_preserves_runtime_order() { .iter() .map(|migration| migration.version) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8]); assert_eq!( inventory.canonical_bytes().expect("canonical inventory"), inventory diff --git a/migrations/inventory.json b/migrations/inventory.json index 9e2e9add..25b8cc13 100644 --- a/migrations/inventory.json +++ b/migrations/inventory.json @@ -84,6 +84,18 @@ "path": "migrations/postgres/0007_projection_program_identity.sql", "sha256": "ad7da74a11637a522279a6bc061d2c79db1f807f869368cf31735a089dfe0204" } + }, + { + "version": 8, + "description": "external command binding", + "sqlite": { + "path": "migrations/sqlite/0008_external_command_binding.sql", + "sha256": "4be4e5583d7f37a4f2820bf7777a750246ff574d43f4b9fbd15324d6712a6240" + }, + "postgres": { + "path": "migrations/postgres/0008_external_command_binding.sql", + "sha256": "2f93c5685cde1d5cafd444d3a28dbcbf01f16cd44a566c7af61a7bd2a793a163" + } } ] } diff --git a/migrations/postgres/0008_external_command_binding.sql b/migrations/postgres/0008_external_command_binding.sql new file mode 100644 index 00000000..0664b3d5 --- /dev/null +++ b/migrations/postgres/0008_external_command_binding.sql @@ -0,0 +1,4 @@ +-- Bind externally dispatched command reservations to their logical route. +-- NULL retains the pre-external-dispatch meaning for local reservations and +-- old cell-only rows; non-NULL values are canonical versioned JSON. +ALTER TABLE command_ledger ADD COLUMN external_binding text; diff --git a/migrations/sqlite/0008_external_command_binding.sql b/migrations/sqlite/0008_external_command_binding.sql new file mode 100644 index 00000000..4ae46667 --- /dev/null +++ b/migrations/sqlite/0008_external_command_binding.sql @@ -0,0 +1,4 @@ +-- Bind externally dispatched command reservations to their logical route. +-- NULL retains the pre-external-dispatch meaning for local reservations and +-- old cell-only rows; non-NULL values are canonical versioned JSON. +ALTER TABLE command_ledger ADD COLUMN external_binding TEXT; diff --git a/src/command_ledger/mod.rs b/src/command_ledger/mod.rs index 0c89ed84..5b069850 100644 --- a/src/command_ledger/mod.rs +++ b/src/command_ledger/mod.rs @@ -32,7 +32,8 @@ pub(crate) use ids::{ pub(crate) use record::{CommandLedgerRecord, ReservationDecision}; pub(crate) use reservation::{ AttemptFence, CausalCommitBatch, CommandAttempt, CommandCompletion, CommandLookup, - CommandLookupScope, CommandReplay, CommandReservation, ReservationOutcome, + CommandLookupScope, CommandReplay, CommandReservation, ExternalCommandCompletion, + ExternalDispatchBinding, ReservationOutcome, }; pub(crate) use state::{CommandLedgerState, TerminalCommandState}; pub(crate) use traits::{ diff --git a/src/command_ledger/record.rs b/src/command_ledger/record.rs index 8ccc9629..115df796 100644 --- a/src/command_ledger/record.rs +++ b/src/command_ledger/record.rs @@ -11,7 +11,8 @@ use super::{ ids::COMMAND_REPLAY_VERSION, state::validate_projection_obligation_semantics, AttemptFence, AttemptToken, CanonicalInputHash, CausationId, CommandAttempt, CommandCompletion, CommandContractFingerprint, CommandLedgerError, CommandLedgerKey, CommandLedgerState, - CommandLookup, CommandLookupScope, CommandReplay, CommandReservation, ReservationOutcome, + CommandLookup, CommandLookupScope, CommandReplay, CommandReservation, + ExternalCommandCompletion, ExternalDispatchBinding, ReservationOutcome, }; /// Storage-neutral row representation shared by built-in adapters. @@ -25,6 +26,7 @@ pub(crate) struct CommandLedgerRecord { pub(crate) causation_id: CausationId, pub(crate) attempt_token: Option, pub(crate) attempt_number: u64, + pub(crate) external_binding: Option, pub(crate) lease_expires_at: Option, pub(crate) outcome_json: Option, #[allow(dead_code)] @@ -49,6 +51,7 @@ impl CommandLedgerRecord { causation_id: reservation.candidate_causation.clone(), attempt_token: Some(reservation.candidate_attempt.clone()), attempt_number: 1, + external_binding: reservation.external_binding.clone(), lease_expires_at: Some(checked_deadline(now, reservation.lease, "attempt lease")?), outcome_json: None, created_at: now, @@ -84,6 +87,11 @@ impl CommandLedgerRecord { principal_partition: self.key.principal_partition().to_string(), command_id: self.key.command_id().to_string(), command_name: self.command_name.clone(), + external_binding: self + .external_binding + .as_ref() + .map(ExternalDispatchBinding::to_storage) + .transpose()?, contract_fingerprint: self.contract_fingerprint.as_bytes().to_vec(), input_hash: self.input_hash.as_bytes().to_vec(), state: self.state.as_str().to_string(), @@ -135,6 +143,11 @@ impl CommandLedgerRecord { let record = Self { key, command_name: wire.command_name, + external_binding: wire + .external_binding + .as_deref() + .map(ExternalDispatchBinding::from_storage) + .transpose()?, contract_fingerprint: CommandContractFingerprint::try_from_slice( &wire.contract_fingerprint, )?, @@ -184,6 +197,7 @@ impl CommandLedgerRecord { causation_id: self.causation_id.clone(), attempt_token: token.clone(), attempt_number: self.attempt_number, + external_binding: self.external_binding.clone(), }) } @@ -198,6 +212,7 @@ impl CommandLedgerRecord { if self.command_name != reservation.command_name || self.contract_fingerprint != reservation.contract_fingerprint || self.input_hash != reservation.input_hash + || self.external_binding != reservation.external_binding { return Ok(ReservationDecision::Conflict); } @@ -385,6 +400,11 @@ impl CommandLedgerRecord { completion: &CommandCompletion, now: SystemTime, ) -> Result<(), CommandLedgerError> { + if self.external_binding.is_some() { + return Err(CommandLedgerError::Invalid( + "local causal completion cannot complete an externally bound reservation".into(), + )); + } completion.validate_direct_projection()?; self.validate_live_attempt(&completion.attempt.fence(), now)?; let retention_expires_at = match completion.retention_expires_at() { @@ -406,6 +426,44 @@ impl CommandLedgerRecord { Ok(()) } + pub(crate) fn complete_external( + &mut self, + completion: &ExternalCommandCompletion, + now: SystemTime, + ) -> Result<(), CommandLedgerError> { + if self.external_binding.as_ref() != Some(completion.binding()) { + return Err(CommandLedgerError::Invalid( + "external completion binding does not match the reserved route".into(), + )); + } + if completion.attempt().external_binding() != Some(completion.binding()) { + return Err(CommandLedgerError::Invalid( + "external completion attempt does not carry the reserved route".into(), + )); + } + // Check the complete attempt fence before any other mutable-row + // invariant. A late completion must report the generation race even + // when its retention deadline has also elapsed. + self.validate_live_attempt(&completion.attempt_fence(), now)?; + let retention_expires_at = match completion.retention_expires_at() { + Some(deadline) if deadline > now => deadline, + Some(_) => { + return Err(CommandLedgerError::Invalid( + "command retention deadline must remain live at commit".into(), + )); + } + None => checked_deadline(now, completion.retention(), "command retention")?, + }; + self.state = completion.state().into(); + self.attempt_token = None; + self.lease_expires_at = None; + self.outcome_json = Some(completion.replay_json().to_string()); + self.updated_at = now; + self.completed_at = Some(now); + self.retention_expires_at = retention_expires_at; + Ok(()) + } + pub(crate) fn replay(&self) -> Result { if !self.state.is_replayable() { return Err(CommandLedgerError::Corrupt(format!( @@ -615,6 +673,8 @@ struct DurableCellCommandRecordV1 { principal_partition: String, command_id: String, command_name: String, + #[serde(default)] + external_binding: Option, contract_fingerprint: Vec, input_hash: Vec, state: String, diff --git a/src/command_ledger/reservation.rs b/src/command_ledger/reservation.rs index a22d273a..e22cc854 100644 --- a/src/command_ledger/reservation.rs +++ b/src/command_ledger/reservation.rs @@ -3,6 +3,7 @@ use std::time::{Duration, SystemTime}; use base64::engine::general_purpose::URL_SAFE_NO_PAD; use base64::Engine as _; +use serde::{Deserialize, Serialize}; use serde_json::Value; use crate::projection_protocol::{ @@ -16,6 +17,97 @@ use super::{ CommandLedgerKey, CommandLedgerState, TerminalCommandState, SHA256_BYTES, }; +const EXTERNAL_BINDING_VERSION: u16 = 1; + +/// Immutable logical route ownership for a command dispatched outside the +/// repository that owns its ledger row. Transport addresses and retry leases +/// are intentionally not part of this identity. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ExternalDispatchBinding { + version: u16, + route_kind: String, + shard: String, +} + +impl ExternalDispatchBinding { + pub(crate) fn new( + route_kind: impl Into, + shard: impl Into, + ) -> Result { + let route_kind = route_kind.into(); + let shard = shard.into(); + validate_binding_part("external route kind", &route_kind)?; + validate_binding_part("external shard", &shard)?; + Ok(Self { + version: EXTERNAL_BINDING_VERSION, + route_kind, + shard, + }) + } + + pub(crate) fn route_kind(&self) -> &str { + &self.route_kind + } + + pub(crate) fn shard(&self) -> &str { + &self.shard + } + + /// Stable storage spelling. Parsing requires this exact canonical form so + /// equivalent-looking bindings cannot bypass the immutable row value. + pub(crate) fn to_storage(&self) -> Result { + serde_json::to_string(self).map_err(|error| { + CommandLedgerError::Invalid(format!( + "external dispatch binding could not be serialized: {error}" + )) + }) + } + + pub(crate) fn from_storage(value: &str) -> Result { + let binding: Self = serde_json::from_str(value).map_err(|error| { + CommandLedgerError::Corrupt(format!( + "stored external dispatch binding is invalid: {error}" + )) + })?; + if binding.version != EXTERNAL_BINDING_VERSION { + return Err(CommandLedgerError::Corrupt(format!( + "stored external dispatch binding version `{}` is unsupported", + binding.version + ))); + } + validate_binding_part("external route kind", &binding.route_kind) + .map_err(|error| CommandLedgerError::Corrupt(error.to_string()))?; + validate_binding_part("external shard", &binding.shard) + .map_err(|error| CommandLedgerError::Corrupt(error.to_string()))?; + let canonical = binding.to_storage().map_err(|error| { + CommandLedgerError::Corrupt(format!( + "stored external dispatch binding cannot be canonicalized: {error}" + )) + })?; + if canonical != value { + return Err(CommandLedgerError::Corrupt( + "stored external dispatch binding is not canonical JSON".into(), + )); + } + Ok(binding) + } +} + +fn validate_binding_part(label: &str, value: &str) -> Result<(), CommandLedgerError> { + if value.trim().is_empty() { + return Err(CommandLedgerError::Invalid(format!( + "{label} must not be empty" + ))); + } + if value.len() > 256 || value.chars().any(char::is_control) { + return Err(CommandLedgerError::Invalid(format!( + "{label} must be at most 256 bytes and contain no control characters" + ))); + } + Ok(()) +} + /// One validated reservation request. Fresh candidate IDs lose a race safely: /// only the inserted row keeps them; every retry reads the winner's causation. pub(crate) struct CommandReservation { @@ -27,6 +119,7 @@ pub(crate) struct CommandReservation { pub(super) retention: Duration, pub(super) candidate_causation: CausationId, pub(super) candidate_attempt: AttemptToken, + pub(super) external_binding: Option, } impl CommandReservation { @@ -60,9 +153,18 @@ impl CommandReservation { retention, candidate_causation: CausationId::new(), candidate_attempt: AttemptToken::new(), + external_binding: None, }) } + /// Mark this reservation as owned by an external logical route. The + /// binding is persisted with the reservation and cannot be changed by a + /// reclaim or completion. + pub(crate) fn with_external_binding(mut self, binding: ExternalDispatchBinding) -> Self { + self.external_binding = Some(binding); + self + } + pub(crate) fn key(&self) -> &CommandLedgerKey { &self.key } @@ -95,6 +197,10 @@ impl CommandReservation { &self.candidate_attempt } + pub(crate) fn external_binding(&self) -> Option<&ExternalDispatchBinding> { + self.external_binding.as_ref() + } + pub(crate) fn acquired_candidate_attempt(&self) -> CommandAttempt { CommandAttempt { key: self.key.clone(), @@ -103,6 +209,7 @@ impl CommandReservation { causation_id: self.candidate_causation.clone(), attempt_token: self.candidate_attempt.clone(), attempt_number: 1, + external_binding: self.external_binding.clone(), } } } @@ -126,6 +233,7 @@ pub(crate) struct CommandAttempt { pub(super) causation_id: CausationId, pub(super) attempt_token: AttemptToken, pub(super) attempt_number: u64, + pub(super) external_binding: Option, } impl CommandAttempt { @@ -137,6 +245,10 @@ impl CommandAttempt { &self.causation_id } + pub(crate) fn external_binding(&self) -> Option<&ExternalDispatchBinding> { + self.external_binding.as_ref() + } + #[cfg(test)] pub(crate) fn attempt_token(&self) -> &AttemptToken { &self.attempt_token @@ -189,6 +301,92 @@ impl CommandAttempt { ) } + /// Complete an externally dispatched command without appending local + /// events or outbox messages. The reservation binding is checked before a + /// replay payload is constructed; the ledger repeats the check at its + /// fenced write boundary. + pub(crate) fn complete_external( + self, + binding: ExternalDispatchBinding, + state: TerminalCommandState, + outcome: Value, + projection_obligations: Vec, + retention: Duration, + ) -> Result { + self.complete_external_with_replay_metadata( + binding, + state, + outcome, + projection_obligations, + None, + retention, + None, + ) + } + + /// Complete a remote command while retaining the exact sealed projection + /// metadata and absolute deadline returned by the remote command host. + /// Metadata is validated by the same versioned protocol checks used by a + /// local causal completion; it is never inferred from the active projector + /// registry. + pub(crate) fn complete_external_with_projection_metadata_until( + self, + binding: ExternalDispatchBinding, + state: TerminalCommandState, + outcome: Value, + projection_metadata: Vec, + retention: Duration, + retention_expires_at: SystemTime, + ) -> Result { + self.complete_external_with_replay_metadata( + binding, + state, + outcome, + Vec::new(), + Some(projection_metadata), + retention, + Some(retention_expires_at), + ) + } + + fn complete_external_with_replay_metadata( + self, + binding: ExternalDispatchBinding, + state: TerminalCommandState, + outcome: Value, + projection_obligations: Vec, + projection_metadata: Option>, + retention: Duration, + retention_expires_at: Option, + ) -> Result { + if state == TerminalCommandState::Atomic { + return Err(CommandLedgerError::Invalid( + "external command completion cannot be atomic".into(), + )); + } + if self.external_binding.as_ref() != Some(&binding) { + return Err(CommandLedgerError::Invalid( + "external command completion binding does not match its reservation".into(), + )); + } + let completion = self.complete_with_replay_metadata( + state, + outcome, + projection_obligations, + projection_metadata, + retention, + retention_expires_at, + )?; + Ok(ExternalCommandCompletion { + attempt: completion.attempt, + binding, + state: completion.state, + replay: completion.replay, + retention: completion.retention, + retention_expires_at: completion.retention_expires_at, + }) + } + /// Complete a command with exact already-canonical role-safe projection /// metadata. /// @@ -451,6 +649,48 @@ pub(crate) struct CommandCompletion { retention_expires_at: Option, } +/// Terminal receipt for a command whose domain effects were committed by a +/// different aggregate cell. It intentionally has no local domain batch +/// companion; storing the receipt never republishes the remote cell's events. +pub(crate) struct ExternalCommandCompletion { + pub(super) attempt: CommandAttempt, + pub(super) binding: ExternalDispatchBinding, + pub(super) state: TerminalCommandState, + pub(super) replay: String, + pub(super) retention: Duration, + retention_expires_at: Option, +} + +impl ExternalCommandCompletion { + pub(crate) fn attempt(&self) -> &CommandAttempt { + &self.attempt + } + + pub(crate) fn binding(&self) -> &ExternalDispatchBinding { + &self.binding + } + + pub(crate) fn state(&self) -> TerminalCommandState { + self.state + } + + pub(crate) fn replay_json(&self) -> &str { + &self.replay + } + + pub(crate) fn retention(&self) -> Duration { + self.retention + } + + pub(crate) fn retention_expires_at(&self) -> Option { + self.retention_expires_at + } + + pub(crate) fn attempt_fence(&self) -> AttemptFence { + self.attempt.fence() + } +} + impl CommandCompletion { pub(crate) fn attempt(&self) -> &CommandAttempt { &self.attempt diff --git a/src/command_ledger/tests.rs b/src/command_ledger/tests.rs index 8bbb1151..5fb03699 100644 --- a/src/command_ledger/tests.rs +++ b/src/command_ledger/tests.rs @@ -249,6 +249,39 @@ fn reservation( ) } +fn external_binding(shard: &str) -> ExternalDispatchBinding { + ExternalDispatchBinding::new("aggregate-cell", shard).unwrap() +} + +fn external_reservation( + command_id: &str, + contract: u8, + input: u8, + shard: &str, +) -> Result { + Ok(reservation(command_id, contract, input)?.with_external_binding(external_binding(shard))) +} + +fn external_reservation_with_policy( + command_id: &str, + contract: u8, + input: u8, + shard: &str, + lease: Duration, + retention: Duration, +) -> Result { + Ok(reservation_for_partition_with_policy( + command_id, + "v1:sha256:principal", + "order.create", + contract, + input, + lease, + retention, + )? + .with_external_binding(external_binding(shard))) +} + trait CommandLedgerAdapterConformance: CommandLedgerStore + CausalTransactionalCommit @@ -281,6 +314,277 @@ where } } +async fn external_completion_is_ledger_only_and_replayable(repo: &R) +where + R: CommandLedgerAdapterConformance, +{ + let id = Uuid::now_v7().to_string(); + let binding = external_binding("orders-1"); + let request = external_reservation(&id, 101, 102, "orders-1").unwrap(); + let key = request.key().clone(); + let attempt = acquire(repo, request).await; + let completion = attempt + .complete_external( + binding, + TerminalCommandState::Succeeded, + serde_json::json!({"remote": true}), + Vec::new(), + Duration::from_secs(300), + ) + .unwrap(); + repo.complete_external_command(completion).await.unwrap(); + + let replay = match repo + .lookup_command(&key, CommandLookupScope::CommandName("order.create")) + .await + .unwrap() + { + CommandLookup::Replay(replay) => replay, + other => panic!("external completion should be replayable, got {other:?}"), + }; + assert_eq!(replay.state, CommandLedgerState::Succeeded); + assert_eq!(replay.outcome, serde_json::json!({"remote": true})); + assert!(repo + .get_stream(&StreamIdentity::new("remote-order", &id).unwrap()) + .await + .unwrap() + .is_none()); + let outbox = repo + .outbox_store() + .messages_by_status(OutboxMessageStatus::Pending, 1_000) + .await + .unwrap(); + assert!(outbox.iter().all(|message| message.id() != id)); + + assert!(matches!( + repo.reserve_command(external_reservation(&id, 101, 102, "orders-1").unwrap()) + .await + .unwrap(), + ReservationOutcome::Replay(_) + )); +} + +#[cfg(feature = "graphql")] +async fn external_completion_retains_the_sealed_projection_metadata(repo: &R) +where + R: CommandLedgerAdapterConformance, +{ + use std::time::UNIX_EPOCH; + + let fixture = include_bytes!("../../tests/fixtures/command-projection-metadata-v1.json"); + let fixture = fixture.strip_suffix(b"\n").unwrap_or(fixture); + let mut metadata = + crate::graphql::protocol::CommandProjectionMetadataV1::from_json(fixture).unwrap(); + let issued_at_unix_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as u64; + metadata.issued_at_unix_ms = issued_at_unix_ms; + metadata.expires_at_unix_ms = issued_at_unix_ms + 600_000; + let metadata_bytes = metadata.canonical_bytes().unwrap(); + let retention_expires_at = UNIX_EPOCH + .checked_add(Duration::from_millis(metadata.expires_at_unix_ms)) + .unwrap(); + + let id = Uuid::now_v7().to_string(); + let binding = external_binding("orders-sealed-metadata"); + let request = external_reservation(&id, 103, 104, "orders-sealed-metadata").unwrap(); + let key = request.key().clone(); + let attempt = acquire(repo, request).await; + let completion = attempt + .complete_external_with_projection_metadata_until( + binding, + TerminalCommandState::SucceededPendingProjection, + serde_json::json!({"remote": "sealed"}), + metadata_bytes.clone(), + Duration::from_secs(300), + retention_expires_at, + ) + .unwrap(); + repo.complete_external_command(completion).await.unwrap(); + + let replay = match repo + .lookup_command(&key, CommandLookupScope::CommandName("order.create")) + .await + .unwrap() + { + CommandLookup::Replay(replay) => replay, + other => panic!("sealed external completion should replay, got {other:?}"), + }; + assert_eq!(replay.state, CommandLedgerState::SucceededPendingProjection); + assert_eq!(replay.projection_obligations, Vec::new()); + assert_eq!( + replay.projection_metadata.as_deref(), + Some(metadata_bytes.as_slice()) + ); + assert_eq!( + crate::graphql::protocol::CommandProjectionMetadataV1::from_json( + replay.projection_metadata.as_deref().unwrap(), + ) + .unwrap(), + metadata + ); +} + +async fn external_binding_and_attempt_fences_are_strict(repo: &R) +where + R: CommandLedgerAdapterConformance, +{ + let id = Uuid::now_v7().to_string(); + let attempt = acquire( + repo, + external_reservation(&id, 111, 112, "orders-2").unwrap(), + ) + .await; + let wrong_binding = attempt.complete_external( + external_binding("orders-3"), + TerminalCommandState::Succeeded, + serde_json::json!({"wrong": true}), + Vec::new(), + Duration::from_secs(300), + ); + assert!(matches!(wrong_binding, Err(CommandLedgerError::Invalid(_)))); + assert!(matches!( + repo.reserve_command(external_reservation(&id, 111, 112, "orders-3").unwrap()) + .await + .unwrap(), + ReservationOutcome::Conflict + )); + + let first = acquire( + repo, + external_reservation(&Uuid::now_v7().to_string(), 113, 114, "orders-4").unwrap(), + ) + .await; + let first_fence = first.fence(); + let key = first.key().clone(); + repo.mark_retryable_unknown(first_fence).await.unwrap(); + let second = acquire( + repo, + external_reservation(key.command_id(), 113, 114, "orders-4").unwrap(), + ) + .await; + let stale = first + .complete_external( + external_binding("orders-4"), + TerminalCommandState::Succeeded, + serde_json::json!({"winner": false}), + Vec::new(), + Duration::from_secs(300), + ) + .unwrap(); + assert!(matches!( + repo.complete_external_command(stale).await, + Err(CommandLedgerError::AttemptFenced { .. }) + )); + let live = second + .complete_external( + external_binding("orders-4"), + TerminalCommandState::Succeeded, + serde_json::json!({"winner": true}), + Vec::new(), + Duration::from_secs(300), + ) + .unwrap(); + repo.complete_external_command(live).await.unwrap(); +} + +async fn external_and_local_reservations_cannot_mix(repo: &R) +where + R: CommandLedgerAdapterConformance, +{ + let external_id = Uuid::now_v7().to_string(); + let external_attempt = acquire( + repo, + external_reservation(&external_id, 121, 122, "orders-5").unwrap(), + ) + .await; + let local_completion = external_attempt + .complete( + TerminalCommandState::Succeeded, + serde_json::json!({"local": true}), + Duration::from_secs(300), + ) + .unwrap(); + let local_result = repo + .commit_causal_batch(CausalCommitBatch::new( + CommitBatch::empty(), + local_completion, + )) + .await; + assert!(matches!(local_result, Err(CommandLedgerError::Invalid(_)))); + + let local_id = Uuid::now_v7().to_string(); + let local_attempt = acquire(repo, reservation(&local_id, 123, 124).unwrap()).await; + let external_result = local_attempt.complete_external( + external_binding("orders-5"), + TerminalCommandState::Succeeded, + serde_json::json!({"external": true}), + Vec::new(), + Duration::from_secs(300), + ); + assert!(matches!( + external_result, + Err(CommandLedgerError::Invalid(_)) + )); + + assert!(matches!( + repo.reserve_command(reservation(&external_id, 121, 122).unwrap()) + .await + .unwrap(), + ReservationOutcome::Conflict + )); +} + +async fn external_reclaims_keep_causation_and_binding(repo: &R) +where + R: CommandLedgerAdapterConformance, +{ + let id = Uuid::now_v7().to_string(); + let first = acquire( + repo, + external_reservation_with_policy( + &id, + 131, + 132, + "orders-6", + Duration::from_millis(100), + Duration::from_secs(300), + ) + .unwrap(), + ) + .await; + let causation = first.causation_id().clone(); + tokio::time::sleep(Duration::from_millis(300)).await; + let second = acquire( + repo, + external_reservation_with_policy( + &id, + 131, + 132, + "orders-6", + Duration::from_secs(30), + Duration::from_secs(300), + ) + .unwrap(), + ) + .await; + assert_eq!(second.causation_id(), &causation); + assert_eq!(second.attempt_number(), 2); + let expected_binding = external_binding("orders-6"); + assert_eq!(second.external_binding(), Some(&expected_binding)); + let completion = second + .complete_external( + external_binding("orders-6"), + TerminalCommandState::Succeeded, + serde_json::json!({"reclaimed": true}), + Vec::new(), + Duration::from_secs(300), + ) + .unwrap(); + repo.complete_external_command(completion).await.unwrap(); +} + async fn same_input_retries_and_identity_conflicts_conform(repo: &R) where R: CommandLedgerAdapterConformance, @@ -945,6 +1249,12 @@ where stale_fence_rolls_back_every_commit_participant(repo).await; compacted_expiry_is_a_permanent_tombstone(repo).await; expired_modeled_metadata_deadline_cannot_commit(repo).await; + external_completion_is_ledger_only_and_replayable(repo).await; + #[cfg(feature = "graphql")] + external_completion_retains_the_sealed_projection_metadata(repo).await; + external_binding_and_attempt_fences_are_strict(repo).await; + external_and_local_reservations_cannot_mix(repo).await; + external_reclaims_keep_causation_and_binding(repo).await; } #[test] @@ -1354,6 +1664,88 @@ fn modeled_projection_metadata_bounds_fail_before_completion() { )); } +#[test] +fn cell_ledger_external_binding_round_trips_and_legacy_rows_stay_unbound() { + let id = Uuid::now_v7().to_string(); + let request = external_reservation(&id, 141, 142, "orders-cell").unwrap(); + let started = SystemTime::UNIX_EPOCH + Duration::from_secs(100); + let row = CommandLedgerRecord::initial(&request, started).unwrap(); + let encoded = row.durable_cell_json().unwrap(); + let restored = CommandLedgerRecord::from_durable_cell_json(&encoded).unwrap(); + assert_eq!(restored.external_binding, row.external_binding); + assert_eq!(restored.durable_cell_key(), row.durable_cell_key()); + + // Cell rows written before external dispatch existed omit the optional + // field. They remain readable, but cannot be treated as externally bound + // evidence after a process restart. + let mut legacy: serde_json::Value = serde_json::from_str(&encoded).unwrap(); + legacy.as_object_mut().unwrap().remove("external_binding"); + let legacy = serde_json::to_string(&legacy).unwrap(); + let restored_legacy = CommandLedgerRecord::from_durable_cell_json(&legacy).unwrap(); + assert!(restored_legacy.external_binding.is_none()); + + let completed_id = Uuid::now_v7().to_string(); + let completed_request = + external_reservation(&completed_id, 143, 144, "orders-cell-completed").unwrap(); + let mut completed = CommandLedgerRecord::initial(&completed_request, started).unwrap(); + let metadata = br#"{"sealed":"projection"}"#.to_vec(); + let completion = completed + .acquired_attempt() + .unwrap() + .complete_external_with_projection_metadata_until( + external_binding("orders-cell-completed"), + TerminalCommandState::SucceededPendingProjection, + serde_json::json!({"remote": true}), + metadata.clone(), + Duration::from_secs(300), + started + Duration::from_secs(301), + ) + .unwrap(); + completed + .complete_external(&completion, started + Duration::from_secs(1)) + .unwrap(); + let restored_completed = + CommandLedgerRecord::from_durable_cell_json(&completed.durable_cell_json().unwrap()) + .unwrap(); + let replay = restored_completed.replay().unwrap(); + assert_eq!(replay.projection_metadata, Some(metadata)); + assert_eq!(replay.outcome, serde_json::json!({"remote": true})); +} + +#[test] +fn external_dispatch_binding_storage_is_versioned_and_canonical() { + let binding = ExternalDispatchBinding::new("aggregate-cell", "orders-1").unwrap(); + let encoded = binding.to_storage().unwrap(); + assert_eq!( + ExternalDispatchBinding::from_storage(&encoded).unwrap(), + binding + ); + assert!(matches!( + ExternalDispatchBinding::from_storage( + r#"{"version":1,"route_kind":"aggregate-cell","shard":"orders-1","extra":true}"#, + ), + Err(CommandLedgerError::Corrupt(_)) + )); + assert!(matches!( + ExternalDispatchBinding::from_storage( + r#"{"version":2,"route_kind":"aggregate-cell","shard":"orders-1"}"#, + ), + Err(CommandLedgerError::Corrupt(_)) + )); + assert!(matches!( + ExternalDispatchBinding::new("", "orders-1"), + Err(CommandLedgerError::Invalid(_)) + )); + assert!(matches!( + ExternalDispatchBinding::new("aggregate-cell", "orders\n1"), + Err(CommandLedgerError::Invalid(_)) + )); + assert!(matches!( + ExternalDispatchBinding::new("é".repeat(129), "orders-1"), + Err(CommandLedgerError::Invalid(_)) + )); +} + #[test] fn causal_batch_applies_the_authoritative_stamp_at_the_final_boundary() { use crate::outbox::OutboxMessage; @@ -1503,6 +1895,30 @@ async fn sqlite_terminal_replay_survives_pool_drop_and_reopen() { .await .unwrap(); + let external_id = Uuid::now_v7().to_string(); + let external_request = + external_reservation(&external_id, 83, 84, "orders-restart-external").unwrap(); + let external_key = external_request.key().clone(); + let external_binding = external_binding("orders-restart-external"); + let external_attempt = acquire(&repo, external_request).await; + let external_completion = external_attempt + .complete_external( + external_binding, + TerminalCommandState::Succeeded, + serde_json::json!({"remote": "restart"}), + Vec::new(), + Duration::from_secs(300), + ) + .unwrap(); + repo.complete_external_command(external_completion) + .await + .unwrap(); + assert!(repo + .get_stream(&StreamIdentity::new("remote-order", &external_id).unwrap()) + .await + .unwrap() + .is_none()); + repo.pool().close().await; drop(repo); @@ -1545,6 +1961,30 @@ async fn sqlite_terminal_replay_survives_pool_drop_and_reopen() { Some(expected_causation.as_str()) ); + let external_replay = match reopened + .lookup_command( + &external_key, + CommandLookupScope::CommandName("order.create"), + ) + .await + .unwrap() + { + CommandLookup::Replay(replay) => replay, + other => panic!("reopened external ledger row should replay, got {other:?}"), + }; + assert_eq!(external_replay.state, CommandLedgerState::Succeeded); + assert_eq!( + external_replay.outcome, + serde_json::json!({"remote": "restart"}) + ); + assert!(matches!( + reopened + .reserve_command(reservation(&external_id, 83, 84).unwrap()) + .await + .unwrap(), + ReservationOutcome::Conflict + )); + reopened.pool().close().await; drop(reopened); } diff --git a/src/command_ledger/traits.rs b/src/command_ledger/traits.rs index 76724d48..0c82dfd3 100644 --- a/src/command_ledger/traits.rs +++ b/src/command_ledger/traits.rs @@ -5,7 +5,8 @@ use crate::repository::{RepositoryError, StreamIdentity}; use super::{ AttemptFence, CausalCommitBatch, CausalStorageIdentity, CommandLedgerError, CommandLedgerKey, - CommandLookup, CommandLookupScope, CommandReservation, ReservationOutcome, + CommandLookup, CommandLookupScope, CommandReservation, ExternalCommandCompletion, + ReservationOutcome, }; /// Read capability used by the causal workspace. Unlike ordinary @@ -51,6 +52,14 @@ pub(crate) trait CommandLedgerStore: Send + Sync { attempt: AttemptFence, ) -> impl Future> + Send + '_; + /// Record a terminal receipt for a remote aggregate-cell command. This + /// updates only the ledger row and is never a substitute for a local + /// domain commit. + fn complete_external_command( + &self, + completion: ExternalCommandCompletion, + ) -> impl Future> + Send + '_; + #[allow(dead_code)] fn compact_expired_commands( &self, diff --git a/src/in_memory_repo/repository.rs b/src/in_memory_repo/repository.rs index 69f90d74..55db2227 100644 --- a/src/in_memory_repo/repository.rs +++ b/src/in_memory_repo/repository.rs @@ -14,7 +14,7 @@ use crate::command_ledger::{ AttemptFence, CausalCommitBatch, CausalGetStream, CausalRepositoryIdentity, CausalStorageIdentity, CausalTransactionalCommit, CommandCompletion, CommandLedgerError, CommandLedgerKey, CommandLedgerRecord, CommandLedgerStore, CommandLookup, CommandLookupScope, - CommandReservation, ReservationDecision, ReservationOutcome, + CommandReservation, ExternalCommandCompletion, ReservationDecision, ReservationOutcome, }; use crate::entity::{Entity, EventRecord}; use crate::outbox::OutboxMessage; @@ -384,6 +384,13 @@ impl InMemoryRepository { .ok_or_else(|| CommandLedgerError::AttemptFenced { command_id: completion.attempt().key().command_id().to_string(), })?; + if completion.attempt().external_binding().is_some() { + return Err(CommandLedgerError::Invalid( + "local causal completion cannot complete an externally bound reservation" + .into(), + ) + .into()); + } record.validate_live_attempt(&completion.attempt_fence(), crate::time::now())?; } @@ -692,6 +699,24 @@ impl CommandLedgerStore for InMemoryRepository { } } + fn complete_external_command( + &self, + completion: ExternalCommandCompletion, + ) -> impl Future> + Send + '_ { + async move { + let mut ledger = self + .command_ledger + .write() + .map_err(|_| RepositoryError::LockPoisoned("external command completion"))?; + let record = ledger.get_mut(completion.attempt().key()).ok_or_else(|| { + CommandLedgerError::AttemptFenced { + command_id: completion.attempt().key().command_id().to_string(), + } + })?; + record.complete_external(&completion, crate::time::now()) + } + } + fn compact_expired_commands( &self, limit: usize, diff --git a/src/microsvc/cell_host/sql_store.rs b/src/microsvc/cell_host/sql_store.rs index 11ac0a05..b39e00ef 100644 --- a/src/microsvc/cell_host/sql_store.rs +++ b/src/microsvc/cell_host/sql_store.rs @@ -9,7 +9,7 @@ use crate::command_ledger::{ AttemptFence, CausalCommitBatch, CausalGetStream, CausalRepositoryIdentity, CausalStorageIdentity, CausalTransactionalCommit, CommandCompletion, CommandLedgerError, CommandLedgerKey, CommandLedgerStore, CommandLookup, CommandLookupScope, CommandReservation, - ReservationOutcome, + ExternalCommandCompletion, ReservationOutcome, }; use crate::entity::Entity; use crate::microsvc::HasOutboxStore; @@ -235,6 +235,14 @@ impl CommandLedgerStore for CellSqlRepository { self.connection .transaction(|executor| finish_sql(ledger::mark_retryable(executor, &attempt))) } + + async fn complete_external_command( + &self, + completion: ExternalCommandCompletion, + ) -> Result<(), CommandLedgerError> { + self.connection + .transaction(|executor| finish_sql(ledger::complete_external(executor, &completion))) + } async fn compact_expired_commands(&self, limit: usize) -> Result { self.connection .transaction(|executor| finish_sql(ledger::compact(executor, limit))) diff --git a/src/microsvc/cell_host/store.rs b/src/microsvc/cell_host/store.rs index af4b3b19..23d26394 100644 --- a/src/microsvc/cell_host/store.rs +++ b/src/microsvc/cell_host/store.rs @@ -14,7 +14,8 @@ use super::sql_store::CellSqlRepository; use crate::command_ledger::{ AttemptFence, CausalCommitBatch, CausalGetStream, CausalRepositoryIdentity, CausalStorageIdentity, CausalTransactionalCommit, CommandLedgerError, CommandLedgerKey, - CommandLedgerStore, CommandLookup, CommandLookupScope, CommandReservation, ReservationOutcome, + CommandLedgerStore, CommandLookup, CommandLookupScope, CommandReservation, + ExternalCommandCompletion, ReservationOutcome, }; use crate::entity::Entity; #[cfg(not(all(feature = "workers-rs", target_arch = "wasm32")))] @@ -542,6 +543,13 @@ impl CommandLedgerStore for CellStreamStore { CommandLedgerStore::mark_retryable_unknown(&self.inner, attempt) } + fn complete_external_command( + &self, + completion: ExternalCommandCompletion, + ) -> impl Future> + Send + '_ { + CommandLedgerStore::complete_external_command(&self.inner, completion) + } + fn compact_expired_commands( &self, limit: usize, diff --git a/src/microsvc/service/tests.rs b/src/microsvc/service/tests.rs index dc2671b9..e9450f88 100644 --- a/src/microsvc/service/tests.rs +++ b/src/microsvc/service/tests.rs @@ -16,7 +16,8 @@ use crate::command::{CommandInputType, CommandOutputType, CommandTypeDef, Comman use crate::command_ledger::{ AttemptFence, CausalCommitBatch, CausalGetStream, CausalRepositoryIdentity, CausalTransactionalCommit, CommandLedgerError, CommandLedgerKey, CommandLedgerState, - CommandLedgerStore, CommandLookup, CommandLookupScope, CommandReservation, ReservationOutcome, + CommandLedgerStore, CommandLookup, CommandLookupScope, CommandReservation, + ExternalCommandCompletion, ReservationOutcome, }; #[cfg(feature = "graphql")] use crate::graphql::identity::VerifiedPrincipal; @@ -1017,6 +1018,13 @@ impl CommandLedgerStore for AmbiguousCommitRepository { CommandLedgerStore::mark_retryable_unknown(&self.inner, attempt) } + fn complete_external_command( + &self, + completion: ExternalCommandCompletion, + ) -> impl Future> + Send + '_ { + CommandLedgerStore::complete_external_command(&self.inner, completion) + } + fn compact_expired_commands( &self, limit: usize, diff --git a/src/postgres_repo/mod.rs b/src/postgres_repo/mod.rs index 041ba1f9..f58cab00 100644 --- a/src/postgres_repo/mod.rs +++ b/src/postgres_repo/mod.rs @@ -57,7 +57,7 @@ impl crate::sqlx_repo::repo::SqlxRepoBackend for Postgres { const CONFLICT_REREAD_IN_TX: bool = false; const NOW: &'static str = "now()"; const COMMAND_LEDGER_SELECT: &'static str = "command_name, command_contract_hash, \ - input_hash, state, causation_id, attempt_token, attempt_number, \ + input_hash, state, causation_id, attempt_token, attempt_number, external_binding, \ EXTRACT(EPOCH FROM lease_expires_at)::double precision AS lease_expires_at, \ outcome::text AS outcome, \ EXTRACT(EPOCH FROM created_at)::double precision AS created_at, \ diff --git a/src/queued_repo/repository.rs b/src/queued_repo/repository.rs index d4138054..58e4125d 100644 --- a/src/queued_repo/repository.rs +++ b/src/queued_repo/repository.rs @@ -9,7 +9,8 @@ use std::sync::Arc; use crate::command_ledger::{ AttemptFence, CausalCommitBatch, CausalGetStream, CausalRepositoryIdentity, CausalStorageIdentity, CausalTransactionalCommit, CommandLedgerError, CommandLedgerKey, - CommandLedgerStore, CommandLookup, CommandLookupScope, CommandReservation, ReservationOutcome, + CommandLedgerStore, CommandLookup, CommandLookupScope, CommandReservation, + ExternalCommandCompletion, ReservationOutcome, }; use crate::entity::Entity; use crate::lock::{InMemoryLockManager, Lock, LockManager}; @@ -243,6 +244,13 @@ where self.inner.mark_retryable_unknown(attempt) } + fn complete_external_command( + &self, + completion: ExternalCommandCompletion, + ) -> impl Future> + Send + '_ { + self.inner.complete_external_command(completion) + } + fn compact_expired_commands( &self, limit: usize, diff --git a/src/repository/migrations.rs b/src/repository/migrations.rs index 8a73455e..5d0a10b1 100644 --- a/src/repository/migrations.rs +++ b/src/repository/migrations.rs @@ -16,5 +16,5 @@ include!(concat!(env!("OUT_DIR"), "/migration_inventory.rs")); pub(crate) fn cell_migrations() -> impl Iterator { SQLITE_MIGRATIONS .iter() - .filter(|migration| matches!(migration.version, 1 | 2 | 4)) + .filter(|migration| matches!(migration.version, 1 | 2 | 4 | 8)) } diff --git a/src/repository/sql/ledger.rs b/src/repository/sql/ledger.rs index 2e1ed41b..1b494f3b 100644 --- a/src/repository/sql/ledger.rs +++ b/src/repository/sql/ledger.rs @@ -5,7 +5,8 @@ use crate::command_ledger::{ AttemptFence, AttemptToken, CanonicalInputHash, CausationId, CommandCompletion, CommandContractFingerprint, CommandId, CommandLedgerError, CommandLedgerKey, CommandLedgerRecord, CommandLedgerState, CommandLookup, CommandLookupScope, CommandReservation, - PrincipalPartitionId, ReservationDecision, ReservationOutcome, + ExternalCommandCompletion, ExternalDispatchBinding, PrincipalPartitionId, ReservationDecision, + ReservationOutcome, }; use std::time::SystemTime; @@ -29,6 +30,11 @@ fn record_from_row( let record = CommandLedgerRecord { key, command_name: row.text("command_name")?, + external_binding: row + .optional_text("external_binding")? + .as_deref() + .map(ExternalDispatchBinding::from_storage) + .transpose()?, contract_fingerprint: CommandContractFingerprint::try_from_slice( &row.bytes("command_contract_hash")?, ) @@ -126,6 +132,11 @@ pub(crate) async fn preflight( command_id: fence.key().command_id().to_string(), } })?; + if completion.attempt().external_binding().is_some() { + return Err(CommandLedgerError::Invalid( + "local causal completion cannot complete an externally bound reservation".into(), + )); + } let now = now(executor).await?; record.validate_live_attempt(&fence, now) } @@ -137,7 +148,7 @@ pub(crate) async fn insert_reservation( let mut builder = Statement::new( "INSERT INTO command_ledger (service_id, principal_partition, command_id, \ command_name, command_contract_hash, input_hash, state, causation_id, attempt_token, \ - attempt_number, lease_expires_at, outcome, created_at, updated_at, completed_at, \ + attempt_number, external_binding, lease_expires_at, outcome, created_at, updated_at, completed_at, \ retention_expires_at, compacted_at) VALUES (", ); builder.push_bind(reservation.key().service_id()); @@ -160,6 +171,15 @@ pub(crate) async fn insert_reservation( builder.push(", "); builder.push_bind(1_i64); builder.push(", "); + let external_binding = reservation + .external_binding() + .map(ExternalDispatchBinding::to_storage) + .transpose()?; + match external_binding.as_deref() { + Some(binding) => builder.push_bind(binding), + None => builder.push("NULL"), + } + builder.push(", "); builder.part(SqlPart::LedgerDeadline(reservation.lease())); builder.push(", NULL, "); builder.part(SqlPart::LedgerNow); @@ -265,6 +285,7 @@ pub(crate) async fn complete( builder.push_bind(fence.contract_fingerprint_bytes().as_slice()); builder.push(" AND input_hash = "); builder.push_bind(fence.input_hash_bytes().as_slice()); + builder.push(" AND external_binding IS NULL"); builder.push(" AND state = 'in_progress' AND causation_id = "); builder.push_bind(fence.causation_id().as_str()); builder.push(" AND attempt_token = "); @@ -286,6 +307,60 @@ pub(crate) async fn complete( Ok(()) } +pub(crate) async fn complete_external( + executor: &mut E, + completion: &ExternalCommandCompletion, +) -> Result<(), CommandLedgerError> { + let fence = completion.attempt_fence(); + let attempt_number = signed(fence.attempt_number(), "command ledger attempt number")?; + let terminal_state = CommandLedgerState::from(completion.state()).as_str(); + let retention_expires_at = completion.retention_expires_at(); + let binding = completion.binding().to_storage()?; + let mut builder = Statement::new("UPDATE command_ledger SET state = "); + builder.push_bind(terminal_state); + builder.push(", attempt_token = NULL, lease_expires_at = NULL, outcome = "); + builder.part(SqlPart::LedgerJson(completion.replay_json().into())); + builder.push(", updated_at = "); + builder.part(SqlPart::LedgerNow); + builder.push(", completed_at = "); + builder.part(SqlPart::LedgerNow); + builder.push(", retention_expires_at = "); + match retention_expires_at.as_ref() { + Some(deadline) => builder.push_bind(SqlBind::Timestamp(*deadline)), + None => builder.part(SqlPart::LedgerDeadline(completion.retention())), + } + builder.push(", compacted_at = NULL WHERE service_id = "); + builder.push_bind(fence.key().service_id()); + builder.push(" AND principal_partition = "); + builder.push_bind(fence.key().principal_partition()); + builder.push(" AND command_id = "); + builder.push_bind(fence.key().command_id()); + builder.push(" AND command_contract_hash = "); + builder.push_bind(fence.contract_fingerprint_bytes().as_slice()); + builder.push(" AND input_hash = "); + builder.push_bind(fence.input_hash_bytes().as_slice()); + builder.push(" AND external_binding = "); + builder.push_bind(binding.as_str()); + builder.push(" AND state = 'in_progress' AND causation_id = "); + builder.push_bind(fence.causation_id().as_str()); + builder.push(" AND attempt_token = "); + builder.push_bind(fence.attempt_token().as_str()); + builder.push(" AND attempt_number = "); + builder.push_bind(attempt_number); + builder.push(" AND lease_expires_at > "); + builder.part(SqlPart::LedgerNow); + if let Some(deadline) = retention_expires_at.as_ref() { + builder.push(" AND "); + builder.part(SqlPart::LedgerDeadlineIsLive(*deadline)); + } + if executor.execute(builder).await? != 1 { + return Err(CommandLedgerError::AttemptFenced { + command_id: fence.key().command_id().to_string(), + }); + } + Ok(()) +} + pub(crate) async fn reserve( executor: &mut impl SqlExecutor, reservation: &CommandReservation, diff --git a/src/repository/sqlite_codec.rs b/src/repository/sqlite_codec.rs index 34dd1257..ed40fedd 100644 --- a/src/repository/sqlite_codec.rs +++ b/src/repository/sqlite_codec.rs @@ -74,4 +74,4 @@ mod tests { assert!(encode(UNIX_EPOCH - Duration::from_secs(1)).is_err()); } } -pub(crate) const COMMAND_LEDGER_SELECT: &str = "command_name, command_contract_hash, input_hash, state, causation_id, attempt_token, attempt_number, lease_expires_at, outcome, created_at, updated_at, completed_at, retention_expires_at, compacted_at"; +pub(crate) const COMMAND_LEDGER_SELECT: &str = "command_name, command_contract_hash, input_hash, state, causation_id, attempt_token, attempt_number, external_binding, lease_expires_at, outcome, created_at, updated_at, completed_at, retention_expires_at, compacted_at"; diff --git a/src/sqlx_repo/repo/backend.rs b/src/sqlx_repo/repo/backend.rs index 3bb2ce53..71eeac17 100644 --- a/src/sqlx_repo/repo/backend.rs +++ b/src/sqlx_repo/repo/backend.rs @@ -47,7 +47,7 @@ mod tests { .iter() .map(|migration| migration.sql) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8]); assert_eq!( descriptions, vec![ @@ -57,7 +57,8 @@ mod tests { "command ledger atomic state", "projection source snapshots", "gateway dependency versions", - "projection program identity" + "projection program identity", + "external command binding" ] ); assert_eq!( @@ -91,6 +92,10 @@ mod tests { env!("CARGO_MANIFEST_DIR"), "/migrations/sqlite/0007_projection_program_identity.sql" )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/migrations/sqlite/0008_external_command_binding.sql" + )), ] ); } @@ -110,7 +115,7 @@ mod tests { .iter() .map(|migration| migration.sql) .collect::>(); - assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7]); + assert_eq!(versions, vec![1, 2, 3, 4, 5, 6, 7, 8]); assert_eq!( descriptions, vec![ @@ -120,7 +125,8 @@ mod tests { "command ledger atomic state", "projection source snapshots", "gateway dependency versions", - "projection program identity" + "projection program identity", + "external command binding" ] ); assert_eq!( @@ -154,6 +160,10 @@ mod tests { env!("CARGO_MANIFEST_DIR"), "/migrations/postgres/0007_projection_program_identity.sql" )), + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/migrations/postgres/0008_external_command_binding.sql" + )), ] ); } diff --git a/src/sqlx_repo/repo/commit.rs b/src/sqlx_repo/repo/commit.rs index c42cc773..4e7fd6c9 100644 --- a/src/sqlx_repo/repo/commit.rs +++ b/src/sqlx_repo/repo/commit.rs @@ -318,6 +318,24 @@ where .await } + async fn complete_external_command( + &self, + completion: ExternalCommandCompletion, + ) -> Result<(), CommandLedgerError> { + let mut tx = self.pool.begin().await.map_err(|error| { + repository_storage_error::("begin external command completion", error) + })?; + crate::repository::sql::ledger::complete_external( + &mut executor::ConnectionExecutor::(&mut *tx), + &completion, + ) + .await?; + tx.commit().await.map_err(|error| { + repository_storage_error::("commit external command completion", error) + })?; + Ok(()) + } + async fn compact_expired_commands(&self, limit: usize) -> Result { if limit == 0 { return Ok(0); diff --git a/src/sqlx_repo/repo/mod.rs b/src/sqlx_repo/repo/mod.rs index 194dd755..9b304726 100644 --- a/src/sqlx_repo/repo/mod.rs +++ b/src/sqlx_repo/repo/mod.rs @@ -30,7 +30,7 @@ use crate::command_ledger::{ AttemptFence, CausalCommitBatch, CausalGetStream, CausalRepositoryIdentity, CausalStorageIdentity, CausalTransactionalCommit, CommandCompletion, CommandLedgerError, CommandLedgerKey, CommandLedgerStore, CommandLookup, CommandLookupScope, CommandReservation, - ReservationOutcome, + ExternalCommandCompletion, ReservationOutcome, }; use crate::entity::{Entity, EventRecord}; use crate::outbox::{OutboxMessage, OutboxMessageStatus}; diff --git a/tests/postgres_repository/main.rs b/tests/postgres_repository/main.rs index db25e06b..f5dd02de 100644 --- a/tests/postgres_repository/main.rs +++ b/tests/postgres_repository/main.rs @@ -234,7 +234,7 @@ async fn projected_command_ledger_rows_upgrade_to_atomic_and_preserve_checks() { .fetch_one(repo.pool()) .await .unwrap(); - assert_eq!(latest_version, 7); + assert_eq!(latest_version, 8); let invalid_service = sqlx::query( r#" diff --git a/tests/sqlite_repository/main.rs b/tests/sqlite_repository/main.rs index 2a98aff4..8f62df91 100644 --- a/tests/sqlite_repository/main.rs +++ b/tests/sqlite_repository/main.rs @@ -201,7 +201,7 @@ async fn projected_command_ledger_rows_upgrade_to_atomic_without_schema_drift() .fetch_one(repo.pool()) .await .unwrap(); - assert_eq!(latest_version, 7); + assert_eq!(latest_version, 8); let created_at_type: String = sqlx::query_scalar( "SELECT typeof(created_at) FROM command_ledger WHERE service_id = 'service'", From 34aa3881bf2551801b70895fe22b9dcfc41a9c92 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Wed, 9 Sep 2026 23:53:31 -0500 Subject: [PATCH 07/10] feat: persist external cell command completions --- README.md | 11 + src/command_dispatch/host.rs | 36 ++- src/command_ledger/reservation.rs | 8 + src/microsvc/cell_host/causal.rs | 30 ++- src/microsvc/cell_host/command.rs | 178 +++++++------ src/microsvc/cell_host/mod.rs | 4 +- src/microsvc/mod.rs | 2 + src/microsvc/service/causal.rs | 149 +++++++++++ src/microsvc/service/mod.rs | 2 +- src/microsvc/service/routes.rs | 288 ++++++++++++++++++++- src/microsvc/service/runtime.rs | 83 +++++- src/microsvc/service/tests.rs | 75 ++++++ tests/causal_wait_path/main.rs | 411 +++++++++++++++++++++++++++++- tests/celld/worker/src/lib.rs | 20 +- 14 files changed, 1195 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index 9ea07ee8..3840938e 100644 --- a/README.md +++ b/README.md @@ -752,6 +752,17 @@ without a gateway binding remain explicit unknowns until a matching gateway reservation and trusted completion protocol can be established; they are never retrofitted into a different route or silently replayed. +The celld command host reserves that gateway intent before making the cell +request. The reservation's causation ID and logical route binding must be +echoed by the trusted cell receipt; a changed command ID, input, causation, or +route is rejected before it can complete the gateway row. A terminal cell +receipt is durably recorded in the gateway ledger before the response is +returned. Status checks current authorization and re-evaluates the durable +receipt's projection evidence, so an in-memory completed-status cache is never +the source of truth. An ambiguous transport result remains retryable against +the same reservation and cell; it does not invoke the local domain handler or +re-publish the cell's events and outbox. + In v5, `OutboxStore::complete` and `complete_many` remove delivered rows instead of retaining `Published` records. Repeating settlement of a removed row returns `NotFound`; stale claims on existing rows still return `InvalidState`. diff --git a/src/command_dispatch/host.rs b/src/command_dispatch/host.rs index 7e44708c..c7f14b3c 100644 --- a/src/command_dispatch/host.rs +++ b/src/command_dispatch/host.rs @@ -9,8 +9,8 @@ use std::time::Duration; use crate::graphql::identity::VerifiedPrincipal; use crate::graphql::protocol::ProtocolResponseAccumulator; use crate::microsvc::cell_host::{ - InternalHttpSecret, CELL_INTERNAL_SECRET_HEADER, CELL_PRINCIPAL_PARTITION_HEADER, - CELL_SERVICE_ID_HEADER, + InternalHttpSecret, CELL_CAUSATION_ID_HEADER, CELL_INTERNAL_SECRET_HEADER, + CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, }; use crate::microsvc::{ CausalCommandPublicStatus, CausalDispatchError, CausalDispatchResult, Service, Session, @@ -340,7 +340,7 @@ impl HttpCommandHost { input: Value, session: &Session, ) -> Result<(u16, Value), CausalDispatchError> { - self.post_wait_path_inner(command, command_id, input, session, None) + self.post_wait_path_inner(command, command_id, input, session, None, None) .await } @@ -355,6 +355,30 @@ impl HttpCommandHost { session: &Session, service_id: &str, principal_partition: &str, + ) -> Result<(u16, Value), CausalDispatchError> { + self.post_cell_wait_path_with_causation( + command, + command_id, + input, + session, + service_id, + principal_partition, + None, + ) + .await + } + + /// POST a cell wait-path command while carrying the gateway's durable + /// causation through the authenticated internal boundary. + pub async fn post_cell_wait_path_with_causation( + &self, + command: &str, + command_id: &str, + input: Value, + session: &Session, + service_id: &str, + principal_partition: &str, + causation_id: Option<&str>, ) -> Result<(u16, Value), CausalDispatchError> { let first = self .post_wait_path_inner( @@ -363,6 +387,7 @@ impl HttpCommandHost { input.clone(), session, Some((service_id, principal_partition)), + causation_id, ) .await; if !matches!(&first, Ok((status, _)) if *status >= 500) { @@ -380,6 +405,7 @@ impl HttpCommandHost { input, session, Some((service_id, principal_partition)), + causation_id, ) .await } @@ -391,6 +417,7 @@ impl HttpCommandHost { input: Value, session: &Session, cell_identity: Option<(&str, &str)>, + causation_id: Option<&str>, ) -> Result<(u16, Value), CausalDispatchError> { let mut request = self.request_json( command, @@ -414,6 +441,9 @@ impl HttpCommandHost { request = request .header(CELL_SERVICE_ID_HEADER, service_id) .header(CELL_PRINCIPAL_PARTITION_HEADER, principal_partition); + if let Some(causation_id) = causation_id { + request = request.header(CELL_CAUSATION_ID_HEADER, causation_id); + } } let response = request.send().await.map_err(|err| { CausalDispatchError::Internal(format!("wait-path HTTP failed: {err}")) diff --git a/src/command_ledger/reservation.rs b/src/command_ledger/reservation.rs index e22cc854..53afef71 100644 --- a/src/command_ledger/reservation.rs +++ b/src/command_ledger/reservation.rs @@ -165,6 +165,14 @@ impl CommandReservation { self } + /// Reuse the causation allocated by a trusted external gateway. The + /// command ID and canonical input still fence the cell row; this field + /// makes the remote receipt prove the same logical command attempt. + pub(crate) fn with_causation_id(mut self, causation_id: CausationId) -> Self { + self.candidate_causation = causation_id; + self + } + pub(crate) fn key(&self) -> &CommandLedgerKey { &self.key } diff --git a/src/microsvc/cell_host/causal.rs b/src/microsvc/cell_host/causal.rs index df11ae6f..532f85b1 100644 --- a/src/microsvc/cell_host/causal.rs +++ b/src/microsvc/cell_host/causal.rs @@ -40,6 +40,11 @@ pub const CELL_SERVICE_ID_HEADER: &str = "x-distributed-service-id"; /// This is an opaque server-derived value, never a public command argument. pub const CELL_PRINCIPAL_PARTITION_HEADER: &str = "x-distributed-principal-partition"; +/// Internal wait-path header carrying the gateway's durable causation ID. +/// The shared internal secret authenticates this value; public clients cannot +/// select it. +pub const CELL_CAUSATION_ID_HEADER: &str = "x-distributed-causation-id"; + /// Trusted command-ledger identity supplied by the cell's authenticated host. /// /// `principal_partition` is the opaque, server-derived partition produced by @@ -47,6 +52,7 @@ pub const CELL_PRINCIPAL_PARTITION_HEADER: &str = "x-distributed-principal-parti #[derive(Clone, Debug)] pub struct CellCommandIdentity { key: CommandLedgerKey, + causation_id: Option, } impl CellCommandIdentity { @@ -60,7 +66,10 @@ impl CellCommandIdentity { PrincipalPartitionId::new(principal_partition).map_err(internal_ledger_error)?; let key = CommandLedgerKey::new(service_id, principal_partition, command_id) .map_err(internal_ledger_error)?; - Ok(Self { key }) + Ok(Self { + key, + causation_id: None, + }) } pub fn service_id(&self) -> &str { @@ -71,6 +80,25 @@ impl CellCommandIdentity { self.key.command_id() } + /// Bind a trusted gateway causation ID to the cell reservation. This is + /// used only across the authenticated internal wait-path boundary; a cell + /// identity without it retains the standalone cell-command behavior. + pub fn with_causation_id( + mut self, + causation_id: impl AsRef, + ) -> Result { + let causation_id = crate::command_ledger::CausationId::parse_stored( + causation_id.as_ref().to_string(), + ) + .map_err(internal_ledger_error)?; + self.causation_id = Some(causation_id); + Ok(self) + } + + pub(crate) fn causation_id(&self) -> Option<&crate::command_ledger::CausationId> { + self.causation_id.as_ref() + } + pub(crate) fn key(&self) -> &CommandLedgerKey { &self.key } diff --git a/src/microsvc/cell_host/command.rs b/src/microsvc/cell_host/command.rs index da7e59cd..9c7b44df 100644 --- a/src/microsvc/cell_host/command.rs +++ b/src/microsvc/cell_host/command.rs @@ -4,9 +4,7 @@ //! aggregate Worker owns outbox delivery through celld Queue; this host only //! invokes commands and seals the returned projection evidence. -use std::collections::{HashMap, VecDeque}; -use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; +use std::sync::Arc; use async_trait::async_trait; use serde_json::Value; @@ -19,57 +17,10 @@ use crate::command_dispatch::{ use crate::graphql::identity::VerifiedPrincipal; use crate::graphql::protocol::ProtocolResponseAccumulator; use crate::microsvc::{ - CausalCommandPublicStatus, CausalDispatchError, CausalDispatchResult, Service, Session, + CausalCommandPublicStatus, CausalDispatchError, CausalDispatchResult, ExternalCausalReservation, + Service, Session, }; - -const COMPLETED_STATUS_CACHE_LIMIT: usize = 4_096; -const COMPLETED_STATUS_CACHE_TTL: Duration = Duration::from_secs(15 * 60); - -type CompletedStatusKey = (String, String); - -#[derive(Default)] -struct CompletedStatusCache { - entries: HashMap, - order: VecDeque, -} - -impl CompletedStatusCache { - fn insert(&mut self, key: CompletedStatusKey, status: CausalCommandPublicStatus) { - self.purge_expired(); - if self.entries.contains_key(&key) { - self.order.retain(|existing| existing != &key); - self.order.push_back(key.clone()); - self.entries.insert(key, (Instant::now(), status)); - return; - } - while self.entries.len() >= COMPLETED_STATUS_CACHE_LIMIT { - let Some(evicted) = self.order.pop_front() else { - break; - }; - self.entries.remove(&evicted); - } - self.order.push_back(key.clone()); - self.entries.insert(key, (Instant::now(), status)); - } - - fn get(&mut self, key: &CompletedStatusKey) -> Option { - self.purge_expired(); - self.entries.get(key).map(|(_, status)| status.clone()) - } - - fn purge_expired(&mut self) { - let now = Instant::now(); - while self.order.front().is_some_and(|key| { - self.entries.get(key).is_none_or(|(inserted, _)| { - now.duration_since(*inserted) >= COMPLETED_STATUS_CACHE_TTL - }) - }) { - if let Some(expired) = self.order.pop_front() { - self.entries.remove(&expired); - } - } - } -} +use crate::command_ledger::ExternalDispatchBinding; /// One aggregate's cell wait-path: command names, URL kind, shard id, payload. #[derive(Clone, Copy)] @@ -102,7 +53,6 @@ pub struct CelldCommandHost { http: HttpCommandHost, local: LocalCommandHost, routes: Vec, - completed: Arc>, } impl CelldCommandHost { @@ -117,7 +67,6 @@ impl CelldCommandHost { http, local: LocalCommandHost::new(service), routes: Vec::new(), - completed: Arc::new(Mutex::new(CompletedStatusCache::default())), }) } @@ -140,12 +89,6 @@ impl CelldCommandHost { }) } - fn remember_completed(&self, key: (String, String), status: CausalCommandPublicStatus) { - let Ok(mut completed) = self.completed.lock() else { - return; - }; - completed.insert(key, status); - } } fn remote_dispatch_error(status: u16, body: &Value) -> CausalDispatchError { @@ -209,34 +152,110 @@ impl CommandHost for CelldCommandHost { })?; let service_id = self.service_id()?.to_string(); let principal_partition = principal.partition_for_service(&service_id); - let http = self.http.retarget_segments(&[route.kind, &shard])?; - let (status, body) = http - .post_cell_wait_path( + let binding = ExternalDispatchBinding::new(route.kind, &shard) + .map_err(|error| CausalDispatchError::BadRequest(error.to_string()))?; + let reservation = self + .local + .service() + .reserve_external_causal( + command, + command_id, + input.clone(), + session.clone(), + principal, + binding, + ) + .await?; + let attempt = match reservation { + ExternalCausalReservation::Acquired(attempt) => attempt, + ExternalCausalReservation::Replay(replay) => return Ok(replay), + }; + let http = match self.http.retarget_segments(&[route.kind, &shard]) { + Ok(http) => http, + Err(error) => { + let _ = self + .local + .service() + .abandon_external_causal(command, attempt) + .await; + return Err(error); + } + }; + let (status, body) = match http + .post_cell_wait_path_with_causation( command, command_id, input.clone(), &session, &service_id, &principal_partition, + Some(attempt.causation_id()), ) - .await?; + .await + { + Ok(response) => response, + Err(error) => { + let _ = self + .local + .service() + .abandon_external_causal(command, attempt) + .await; + return Err(error); + } + }; if status >= 400 { - return Err(remote_dispatch_error(status, &body)); + let error = remote_dispatch_error(status, &body); + let _ = self + .local + .service() + .abandon_external_causal(command, attempt) + .await; + return Err(error); + } + let remote = match CausalDispatchResult::from_wait_path_wire(body) { + Ok(remote) => remote, + Err(error) => { + let _ = self + .local + .service() + .abandon_external_causal(command, attempt) + .await; + return Err(CausalDispatchError::Internal(format!( + "wait-path decode: {error}" + ))); + } + }; + if let Err(error) = attempt.validate_remote(&remote) { + let _ = self + .local + .service() + .abandon_external_causal(command, attempt) + .await; + return Err(error); } - let remote = CausalDispatchResult::from_wait_path_wire(body) - .map_err(|error| CausalDispatchError::Internal(format!("wait-path decode: {error}")))?; let payload = (route.payload)(command, &input, remote.payload(), &session); let mut remote = remote.with_payload(payload); if let Some(protocol) = protocol { - remote = self + remote = match self .local .service() - .seal_wait_path_dispatch(command, &protocol, remote)?; + .seal_wait_path_dispatch(command, &protocol, remote) + { + Ok(remote) => remote, + Err(error) => { + let _ = self + .local + .service() + .abandon_external_causal(command, attempt) + .await; + return Err(error); + } + }; } - self.remember_completed( - (principal_partition, command_id.to_string()), - remote.public_status(), - ); + self.local + .service() + .complete_external_causal(command, attempt, &remote) + .await?; Ok(remote) } @@ -248,17 +267,6 @@ impl CommandHost for CelldCommandHost { protocol: Option, ) -> Result { validate_principal_session_if_present(session, &principal)?; - let service_id = self.service_id()?; - let principal_partition = principal.partition_for_service(service_id); - let key = (principal_partition, command_id.to_string()); - if let Some(status) = self - .completed - .lock() - .ok() - .and_then(|mut guard| guard.get(&key)) - { - return Ok(status); - } self.local .status(command_id, session, principal, protocol) .await diff --git a/src/microsvc/cell_host/mod.rs b/src/microsvc/cell_host/mod.rs index 51f94eb7..8209bd24 100644 --- a/src/microsvc/cell_host/mod.rs +++ b/src/microsvc/cell_host/mod.rs @@ -28,8 +28,8 @@ mod store; mod wire; pub use causal::{ - CellCommandIdentity, CellDispatchError, CellDispatchResult, CELL_PRINCIPAL_PARTITION_HEADER, - CELL_SERVICE_ID_HEADER, + CellCommandIdentity, CellDispatchError, CellDispatchResult, CELL_CAUSATION_ID_HEADER, + CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, }; pub use cell::{instance_name, parent_cell_name, AggregateCell, CellNamespace}; #[cfg(feature = "workers-rs")] diff --git a/src/microsvc/mod.rs b/src/microsvc/mod.rs index 2b38adc8..3ea157bb 100644 --- a/src/microsvc/mod.rs +++ b/src/microsvc/mod.rs @@ -106,6 +106,8 @@ pub use runtime::{DEFAULT_MAX_PUBLISH_ATTEMPTS, DEFAULT_PUBLISH_LEASE}; pub(crate) use service::CausalCommandProjectionEvidence; #[cfg(feature = "graphql")] pub use service::GraphqlServiceBindError; +#[cfg(feature = "graphql")] +pub(crate) use service::ExternalCausalReservation; pub use service::{ direct_read_model, invoke_transition, require_loaded, CausalCommandContext, CausalCommitBuilder, CausalRepository, CommandRequest, CommandResponse, DeliveryKind, diff --git a/src/microsvc/service/causal.rs b/src/microsvc/service/causal.rs index aa5a705a..7c70703b 100644 --- a/src/microsvc/service/causal.rs +++ b/src/microsvc/service/causal.rs @@ -221,6 +221,121 @@ pub struct CausalDispatchResult { pub(crate) projection_events: Vec, } +/// Gateway-side capability for one externally dispatched command. The attempt +/// owns the durable reservation and is consumed only by terminal completion or +/// retryable-unknown recovery; it is never reconstructed from the wire. +#[cfg(feature = "graphql")] +pub(crate) struct ExternalCausalAttempt { + pub(crate) command_name: String, + pub(crate) attempt: CommandAttempt, + pub(crate) retention: Duration, +} + +#[cfg(feature = "graphql")] +pub(crate) enum ExternalCausalReservation { + Acquired(ExternalCausalAttempt), + Replay(CausalDispatchResult), +} + +#[cfg(feature = "graphql")] +impl ExternalCausalAttempt { + pub(crate) fn command_id(&self) -> &str { + self.attempt.key().command_id() + } + + pub(crate) fn causation_id(&self) -> &str { + self.attempt.causation_id().as_str() + } + + pub(crate) fn fence(&self) -> AttemptFence { + self.attempt.fence() + } + + pub(crate) fn validate_remote( + &self, + result: &CausalDispatchResult, + ) -> Result<(), CausalDispatchError> { + if result.command_id() != self.command_id() { + return Err(CausalDispatchError::Internal( + "cell wait-path returned a different command ID".into(), + )); + } + if result.causation_id() != self.causation_id() { + return Err(CausalDispatchError::Internal( + "cell wait-path returned a different causation ID".into(), + )); + } + Ok(()) + } + + pub(crate) fn into_external_completion( + self, + result: &CausalDispatchResult, + ) -> Result { + self.validate_remote(result)?; + let state = match result.receipt.state { + CommandLedgerState::Succeeded => TerminalCommandState::Succeeded, + CommandLedgerState::SucceededPendingProjection => { + TerminalCommandState::SucceededPendingProjection + } + CommandLedgerState::Atomic => { + return Err(CausalDispatchError::BadRequest( + "atomic commands cannot be dispatched to an aggregate cell".into(), + )); + } + other => { + return Err(CausalDispatchError::Internal(format!( + "cell wait-path returned non-terminal state `{}`", + other.as_str() + ))); + } + }; + if !result.receipt.obligations.is_empty() && result.receipt.projection_metadata.is_none() { + return Err(CausalDispatchError::Internal( + "cell wait-path returned legacy projection obligations without their canonical key identity" + .into(), + )); + } + let binding = self.attempt.external_binding().cloned().ok_or_else(|| { + CausalDispatchError::Internal( + "external command attempt lost its immutable route binding".into(), + ) + })?; + if let Some(metadata) = result.receipt.projection_metadata.as_ref() { + let bytes = metadata.canonical_bytes().map_err(|error| { + CausalDispatchError::Internal(format!( + "cell projection metadata could not be canonicalized: {error}" + )) + })?; + let expires_at = metadata.expires_at().map_err(|error| { + CausalDispatchError::Internal(format!( + "cell projection metadata retention deadline is invalid: {error}" + )) + })?; + self.attempt + .complete_external_with_projection_metadata_until( + binding, + state, + result.payload().clone(), + bytes, + self.retention, + expires_at, + ) + .map_err(internal_ledger_error) + } else { + self.attempt + .complete_external( + binding, + state, + result.payload().clone(), + Vec::new(), + self.retention, + ) + .map_err(internal_ledger_error) + } + } +} + #[cfg(feature = "graphql")] impl CausalDispatchResult { /// Handler payload returned to the wait-path caller. @@ -576,6 +691,40 @@ pub(super) fn internal_ledger_error(error: CommandLedgerError) -> CausalDispatch CausalDispatchError::Internal(error.to_string()) } +#[cfg(feature = "graphql")] +pub(super) async fn abandon_external_attempt( + repository: &R, + attempt: ExternalCausalAttempt, + detail: String, +) -> Result<(), CausalDispatchError> +where + R: CommandLedgerStore + Send + Sync, +{ + let fence = attempt.fence(); + match repository.mark_retryable_unknown(fence.clone()).await { + Ok(()) => Err(CausalDispatchError::Internal(detail)), + Err(CommandLedgerError::AttemptFenced { .. }) => match repository + .lookup_command(fence.key(), CommandLookupScope::Attempt(&fence)) + .await + { + Ok(CommandLookup::Replay(_)) => Ok(()), + Ok(CommandLookup::Expired) => Err(CausalDispatchError::Expired), + Ok(CommandLookup::RetryableUnknown { .. }) => { + Err(CausalDispatchError::Internal(detail)) + } + Ok(CommandLookup::InProgress { .. }) | Ok(CommandLookup::Unknown) => { + Err(CausalDispatchError::Internal(detail)) + } + Err(error) => Err(CausalDispatchError::Internal(format!( + "{detail}; external command recovery failed: {error}" + ))), + }, + Err(error) => Err(CausalDispatchError::Internal(format!( + "{detail}; failed to mark external command retryable: {error}" + ))), + } +} + #[cfg(feature = "graphql")] pub(super) fn replay_result( consistency: CommandConsistency, diff --git a/src/microsvc/service/mod.rs b/src/microsvc/service/mod.rs index c5cb368c..a9592c7d 100644 --- a/src/microsvc/service/mod.rs +++ b/src/microsvc/service/mod.rs @@ -43,7 +43,7 @@ pub use causal::GraphqlServiceBindError; #[cfg(feature = "graphql")] pub(crate) use causal::{ CausalCommandProjectionObligation, CausalCommandPublicState, CausalCommandReceiptSource, - CausalProjectionEvidenceState, + CausalProjectionEvidenceState, ExternalCausalReservation, }; #[cfg(feature = "graphql")] pub use causal::{CausalCommandPublicStatus, CausalDispatchError, CausalDispatchResult}; diff --git a/src/microsvc/service/routes.rs b/src/microsvc/service/routes.rs index 3dc6022c..2f61bbcc 100644 --- a/src/microsvc/service/routes.rs +++ b/src/microsvc/service/routes.rs @@ -5,15 +5,13 @@ use std::future::Future; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; -#[cfg(feature = "graphql")] -use std::time::SystemTime; - #[cfg(feature = "graphql")] use super::causal::{ abandon_causal_attempt, causal_handler_error_code, commit_causal_rejection, ensure_causal_grant, evaluate_causal_command_status, internal_ledger_error, load_committed_dispatch_result, recover_causal_commit_error, replay_result, CausalCommandPublicStatus, CausalDispatchError, CausalDispatchResult, + ExternalCausalAttempt, ExternalCausalReservation, }; use super::handlers::{ boxed_causal_guard, boxed_handler, boxed_prepared_handler, CausalCommandContext, CausalGuardFn, @@ -35,7 +33,7 @@ use crate::command_ledger::{ #[cfg(feature = "graphql")] use crate::command_ledger::{ CausalRepositoryIdentity, CommandId, CommandLedgerKey, CommandLookup, CommandLookupScope, - PrincipalPartitionId, + ExternalDispatchBinding, PrincipalPartitionId, }; #[cfg(feature = "graphql")] use crate::graphql::identity::VerifiedPrincipal; @@ -189,6 +187,16 @@ pub(super) type CausalHandlerFuture<'a> = pub(super) type CausalStatusFuture<'a> = Pin< Box> + Send + 'a>, >; +#[cfg(feature = "graphql")] +pub(super) type ExternalReservationFuture<'a> = Pin< + Box> + Send + 'a>, +>; +#[cfg(feature = "graphql")] +pub(super) type ExternalCompletionFuture<'a> = + Pin> + Send + 'a>>; +#[cfg(feature = "graphql")] +pub(super) type ExternalAbandonFuture<'a> = + Pin> + Send + 'a>>; pub(super) type CellCausalHandlerFuture<'a> = Pin> + Send + 'a>>; @@ -246,6 +254,34 @@ pub(super) trait ErasedCausalHandler: Send + Sync { protocol: Option, ) -> CausalStatusFuture<'a>; + #[cfg(feature = "graphql")] + fn reserve_external<'a>( + &'a self, + dependencies: &'a D, + service_id: &'a str, + command_id: &'a str, + input: Value, + session: Session, + principal: VerifiedPrincipal, + policy: CausalCommandPolicy, + binding: ExternalDispatchBinding, + ) -> ExternalReservationFuture<'a>; + + #[cfg(feature = "graphql")] + fn complete_external<'a>( + &'a self, + dependencies: &'a D, + attempt: ExternalCausalAttempt, + result: &'a CausalDispatchResult, + ) -> ExternalCompletionFuture<'a>; + + #[cfg(feature = "graphql")] + fn abandon_external<'a>( + &'a self, + dependencies: &'a D, + attempt: ExternalCausalAttempt, + ) -> ExternalAbandonFuture<'a>; + /// Run the same typed `handle` inside one cell, without GraphQL receipts. fn invoke_cell<'a>( &'a self, @@ -368,6 +404,34 @@ pub(super) trait ErasedRoutes: Send + Sync { protocol: Option, ) -> CausalStatusFuture<'a>; + #[cfg(feature = "graphql")] + fn reserve_external<'a>( + &'a self, + command: &'a str, + service_id: &'a str, + command_id: &'a str, + input: Value, + session: Session, + principal: VerifiedPrincipal, + policy: CausalCommandPolicy, + binding: ExternalDispatchBinding, + ) -> ExternalReservationFuture<'a>; + + #[cfg(feature = "graphql")] + fn complete_external<'a>( + &'a self, + command: &'a str, + attempt: ExternalCausalAttempt, + result: &'a CausalDispatchResult, + ) -> ExternalCompletionFuture<'a>; + + #[cfg(feature = "graphql")] + fn abandon_external<'a>( + &'a self, + command: &'a str, + attempt: ExternalCausalAttempt, + ) -> ExternalAbandonFuture<'a>; + #[cfg(feature = "graphql")] fn projected_storage_identities(&self) -> Vec; @@ -1575,6 +1639,10 @@ where policy.replay_retention, ) .map_err(crate::microsvc::cell_host::causal::internal_ledger_error)?; + let reservation = match identity.causation_id() { + Some(causation_id) => reservation.with_causation_id(causation_id.clone()), + None => reservation, + }; let aggregate_repository = dependencies.__causal_aggregate_repository(); let repository = aggregate_repository.repo(); @@ -2263,6 +2331,132 @@ where }) } + #[cfg(feature = "graphql")] + fn reserve_external<'a>( + &'a self, + dependencies: &'a D, + service_id: &'a str, + command_id: &'a str, + input: Value, + session: Session, + principal: VerifiedPrincipal, + policy: CausalCommandPolicy, + binding: ExternalDispatchBinding, + ) -> ExternalReservationFuture<'a> { + Box::pin(async move { + ensure_causal_grant(&self.contract, &session)?; + if self.contract.consistency == CommandConsistency::Atomic { + return Err(CausalDispatchError::BadRequest( + "atomic typed commands cannot be dispatched to an aggregate cell".into(), + )); + } + let canonical = canonicalize_command_input(&self.contract.input, input) + .map_err(|error| CausalDispatchError::BadRequest(error.to_string()))?; + let typed = canonical + .decode::() + .map_err(|error| CausalDispatchError::BadRequest(error.to_string()))?; + let (_, _, input_digest) = typed.into_parts(); + let command_id = CommandId::parse(command_id) + .map_err(|error| CausalDispatchError::BadRequest(error.to_string()))?; + let partition = PrincipalPartitionId::new(principal.partition_for_service(service_id)) + .map_err(internal_ledger_error)?; + let key = CommandLedgerKey::new(service_id, partition, command_id) + .map_err(internal_ledger_error)?; + let reservation = CommandReservation::new( + key, + self.contract.name.clone(), + CommandContractFingerprint::new(self.contract.fingerprint_bytes()), + CanonicalInputHash::new(input_digest), + policy.attempt_lease, + policy.replay_retention, + ) + .map_err(internal_ledger_error)? + .with_external_binding(binding); + let repository = dependencies.__causal_aggregate_repository().repo(); + match repository + .reserve_command(reservation) + .await + .map_err(internal_ledger_error)? + { + ReservationOutcome::Acquired(attempt) => { + Ok(ExternalCausalReservation::Acquired(ExternalCausalAttempt { + command_name: self.contract.name.clone(), + attempt, + retention: policy.replay_retention, + })) + } + ReservationOutcome::Replay(replay) => Ok(ExternalCausalReservation::Replay( + replay_result(self.contract.consistency, replay)?, + )), + ReservationOutcome::InProgress { .. } => Err(CausalDispatchError::InProgress), + ReservationOutcome::Conflict => Err(CausalDispatchError::CommandIdReuse), + ReservationOutcome::Expired => Err(CausalDispatchError::Expired), + } + }) + } + + #[cfg(feature = "graphql")] + fn complete_external<'a>( + &'a self, + dependencies: &'a D, + attempt: ExternalCausalAttempt, + result: &'a CausalDispatchResult, + ) -> ExternalCompletionFuture<'a> { + Box::pin(async move { + if attempt.command_name != self.contract.name { + return Err(CausalDispatchError::Internal( + "external command completion route does not match its reservation".into(), + )); + } + let fence = attempt.fence(); + let completion = attempt.into_external_completion(result)?; + let repository = dependencies.__causal_aggregate_repository().repo(); + match repository.complete_external_command(completion).await { + Ok(()) => Ok(()), + Err(error) => match repository + .lookup_command(&fence.key(), CommandLookupScope::Attempt(&fence)) + .await + { + Ok(CommandLookup::Replay(_)) => Ok(()), + Ok(CommandLookup::InProgress { .. }) => { + match repository.mark_retryable_unknown(fence).await { + Ok(()) => Err(CausalDispatchError::Internal(error.to_string())), + Err(recovery) => Err(CausalDispatchError::Internal(format!( + "{error}; external completion recovery failed: {recovery}" + ))), + } + } + Ok(CommandLookup::RetryableUnknown { .. }) => { + Err(CausalDispatchError::Internal(error.to_string())) + } + Ok(CommandLookup::Expired) => Err(CausalDispatchError::Expired), + Ok(CommandLookup::Unknown) => Err(CausalDispatchError::Internal(format!( + "{error}; external command ledger row disappeared" + ))), + Err(recovery) => Err(CausalDispatchError::Internal(format!( + "{error}; external completion lookup failed: {recovery}" + ))), + }, + } + }) + } + + #[cfg(feature = "graphql")] + fn abandon_external<'a>( + &'a self, + dependencies: &'a D, + attempt: ExternalCausalAttempt, + ) -> ExternalAbandonFuture<'a> { + Box::pin(async move { + crate::microsvc::service::causal::abandon_external_attempt( + dependencies.__causal_aggregate_repository().repo(), + attempt, + "external cell dispatch did not produce a durable terminal receipt".into(), + ) + .await + }) + } + fn invoke_cell<'a>( &'a self, dependencies: &'a D, @@ -2585,6 +2779,92 @@ where }) } + #[cfg(feature = "graphql")] + fn reserve_external<'a>( + &'a self, + command: &'a str, + service_id: &'a str, + command_id: &'a str, + input: Value, + session: Session, + principal: VerifiedPrincipal, + policy: CausalCommandPolicy, + binding: ExternalDispatchBinding, + ) -> ExternalReservationFuture<'a> { + let handler = self + .handlers + .get(&MessageKind::Command) + .and_then(|handlers| handlers.get(command)); + match handler { + Some(RegisteredHandler::Causal(handler)) => handler.reserve_external( + &self.dependencies, + service_id, + command_id, + input, + session, + principal, + policy, + binding, + ), + Some(RegisteredHandler::Legacy { .. }) + | Some(RegisteredHandler::Projector(_)) + | None => Box::pin(async move { + Err(CausalDispatchError::BadRequest(format!( + "`{command}` is not a typed causal command" + ))) + }), + } + } + + #[cfg(feature = "graphql")] + fn complete_external<'a>( + &'a self, + command: &'a str, + attempt: ExternalCausalAttempt, + result: &'a CausalDispatchResult, + ) -> ExternalCompletionFuture<'a> { + let handler = self + .handlers + .get(&MessageKind::Command) + .and_then(|handlers| handlers.get(command)); + match handler { + Some(RegisteredHandler::Causal(handler)) => { + handler.complete_external(&self.dependencies, attempt, result) + } + Some(RegisteredHandler::Legacy { .. }) + | Some(RegisteredHandler::Projector(_)) + | None => Box::pin(async move { + Err(CausalDispatchError::BadRequest(format!( + "`{command}` is not a typed causal command" + ))) + }), + } + } + + #[cfg(feature = "graphql")] + fn abandon_external<'a>( + &'a self, + command: &'a str, + attempt: ExternalCausalAttempt, + ) -> ExternalAbandonFuture<'a> { + let handler = self + .handlers + .get(&MessageKind::Command) + .and_then(|handlers| handlers.get(command)); + match handler { + Some(RegisteredHandler::Causal(handler)) => { + handler.abandon_external(&self.dependencies, attempt) + } + Some(RegisteredHandler::Legacy { .. }) + | Some(RegisteredHandler::Projector(_)) + | None => Box::pin(async move { + Err(CausalDispatchError::BadRequest(format!( + "`{command}` is not a typed causal command" + ))) + }), + } + } + #[cfg(feature = "graphql")] fn projected_storage_identities(&self) -> Vec { self.handlers diff --git a/src/microsvc/service/runtime.rs b/src/microsvc/service/runtime.rs index bc680536..688b5a46 100644 --- a/src/microsvc/service/runtime.rs +++ b/src/microsvc/service/runtime.rs @@ -9,7 +9,7 @@ use serde_json::Value; #[cfg(feature = "graphql")] use super::causal::{ internal_ledger_error, CausalCommandPublicStatus, CausalDispatchError, CausalDispatchResult, - GraphqlServiceBindError, + ExternalCausalAttempt, ExternalCausalReservation, GraphqlServiceBindError, }; use super::helpers::{ is_json_content_type, message_to_json_input, message_to_session, names_by_kind, @@ -29,6 +29,8 @@ use crate::command::{TypedCommandContract, TypedServiceCommandBinding}; #[cfg(feature = "graphql")] use crate::command_ledger::{CommandId, CommandLookup, PrincipalPartitionId}; #[cfg(feature = "graphql")] +use crate::command_ledger::ExternalDispatchBinding; +#[cfg(feature = "graphql")] use crate::graphql::identity::VerifiedPrincipal; use crate::microsvc::error::HandlerError; use crate::microsvc::projector::{ProjectionRepairHandle, ProjectorRegistration}; @@ -668,6 +670,85 @@ impl Service { ) } + /// Reserve the gateway ledger row for a command that will be committed by + /// an aggregate cell. The route performs canonical input and grant checks + /// before any remote request is made. + #[cfg(feature = "graphql")] + pub(crate) async fn reserve_external_causal( + &self, + command: &str, + command_id: &str, + input: Value, + session: Session, + principal: VerifiedPrincipal, + binding: ExternalDispatchBinding, + ) -> Result { + ensure_lifecycle_mutations_open().map_err(CausalDispatchError::Handler)?; + let service_id = self.name().ok_or_else(|| { + CausalDispatchError::Internal( + "external typed causal dispatch requires Service::named identity".into(), + ) + })?; + let route_index = self + .index + .get(&MessageKind::Command) + .and_then(|commands| commands.get(command)) + .and_then(|indices| (indices.len() == 1).then_some(indices[0])) + .ok_or_else(|| CausalDispatchError::BadRequest("unknown typed command".into()))?; + self.routes[route_index] + .reserve_external( + command, + service_id, + command_id, + input, + session, + principal, + self.causal_command_policy, + binding, + ) + .await + } + + /// Persist a sealed terminal receipt for a command committed by a cell. + /// This path only updates the gateway ledger and never invokes a local + /// handler or appends a local event batch. + #[cfg(feature = "graphql")] + pub(crate) async fn complete_external_causal( + &self, + command: &str, + attempt: ExternalCausalAttempt, + result: &CausalDispatchResult, + ) -> Result<(), CausalDispatchError> { + let route_index = self + .index + .get(&MessageKind::Command) + .and_then(|commands| commands.get(command)) + .and_then(|indices| (indices.len() == 1).then_some(indices[0])) + .ok_or_else(|| CausalDispatchError::BadRequest("unknown typed command".into()))?; + self.routes[route_index] + .complete_external(command, attempt, result) + .await + } + + /// Mark a remote attempt retryable after a transport/protocol failure. + /// The durable fence remains the authority for later replay or reclaim. + #[cfg(feature = "graphql")] + pub(crate) async fn abandon_external_causal( + &self, + command: &str, + attempt: ExternalCausalAttempt, + ) -> Result<(), CausalDispatchError> { + let route_index = self + .index + .get(&MessageKind::Command) + .and_then(|commands| commands.get(command)) + .and_then(|indices| (indices.len() == 1).then_some(indices[0])) + .ok_or_else(|| CausalDispatchError::BadRequest("unknown typed command".into()))?; + self.routes[route_index] + .abandon_external(command, attempt) + .await + } + pub(crate) fn typed_command_binding(&self) -> Result { let service_id = self .name() diff --git a/src/microsvc/service/tests.rs b/src/microsvc/service/tests.rs index e9450f88..0e99d74e 100644 --- a/src/microsvc/service/tests.rs +++ b/src/microsvc/service/tests.rs @@ -20,11 +20,15 @@ use crate::command_ledger::{ ExternalCommandCompletion, ReservationOutcome, }; #[cfg(feature = "graphql")] +use crate::command_dispatch::SharedCommandHost; +#[cfg(feature = "graphql")] use crate::graphql::identity::VerifiedPrincipal; #[cfg(feature = "graphql")] use crate::graphql::{SurfaceDirectProjection, SurfaceProjector}; #[cfg(feature = "graphql")] use crate::microsvc::HasOutboxStore; +#[cfg(feature = "graphql")] +use crate::microsvc::cell_host::{CelldCommandHost, InternalHttpSecret}; use crate::microsvc::{ CommandRequest, Context, HandlerError, RepoReadModelDependencies, Routes, Service, Session, }; @@ -774,6 +778,19 @@ fn command_host(service: &Arc) -> crate::command_dispatch::SharedComman ))) } +#[cfg(feature = "graphql")] +fn celld_status_host(service: &Arc) -> SharedCommandHost { + Arc::new( + CelldCommandHost::new( + "http://127.0.0.1:1", + Arc::clone(service), + InternalHttpSecret::new("test-only-internal-secret-32-bytes") + .expect("test internal secret should be valid"), + ) + .expect("status-only celld host should accept a local URL"), + ) +} + #[cfg(feature = "graphql")] #[derive(Clone, Copy)] enum InjectedCommitBehavior { @@ -2829,6 +2846,37 @@ async fn graphql_succeeded_status_evaluates_retained_projection_evidence() { ); assert!(before_envelope["command"].get("observations").is_none()); + // The celld host must use the same durable status evaluator. In + // particular, a terminal cell receipt cannot make the gateway claim a + // projection observation before the modeled projector has supplied proof. + let before_celld = service + .graphql_engine() + .unwrap() + .execute( + &session, + async_graphql::Request::new(&status_query) + .data(celld_status_host(&service)) + .data(principal.clone()), + ) + .await; + assert!(before_celld.errors.is_empty(), "{before_celld:?}"); + let before_celld_envelope = serde_json::to_value( + before_celld + .extensions + .get("distributed") + .expect("celld status should carry its protocol envelope"), + ) + .unwrap(); + assert_eq!(before_celld_envelope["command"]["state"], "succeeded"); + assert_eq!( + before_celld_envelope["command"]["expects"] + .as_array() + .unwrap() + .len(), + 1 + ); + assert!(before_celld_envelope["command"].get("observations").is_none()); + let pending = repository .outbox_store() .pending(10) @@ -2882,6 +2930,33 @@ async fn graphql_succeeded_status_evaluates_retained_projection_evidence() { causation_id ); + let after_celld = service + .graphql_engine() + .unwrap() + .execute( + &session, + async_graphql::Request::new(&status_query) + .data(celld_status_host(&service)) + .data(principal.clone()), + ) + .await; + assert!(after_celld.errors.is_empty(), "{after_celld:?}"); + let after_celld_envelope = serde_json::to_value( + after_celld + .extensions + .get("distributed") + .expect("celld status should carry its protocol envelope"), + ) + .unwrap(); + assert_eq!(after_celld_envelope["command"]["state"], "succeeded"); + assert_eq!( + after_celld_envelope["command"]["observations"] + .as_array() + .expect("celld status should expose matching durable proof") + .len(), + 1 + ); + // A proof authored by a different semantic program is not an observation // for this command, even when its physical topology and scope are equal. let wrong_program = diff --git a/tests/causal_wait_path/main.rs b/tests/causal_wait_path/main.rs index 80875069..ab96b78f 100644 --- a/tests/causal_wait_path/main.rs +++ b/tests/causal_wait_path/main.rs @@ -4,7 +4,9 @@ use std::sync::Arc; use distributed::bus::{Bus, BusConsumer, InMemoryBus, TransportError}; -use distributed::cell_host::InternalHttpSecret; +use distributed::cell_host::{ + CelldCommandHost, CelldRoute, InternalHttpSecret, CELL_CAUSATION_ID_HEADER, +}; use distributed::command::{ typed_command, CommandInputType, CommandOutputType, CommandTypeDef, CommandTypeField, Succeeded, }; @@ -12,8 +14,11 @@ use distributed::command_dispatch::{CommandHost, HttpCommandHost, SharedCommandH use distributed::graphql::VerifiedPrincipal; use distributed::microsvc::{router, Routes, Service, ROLE_KEY, USER_ID_KEY}; use distributed::{Aggregate, AggregateBuilder, Entity, InMemoryRepository, Snapshot}; +#[cfg(feature = "sqlite")] +use distributed::{AggregateRepository, SqliteRepository}; use serde::{Deserialize, Serialize}; use serde_json::json; +use serde_json::Value; #[derive(Default, Snapshot)] struct WaitAgg { @@ -91,9 +96,9 @@ impl CommandOutputType for IdPayload { } } -fn wait_service() -> Arc { +fn wait_service_with_repo(repo: distributed::InMemoryRepository) -> Arc { let causal = Routes::new() - .with_repo(InMemoryRepository::new().aggregate::()) + .with_repo(repo.aggregate::()) .typed_command( typed_command::>("todo.create").roles(["user"]), ) @@ -128,6 +133,33 @@ fn wait_service() -> Arc { ) } +fn wait_service() -> Arc { + wait_service_with_repo(InMemoryRepository::new()) +} + +#[cfg(feature = "sqlite")] +fn sqlite_wait_service(repository: SqliteRepository) -> Arc { + let causal = Routes::new() + .with_repo(AggregateRepository::<_, WaitAgg>::new(repository)) + .typed_command( + typed_command::>("todo.create").roles(["user"]), + ) + .create() + .invoke(|aggregate, input, _owner| { + aggregate.record(input.id.clone())?; + Ok::<_, distributed::EventRecordError>(()) + }) + .succeeded(|aggregate| IdPayload { + id: aggregate.entity().id().to_string(), + }); + Arc::new( + Service::new() + .named("causal-wait-path") + .with_http_command_routes() + .routes(causal), + ) +} + async fn start_http(service: Arc) -> String { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -203,6 +235,379 @@ async fn cell_wait_path_replays_once_after_internal_failure() { assert_eq!(attempts.load(Ordering::SeqCst), 2); } +#[tokio::test] +async fn celld_host_completes_remote_commit_once_and_rejects_changed_shard() { + use axum::{extract::State, http::HeaderMap, http::StatusCode, routing::post, Json, Router}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + async fn command( + State(calls): State>, + headers: HeaderMap, + Json(body): Json, + ) -> (StatusCode, Json) { + calls.fetch_add(1, Ordering::SeqCst); + let causation_id = headers + .get(CELL_CAUSATION_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .expect("gateway must bind the cell request to its reserved causation") + .to_string(); + ( + StatusCode::CREATED, + Json(json!({ + "payload": { "id": body["input"]["id"] }, + "receipt": { + "commandId": body["commandId"], + "causationId": causation_id, + "state": "succeeded", + "replayed": false + }, + "events": [] + })), + ) + } + + let calls = Arc::new(AtomicUsize::new(0)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let app = Router::new() + .fallback(post(command)) + .with_state(Arc::clone(&calls)); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let repo = InMemoryRepository::new(); + let service = wait_service_with_repo(repo.clone()); + let host = CelldCommandHost::new( + format!("http://{addr}"), + Arc::clone(&service), + InternalHttpSecret::new("test-only-internal-secret-32-bytes").unwrap(), + ) + .unwrap() + .route(CelldRoute::new( + &["todo.create"], + "todo", + |input| input.get("id").and_then(Value::as_str).map(str::to_owned), + |_command, _input, remote, _session| remote.clone(), + )); + let mut session = distributed::microsvc::Session::new(); + session.set(USER_ID_KEY, "alice"); + session.set(ROLE_KEY, "user"); + let principal = VerifiedPrincipal::from_trusted_transport("alice"); + let command_id = "0190a000-0000-7000-8000-000000000109"; + + let first = host + .invoke( + "todo.create", + command_id, + json!({ "id": "todo-cell-once" }), + session.clone(), + principal.clone(), + None, + ) + .await + .expect("remote cell commit should complete the gateway ledger"); + assert_eq!(first.payload(), &json!({ "id": "todo-cell-once" })); + assert_eq!(first.state(), "succeeded"); + + drop(host); + drop(service); + let restarted_service = wait_service_with_repo(repo); + let restarted_host = CelldCommandHost::new( + format!("http://{addr}"), + Arc::clone(&restarted_service), + InternalHttpSecret::new("test-only-internal-secret-32-bytes").unwrap(), + ) + .unwrap() + .route(CelldRoute::new( + &["todo.create"], + "todo", + |input| input.get("id").and_then(Value::as_str).map(str::to_owned), + |_command, _input, remote, _session| remote.clone(), + )); + let replay = restarted_host + .invoke( + "todo.create", + command_id, + json!({ "id": "todo-cell-once" }), + session.clone(), + principal.clone(), + None, + ) + .await + .expect("completed external receipt should replay durably"); + assert_eq!(replay.state(), "succeeded"); + assert_eq!(replay.payload(), first.payload()); + assert_eq!(calls.load(Ordering::SeqCst), 1); + + let conflict = restarted_host + .invoke( + "todo.create", + command_id, + json!({ "id": "todo-other-cell" }), + session, + principal, + None, + ) + .await + .expect_err("same command ID cannot move to another cell or input"); + assert!(matches!( + conflict, + distributed::microsvc::CausalDispatchError::CommandIdReuse + )); + assert_eq!(calls.load(Ordering::SeqCst), 1); + + let different_binding_host = CelldCommandHost::new( + format!("http://{addr}"), + Arc::clone(&restarted_service), + InternalHttpSecret::new("test-only-internal-secret-32-bytes").unwrap(), + ) + .unwrap() + .route(CelldRoute::new( + &["todo.create"], + "todo", + |_input| Some("todo-different-cell".into()), + |_command, _input, remote, _session| remote.clone(), + )); + let mut binding_session = distributed::microsvc::Session::new(); + binding_session.set(USER_ID_KEY, "alice"); + binding_session.set(ROLE_KEY, "user"); + let binding_conflict = different_binding_host + .invoke( + "todo.create", + command_id, + json!({ "id": "todo-cell-once" }), + binding_session, + VerifiedPrincipal::from_trusted_transport("alice"), + None, + ) + .await + .expect_err("the immutable route binding must fence a changed shard"); + assert!(matches!( + binding_conflict, + distributed::microsvc::CausalDispatchError::CommandIdReuse + )); + assert_eq!(calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn celld_host_reclaims_an_ambiguous_receipt_with_the_same_causation() { + use axum::{extract::State, http::HeaderMap, http::StatusCode, routing::post, Json, Router}; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Mutex; + + async fn command( + State(state): State>) >>, + headers: HeaderMap, + Json(body): Json, + ) -> (StatusCode, Json) { + let causation_id = headers + .get(CELL_CAUSATION_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .expect("gateway must bind the cell request to its reserved causation") + .to_string(); + let call = state.0.fetch_add(1, Ordering::SeqCst); + if call == 0 { + *state.1.lock().unwrap() = Some(causation_id); + return (StatusCode::OK, Json(json!({ "not": "a receipt" }))); + } + assert_eq!( + state.1.lock().unwrap().as_deref(), + Some(causation_id.as_str()), + "reclaim must reuse the cell's original causation identity" + ); + ( + StatusCode::CREATED, + Json(json!({ + "payload": { "id": body["input"]["id"] }, + "receipt": { + "commandId": body["commandId"], + "causationId": causation_id, + "state": "succeeded", + "replayed": true + }, + "events": [] + })), + ) + } + + let state = Arc::new((AtomicUsize::new(0), Mutex::new(None))); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let app = Router::new() + .fallback(post(command)) + .with_state(Arc::clone(&state)); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let service = wait_service(); + let host = CelldCommandHost::new( + format!("http://{addr}"), + service, + InternalHttpSecret::new("test-only-internal-secret-32-bytes").unwrap(), + ) + .unwrap() + .route(CelldRoute::new( + &["todo.create"], + "todo", + |input| input.get("id").and_then(Value::as_str).map(str::to_owned), + |_command, _input, remote, _session| remote.clone(), + )); + let mut session = distributed::microsvc::Session::new(); + session.set(USER_ID_KEY, "alice"); + session.set(ROLE_KEY, "user"); + let principal = VerifiedPrincipal::from_trusted_transport("alice"); + let command_id = "0190a000-0000-7000-8000-000000000110"; + + let first = host + .invoke( + "todo.create", + command_id, + json!({ "id": "todo-ambiguous" }), + session.clone(), + principal.clone(), + None, + ) + .await + .expect_err("an undecodable remote response must remain retryable"); + assert!(matches!( + first, + distributed::microsvc::CausalDispatchError::Internal(_) + )); + + let retry = host + .invoke( + "todo.create", + command_id, + json!({ "id": "todo-ambiguous" }), + session, + principal, + None, + ) + .await + .expect("retry should reclaim the durable reservation"); + assert_eq!(retry.state(), "succeeded"); + assert_eq!(state.0.load(Ordering::SeqCst), 2); +} + +#[cfg(feature = "sqlite")] +#[tokio::test] +async fn celld_host_replays_external_completion_after_sqlite_reopen() { + use axum::{extract::State, http::HeaderMap, http::StatusCode, routing::post, Json, Router}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + async fn command( + State(calls): State>, + headers: HeaderMap, + Json(body): Json, + ) -> (StatusCode, Json) { + calls.fetch_add(1, Ordering::SeqCst); + let causation_id = headers + .get(CELL_CAUSATION_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .expect("gateway must bind the cell request to its reserved causation"); + ( + StatusCode::CREATED, + Json(json!({ + "payload": { "id": body["input"]["id"] }, + "receipt": { + "commandId": body["commandId"], + "causationId": causation_id, + "state": "succeeded", + "replayed": false + }, + "events": [] + })), + ) + } + + let calls = Arc::new(AtomicUsize::new(0)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let app = Router::new() + .fallback(post(command)) + .with_state(Arc::clone(&calls)); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let database_path = std::env::temp_dir().join(format!( + "distributed-celld-host-{}.sqlite", + uuid::Uuid::now_v7() + )); + let database_url = format!("sqlite://{}?mode=rwc", database_path.display()); + let repository = SqliteRepository::connect_and_migrate(&database_url) + .await + .expect("initial gateway ledger migration"); + let service = sqlite_wait_service(repository.clone()); + let host = CelldCommandHost::new( + format!("http://{addr}"), + Arc::clone(&service), + InternalHttpSecret::new("test-only-internal-secret-32-bytes").unwrap(), + ) + .unwrap() + .route(CelldRoute::new( + &["todo.create"], + "todo", + |input| input.get("id").and_then(Value::as_str).map(str::to_owned), + |_command, _input, remote, _session| remote.clone(), + )); + let mut session = distributed::microsvc::Session::new(); + session.set(USER_ID_KEY, "alice"); + session.set(ROLE_KEY, "user"); + let principal = VerifiedPrincipal::from_trusted_transport("alice"); + let command_id = "0190a000-0000-7000-8000-000000000111"; + let input = json!({ "id": "todo-sqlite-reopen" }); + + host.invoke( + "todo.create", + command_id, + input.clone(), + session.clone(), + principal.clone(), + None, + ) + .await + .expect("cell completion should be durable before response"); + drop(host); + drop(service); + drop(repository); + + let reopened_repository = SqliteRepository::connect_and_migrate(&database_url) + .await + .expect("reopen gateway ledger"); + let reopened_service = sqlite_wait_service(reopened_repository); + let reopened_host = CelldCommandHost::new( + format!("http://{addr}"), + Arc::clone(&reopened_service), + InternalHttpSecret::new("test-only-internal-secret-32-bytes").unwrap(), + ) + .unwrap() + .route(CelldRoute::new( + &["todo.create"], + "todo", + |input| input.get("id").and_then(Value::as_str).map(str::to_owned), + |_command, _input, remote, _session| remote.clone(), + )); + let replay = reopened_host + .invoke( + "todo.create", + command_id, + input, + session, + principal, + None, + ) + .await + .expect("reopened gateway should replay the durable cell receipt"); + assert_eq!(replay.state(), "succeeded"); + assert_eq!(replay.payload(), &json!({ "id": "todo-sqlite-reopen" })); + assert_eq!(calls.load(Ordering::SeqCst), 1); + + let _ = std::fs::remove_file(database_path); +} + #[tokio::test] async fn http_wait_path_returns_command_id_and_receipt() { let base = start_http(wait_service()).await; diff --git a/tests/celld/worker/src/lib.rs b/tests/celld/worker/src/lib.rs index dd287ff3..7ff17413 100644 --- a/tests/celld/worker/src/lib.rs +++ b/tests/celld/worker/src/lib.rs @@ -9,7 +9,7 @@ use chat_domain::{post, ChatMessage, ChatMessageState}; use distributed::cell_host::{ AggregateCell, CellCommandIdentity, CellDispatchError, CellDispatchResult, CellWaitPathRequest, CelldOutbox, InternalHttpSecret, CELL_INTERNAL_SECRET_ENV, CELL_INTERNAL_SECRET_HEADER, - CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, + CELL_CAUSATION_ID_HEADER, CELL_PRINCIPAL_PARTITION_HEADER, CELL_SERVICE_ID_HEADER, }; use distributed::microsvc::{Session, ROLE_KEY, USER_ID_KEY}; use serde::de::DeserializeOwned; @@ -372,7 +372,11 @@ fn request_cell_identity( ) -> std::result::Result { let service_id = required_internal_header(req, CELL_SERVICE_ID_HEADER)?; let principal_partition = required_internal_header(req, CELL_PRINCIPAL_PARTITION_HEADER)?; - CellCommandIdentity::new(service_id, principal_partition, command_id) + let identity = CellCommandIdentity::new(service_id, principal_partition, command_id)?; + match optional_internal_header(req, CELL_CAUSATION_ID_HEADER)? { + Some(causation_id) => identity.with_causation_id(causation_id), + None => Ok(identity), + } } fn required_internal_header( @@ -389,6 +393,18 @@ fn required_internal_header( .ok_or(CellDispatchError::Unauthorized) } +fn optional_internal_header( + req: &Request, + name: &str, +) -> std::result::Result, CellDispatchError> { + req.headers() + .get(name) + .map_err(|error| { + CellDispatchError::Internal(format!("could not read internal cell header: {error}")) + }) + .map(|value| value.map(|value| value.trim().to_string()).filter(|value| !value.is_empty())) +} + fn wait_path_ok( payload: Value, dispatch: &CellDispatchResult, From 87c9d637445adaa68d2c7b40d93f5c01bf233cb8 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 10 Sep 2026 01:13:18 -0500 Subject: [PATCH 08/10] fix: settle proven eventual command delivery --- js/README.md | 8 + js/src/replica/command-runtime/create.ts | 32 ++++ js/tests/replica-command-runtime.test.mjs | 170 ++++++++++++++++++++++ 3 files changed, 210 insertions(+) diff --git a/js/README.md b/js/README.md index e0669ad5..8dbe828b 100644 --- a/js/README.md +++ b/js/README.md @@ -442,6 +442,14 @@ same replica and GraphQL transport. A command call: (`confirmDirectProjection`) before the call settles. The server waited in the command handler because it could; an event handler cannot. +An Eventual command may be terminal `succeeded` at a cell boundary while its +status envelope already carries every exact projection observation. Those +observations settle `receipt.projected` because they prove delivery of the +projection obligation; they do not retire the accepted optimistic layer. The +layer is retired only by a matching canonical query or live frame, so an +`@load` operation does not need to become `@live` merely to complete a causal +wait. A later query or navigation can still supply that canonical frame. + Applications do not provide list targets, merge functions, mutation update callbacks, board simulators, or invalidation maps. If the compiler cannot prove safe maintenance, the generated plan marks the affected projection stale and diff --git a/js/src/replica/command-runtime/create.ts b/js/src/replica/command-runtime/create.ts index 471ef75c..f555e0b8 100644 --- a/js/src/replica/command-runtime/create.ts +++ b/js/src/replica/command-runtime/create.ts @@ -1,5 +1,6 @@ import { parseGraphqlResponseExtensions, + type DistributedCommandMetadata, type DistributedProtocolEnvelope } from '../../protocol.js'; import { @@ -34,6 +35,7 @@ import { pendingProjection, preparedDispatchKeys, preparedSemanticChanges, + projectionExpectationFingerprint, requireCommandEnvelope, requireCommandRejectionEnvelope, requireStatusEnvelope, @@ -85,6 +87,22 @@ import { type ReplicaCommandProjection } from '../projection-delta/index.js'; +function hasCompleteProjectionObservations( + metadata: DistributedCommandMetadata +): boolean { + if (metadata.expects.length === 0 || metadata.observations.length === 0) { + return false; + } + const observed = new Set( + metadata.observations + .filter((observation) => observation.causationId === metadata.causationId) + .map(projectionExpectationFingerprint) + ); + return metadata.expects.every((expectation) => + observed.has(projectionExpectationFingerprint(expectation)) + ); +} + function assertActualProjectionCapabilities( contract: ReplicaCommandProjection, delta: ProjectionDelta @@ -1038,6 +1056,20 @@ export function createReplicaCommandRuntime< if (tracker.pending !== undefined) { settleTrackedProjection(tracker, pending); } + } else if ( + metadata.state === 'succeeded' && + !prepared.revalidation.required && + !statusRequiresRevalidation && + hasCompleteProjectionObservations(metadata) + ) { + /* + * A cell may keep a committed external receipt in the public + * `succeeded` state after its exact modeled observation is durable. + * That proof settles the projected delivery wait, but it does not + * retire the accepted layer: no canonical query/live frame has + * confirmed read-model membership yet. + */ + settleTrackedProjection(tracker, pending); } else if ( metadata.state === 'atomic' && !prepared.revalidation.required && diff --git a/js/tests/replica-command-runtime.test.mjs b/js/tests/replica-command-runtime.test.mjs index a17865ff..09348fd2 100644 --- a/js/tests/replica-command-runtime.test.mjs +++ b/js/tests/replica-command-runtime.test.mjs @@ -1769,6 +1769,176 @@ test('terminal exact projection status settles without command-triggered revalid runtime.dispose(); }); +test('terminal succeeded status with exact observations settles delivery without retiring the layer', async () => { + const replica = new TestReplica(); + let pendingMetadata; + const runtime = createReplicaCommandRuntime( + replica, + { + dispatch(request) { + pendingMetadata = commandMetadata(request, { + actualTitle: 'accepted', + state: 'succeeded' + }); + return Promise.resolve( + envelope(request, { command: pendingMetadata }) + ); + }, + status(request) { + const terminalMetadata = Object.freeze({ + ...pendingMetadata, + state: 'succeeded', + observations: Object.freeze( + pendingMetadata.expects.map((expectation) => + Object.freeze({ + ...expectation, + causationId: pendingMetadata.causationId + }) + ) + ) + }); + return Promise.resolve( + statusEnvelope(request, terminalMetadata) + ); + } + }, + { change: artifact() }, + { status: STATUS } + ); + const receipt = await runtime.commands.change( + { id: 'todo-1', title: 'preview' }, + { commandId: COMMAND_A } + ); + + const projectedState = receipt.projected.then( + (outcome) => outcome.state, + () => 'rejected' + ); + try { + assert.equal((await receipt.status()).state, 'succeeded'); + const state = await Promise.race([ + projectedState, + new Promise((resolve) => + setTimeout(() => resolve('timed_out'), 100) + ) + ]); + assert.equal(state, 'atomic'); + assert.deepEqual(replica.revalidations, []); + assert.equal(replica.layer(COMMAND_A), 'accepted'); + assert.equal(replica.record('todo-1').fields.title, 'accepted'); + } finally { + runtime.dispose(); + } +}); + +test('terminal succeeded status ignores incomplete or mismatched observations', async () => { + const cases = [ + { + name: 'missing', + transform: () => [] + }, + { + name: 'causation', + transform: (observations) => + observations.map((observation) => ({ + ...observation, + causationId: 'cause:other' + })), + protocolFailure: true + }, + { + name: 'projection', + transform: (observations) => + observations.map((observation) => ({ + ...observation, + projection: 'program:other' + })), + protocolFailure: true + }, + { + name: 'scope', + transform: (observations) => + observations.map((observation) => ({ + ...observation, + scopeToken: 'scope:other' + })), + protocolFailure: true + } + ]; + + for (const { name, transform, protocolFailure = false } of cases) { + const replica = new TestReplica(); + let pendingMetadata; + const runtime = createReplicaCommandRuntime( + replica, + { + dispatch(request) { + pendingMetadata = commandMetadata(request, { + actualTitle: 'accepted', + state: 'succeeded' + }); + return Promise.resolve( + envelope(request, { command: pendingMetadata }) + ); + }, + status(request) { + const observations = pendingMetadata.expects.map( + (expectation) => ({ + ...expectation, + causationId: pendingMetadata.causationId + }) + ); + return Promise.resolve( + statusEnvelope(request, { + ...pendingMetadata, + state: 'succeeded', + observations: transform(observations) + }) + ); + } + }, + { change: artifact() }, + { status: STATUS } + ); + const receipt = await runtime.commands.change( + { id: 'todo-1', title: 'preview' }, + { commandId: COMMAND_A } + ); + const projectedState = receipt.projected.then( + (outcome) => outcome.state, + () => 'rejected' + ); + try { + if (protocolFailure) { + await assert.rejects( + receipt.status(), + { code: 'REPLICA_COMMAND_PROTOCOL_INVALID' }, + name + ); + await assert.rejects( + receipt.projected, + { code: 'REPLICA_COMMAND_PROTOCOL_INVALID' }, + name + ); + assert.equal(replica.layer(COMMAND_A), undefined, name); + continue; + } + assert.equal((await receipt.status()).state, 'succeeded', name); + const state = await Promise.race([ + projectedState, + new Promise((resolve) => + setTimeout(() => resolve('timed_out'), 50) + ) + ]); + assert.equal(state, 'timed_out', name); + assert.deepEqual(replica.revalidations, [], name); + assert.equal(replica.layer(COMMAND_A), 'accepted', name); + } finally { + runtime.dispose(); + } + } +}); + test('invalid live progression cannot poison a later valid status transition', async () => { const replica = new TestReplica(); let request; From 3cba1a97b470306b1c75360449adea2abd325da1 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 10 Sep 2026 02:06:20 -0500 Subject: [PATCH 09/10] test: add bounded auth refresh diagnostics --- js/src/sveltekit/replica.ts | 111 +++++++++++++++++- tests/e2e-ui/gateway/refresh.mjs | 4 + .../lib/components/shared/AuthRefresh.svelte | 58 ++++++++- tests/e2e-ui/ui/src/routes/+layout.svelte | 11 +- 4 files changed, 179 insertions(+), 5 deletions(-) diff --git a/js/src/sveltekit/replica.ts b/js/src/sveltekit/replica.ts index caf785cb..5542999d 100644 --- a/js/src/sveltekit/replica.ts +++ b/js/src/sveltekit/replica.ts @@ -80,6 +80,81 @@ export type SveltekitPageDataSessionSource = set(next: TData): void; }>; +function recordSessionDiagnostic(event: Readonly>): void { + const maxEvents = 128; + if (typeof globalThis === 'undefined') return; + const diagnostics = globalThis as typeof globalThis & { + __captureReplicaDiagnostics?: unknown; + __distributedSessionTrace?: unknown; + }; + if (diagnostics.__captureReplicaDiagnostics !== true) return; + if (!Array.isArray(diagnostics.__distributedSessionTrace)) { + diagnostics.__distributedSessionTrace = []; + } + const trace = diagnostics.__distributedSessionTrace as unknown[]; + trace.push(Object.freeze({ time: Date.now(), ...event })); + if (trace.length > maxEvents) trace.splice(0, trace.length - maxEvents); +} + +const sessionDiagnosticCredentialOrdinals = new Map(); +let nextSessionDiagnosticCredentialOrdinal = 1; +const maxSessionDiagnosticCredentials = 32; + +function sessionDiagnosticCredentialOrdinal( + value: Readonly<{ accessToken?: unknown }> | null | undefined +): number | undefined { + if (typeof globalThis === 'undefined') return undefined; + const diagnostics = globalThis as typeof globalThis & { + __captureReplicaDiagnostics?: unknown; + }; + if (diagnostics.__captureReplicaDiagnostics !== true) return undefined; + if (typeof value?.accessToken !== 'string' || value.accessToken.length === 0) { + return undefined; + } + let ordinal = sessionDiagnosticCredentialOrdinals.get(value.accessToken); + if (ordinal === undefined) { + if (sessionDiagnosticCredentialOrdinals.size >= maxSessionDiagnosticCredentials) { + const oldest = sessionDiagnosticCredentialOrdinals.keys().next().value; + if (typeof oldest === 'string') sessionDiagnosticCredentialOrdinals.delete(oldest); + } + ordinal = nextSessionDiagnosticCredentialOrdinal++; + sessionDiagnosticCredentialOrdinals.set(value.accessToken, ordinal); + } + return ordinal; +} + +function sessionDiagnosticSourceOrigin(): string { + if (typeof globalThis === 'undefined') return 'unattributed'; + const diagnostics = globalThis as typeof globalThis & { + __distributedSessionSourceOrigin?: unknown; + }; + return typeof diagnostics.__distributedSessionSourceOrigin === 'string' + ? diagnostics.__distributedSessionSourceOrigin + : 'unattributed'; +} + +function installSessionDiagnosticOrdinalBridge(): void { + if (typeof globalThis === 'undefined') return; + const diagnostics = globalThis as typeof globalThis & { + __captureReplicaDiagnostics?: unknown; + __distributedSessionCredentialOrdinal?: unknown; + }; + if (diagnostics.__captureReplicaDiagnostics !== true) return; + diagnostics.__distributedSessionCredentialOrdinal = (value: unknown): number | undefined => { + if (value === null || typeof value !== 'object') return undefined; + const pageData = value as { + accessToken?: unknown; + session?: { accessToken?: unknown } | null; + }; + return sessionDiagnosticCredentialOrdinal({ + accessToken: + typeof pageData.accessToken === 'string' + ? pageData.accessToken + : pageData.session?.accessToken + }); + }; +} + export type SveltekitReplicaHydration = Readonly<{ version: 1; state: import('../replica/index.js').ReplicaDehydratedState; @@ -611,6 +686,7 @@ export function sessionSourceFromPageData( export function createPageDataSessionSource( initial: TData ): SveltekitPageDataSessionSource { + installSessionDiagnosticOrdinalBridge(); let current = initial; const listeners = new Set<() => void>(); const source = Object.freeze({ @@ -626,6 +702,17 @@ export function createPageDataSessionSource( session: sessionSourceFromPageData(source), get: source.get, set(next: TData): void { + const pageData = next as SveltekitDistributedPageData; + const auth = authFromPageData(next); + recordSessionDiagnostic({ + kind: 'page-data-set', + origin: sessionDiagnosticSourceOrigin(), + credentialOrdinal: sessionDiagnosticCredentialOrdinal(auth), + hasHydration: pageData.distributed !== undefined, + hasAuthority: pageData.distributedAuthority !== undefined, + hasSession: next.session !== null && next.session !== undefined, + hasAccessToken: typeof pageData.accessToken === 'string' + }); current = next; for (const listener of [...listeners]) listener(); } @@ -1038,16 +1125,17 @@ function createAuthorizationFence( let queue = Promise.resolve(); let disposed = false; const read = (): Promise => { + const sourceOrigin = sessionDiagnosticSourceOrigin(); const candidate = Promise.resolve().then(() => { // Capture the credential and its independent server transfer together, // before another source notification can replace either value. const credential = source.getAuth(); const transfer = source.getHydration?.(); - return Promise.resolve(credential).then((auth) => ({ auth, transfer })); + return Promise.resolve(credential).then((auth) => ({ auth, transfer, sourceOrigin })); }); const transition = queue.then(async () => { try { - const { auth, transfer } = await candidate; + const { auth, transfer, sourceOrigin } = await candidate; const next = snapshotAuthCredential(auth); if ( current !== undefined && @@ -1056,7 +1144,24 @@ function createAuthorizationFence( const freshTransfer = transfer !== undefined && !seenHydrations.has(transfer.hydration) && !seenAuthorities.has(transfer.authority); - if (!freshTransfer || !refresh(transfer)) invalidate(); + const retained = freshTransfer && refresh(transfer); + recordSessionDiagnostic({ + kind: 'auth-credential-change', + sourceOrigin, + previousCredentialOrdinal: sessionDiagnosticCredentialOrdinal(current), + nextCredentialOrdinal: sessionDiagnosticCredentialOrdinal(next), + hasTransfer: transfer !== undefined, + freshTransfer, + retained, + reason: retained + ? 'fresh-transfer-retained' + : transfer === undefined + ? 'missing-transfer' + : !freshTransfer + ? 'replayed-transfer' + : 'refresh-rejected' + }); + if (!retained) invalidate(); } current = next; if (transfer !== undefined) { diff --git a/tests/e2e-ui/gateway/refresh.mjs b/tests/e2e-ui/gateway/refresh.mjs index c5895a35..6182477a 100644 --- a/tests/e2e-ui/gateway/refresh.mjs +++ b/tests/e2e-ui/gateway/refresh.mjs @@ -59,6 +59,10 @@ export async function verifySessionRefreshContinuity(page, origin) { const lost = await page.evaluate(()=>globalThis.__refreshContinuity.lost); if (lost) { console.error('Redacted refresh replica diagnostics:', JSON.stringify(await page.evaluate(()=>globalThis.__replicaDiagnosticSnapshot?.()))); + console.error('Redacted refresh lifecycle trace:', JSON.stringify(await page.evaluate(() => ({ + refresh: globalThis.__distributedRefreshTrace ?? [], + session: globalThis.__distributedSessionTrace ?? [] + })))); } assert.equal(lost,false,route+' rows were removed during token refresh: '+JSON.stringify(await page.evaluate(()=>globalThis.__refreshContinuity.events))); } finally { diff --git a/tests/e2e-ui/ui/src/lib/components/shared/AuthRefresh.svelte b/tests/e2e-ui/ui/src/lib/components/shared/AuthRefresh.svelte index 0f32da8b..e970f89d 100644 --- a/tests/e2e-ui/ui/src/lib/components/shared/AuthRefresh.svelte +++ b/tests/e2e-ui/ui/src/lib/components/shared/AuthRefresh.svelte @@ -13,6 +13,48 @@ let refreshTimer: number | undefined; let retryTimer: number | undefined; + function traceRefresh(event: Readonly>) { + const maxEvents = 64; + const diagnostics = globalThis as typeof globalThis & { + __captureReplicaDiagnostics?: unknown; + __distributedRefreshTrace?: unknown; + }; + if (diagnostics.__captureReplicaDiagnostics !== true) return; + if (!Array.isArray(diagnostics.__distributedRefreshTrace)) { + diagnostics.__distributedRefreshTrace = []; + } + const trace = diagnostics.__distributedRefreshTrace as unknown[]; + trace.push({ + time: performance.now(), + ...event + }); + if (trace.length > maxEvents) trace.splice(0, trace.length - maxEvents); + } + + function withSessionSource(origin: string, action: () => T): T { + const diagnostics = globalThis as typeof globalThis & { + __captureReplicaDiagnostics?: unknown; + __distributedSessionSourceOrigin?: unknown; + }; + if (diagnostics.__captureReplicaDiagnostics !== true) return action(); + const previous = diagnostics.__distributedSessionSourceOrigin; + diagnostics.__distributedSessionSourceOrigin = origin; + try { + return action(); + } finally { + diagnostics.__distributedSessionSourceOrigin = previous; + } + } + + function credentialOrdinal(value: unknown): number | undefined { + const diagnostics = globalThis as typeof globalThis & { + __distributedSessionCredentialOrdinal?: unknown; + }; + return typeof diagnostics.__distributedSessionCredentialOrdinal === 'function' + ? (diagnostics.__distributedSessionCredentialOrdinal as (value: unknown) => number | undefined)(value) + : undefined; + } + function clearTimers() { if (refreshTimer !== undefined) window.clearTimeout(refreshTimer); if (retryTimer !== undefined) window.clearTimeout(retryTimer); @@ -22,6 +64,7 @@ async function refreshSession() { try { + traceRefresh({ kind: 'refresh-start' }); const response = await fetch('/api/auth/refresh', { method: 'POST', credentials: 'same-origin', @@ -38,11 +81,24 @@ if (response.ok) { const result = await response.json(); - if (result.pageData) onRefresh(result.pageData); + traceRefresh({ + kind: 'refresh-response', + status: response.status, + hasPageData: Boolean(result.pageData), + hasDistributed: result.pageData?.distributed !== undefined, + hasAuthority: result.pageData?.distributedAuthority !== undefined, + credentialOrdinal: credentialOrdinal(result.pageData) + }); + if (result.pageData) { + withSessionSource('refresh-response', () => onRefresh(result.pageData)); + } + if (result.pageData) traceRefresh({ kind: 'refresh-seed-applied' }); } if (response.ok || response.status === 401) { + traceRefresh({ kind: 'invalidate-all-start', status: response.status }); await invalidateAll(); + traceRefresh({ kind: 'invalidate-all-complete', status: response.status }); return; } } catch (error) { diff --git a/tests/e2e-ui/ui/src/routes/+layout.svelte b/tests/e2e-ui/ui/src/routes/+layout.svelte index cd8bb97f..828ced00 100644 --- a/tests/e2e-ui/ui/src/routes/+layout.svelte +++ b/tests/e2e-ui/ui/src/routes/+layout.svelte @@ -68,7 +68,16 @@ }); function applyPageData(next: SveltekitDistributedPageData) { - pageData.set(next); + const diagnostics = globalThis as typeof globalThis & Record; + const previousSourceOrigin = diagnostics.__distributedSessionSourceOrigin; + if (diagnostics.__captureReplicaDiagnostics === true) { + diagnostics.__distributedSessionSourceOrigin = 'sveltekit-data'; + } + try { + pageData.set(next); + } finally { + diagnostics.__distributedSessionSourceOrigin = previousSourceOrigin; + } if ( next.distributed === undefined || next.distributedAuthority === undefined || From 308bebb7fd411c9ba640c2e0f7f622674bb5c607 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Thu, 10 Sep 2026 02:27:21 -0500 Subject: [PATCH 10/10] Revert "test: add bounded auth refresh diagnostics" This reverts commit 3cba1a97b470306b1c75360449adea2abd325da1. --- js/src/sveltekit/replica.ts | 111 +----------------- tests/e2e-ui/gateway/refresh.mjs | 4 - .../lib/components/shared/AuthRefresh.svelte | 58 +-------- tests/e2e-ui/ui/src/routes/+layout.svelte | 11 +- 4 files changed, 5 insertions(+), 179 deletions(-) diff --git a/js/src/sveltekit/replica.ts b/js/src/sveltekit/replica.ts index 5542999d..caf785cb 100644 --- a/js/src/sveltekit/replica.ts +++ b/js/src/sveltekit/replica.ts @@ -80,81 +80,6 @@ export type SveltekitPageDataSessionSource = set(next: TData): void; }>; -function recordSessionDiagnostic(event: Readonly>): void { - const maxEvents = 128; - if (typeof globalThis === 'undefined') return; - const diagnostics = globalThis as typeof globalThis & { - __captureReplicaDiagnostics?: unknown; - __distributedSessionTrace?: unknown; - }; - if (diagnostics.__captureReplicaDiagnostics !== true) return; - if (!Array.isArray(diagnostics.__distributedSessionTrace)) { - diagnostics.__distributedSessionTrace = []; - } - const trace = diagnostics.__distributedSessionTrace as unknown[]; - trace.push(Object.freeze({ time: Date.now(), ...event })); - if (trace.length > maxEvents) trace.splice(0, trace.length - maxEvents); -} - -const sessionDiagnosticCredentialOrdinals = new Map(); -let nextSessionDiagnosticCredentialOrdinal = 1; -const maxSessionDiagnosticCredentials = 32; - -function sessionDiagnosticCredentialOrdinal( - value: Readonly<{ accessToken?: unknown }> | null | undefined -): number | undefined { - if (typeof globalThis === 'undefined') return undefined; - const diagnostics = globalThis as typeof globalThis & { - __captureReplicaDiagnostics?: unknown; - }; - if (diagnostics.__captureReplicaDiagnostics !== true) return undefined; - if (typeof value?.accessToken !== 'string' || value.accessToken.length === 0) { - return undefined; - } - let ordinal = sessionDiagnosticCredentialOrdinals.get(value.accessToken); - if (ordinal === undefined) { - if (sessionDiagnosticCredentialOrdinals.size >= maxSessionDiagnosticCredentials) { - const oldest = sessionDiagnosticCredentialOrdinals.keys().next().value; - if (typeof oldest === 'string') sessionDiagnosticCredentialOrdinals.delete(oldest); - } - ordinal = nextSessionDiagnosticCredentialOrdinal++; - sessionDiagnosticCredentialOrdinals.set(value.accessToken, ordinal); - } - return ordinal; -} - -function sessionDiagnosticSourceOrigin(): string { - if (typeof globalThis === 'undefined') return 'unattributed'; - const diagnostics = globalThis as typeof globalThis & { - __distributedSessionSourceOrigin?: unknown; - }; - return typeof diagnostics.__distributedSessionSourceOrigin === 'string' - ? diagnostics.__distributedSessionSourceOrigin - : 'unattributed'; -} - -function installSessionDiagnosticOrdinalBridge(): void { - if (typeof globalThis === 'undefined') return; - const diagnostics = globalThis as typeof globalThis & { - __captureReplicaDiagnostics?: unknown; - __distributedSessionCredentialOrdinal?: unknown; - }; - if (diagnostics.__captureReplicaDiagnostics !== true) return; - diagnostics.__distributedSessionCredentialOrdinal = (value: unknown): number | undefined => { - if (value === null || typeof value !== 'object') return undefined; - const pageData = value as { - accessToken?: unknown; - session?: { accessToken?: unknown } | null; - }; - return sessionDiagnosticCredentialOrdinal({ - accessToken: - typeof pageData.accessToken === 'string' - ? pageData.accessToken - : pageData.session?.accessToken - }); - }; -} - export type SveltekitReplicaHydration = Readonly<{ version: 1; state: import('../replica/index.js').ReplicaDehydratedState; @@ -686,7 +611,6 @@ export function sessionSourceFromPageData( export function createPageDataSessionSource( initial: TData ): SveltekitPageDataSessionSource { - installSessionDiagnosticOrdinalBridge(); let current = initial; const listeners = new Set<() => void>(); const source = Object.freeze({ @@ -702,17 +626,6 @@ export function createPageDataSessionSource( session: sessionSourceFromPageData(source), get: source.get, set(next: TData): void { - const pageData = next as SveltekitDistributedPageData; - const auth = authFromPageData(next); - recordSessionDiagnostic({ - kind: 'page-data-set', - origin: sessionDiagnosticSourceOrigin(), - credentialOrdinal: sessionDiagnosticCredentialOrdinal(auth), - hasHydration: pageData.distributed !== undefined, - hasAuthority: pageData.distributedAuthority !== undefined, - hasSession: next.session !== null && next.session !== undefined, - hasAccessToken: typeof pageData.accessToken === 'string' - }); current = next; for (const listener of [...listeners]) listener(); } @@ -1125,17 +1038,16 @@ function createAuthorizationFence( let queue = Promise.resolve(); let disposed = false; const read = (): Promise => { - const sourceOrigin = sessionDiagnosticSourceOrigin(); const candidate = Promise.resolve().then(() => { // Capture the credential and its independent server transfer together, // before another source notification can replace either value. const credential = source.getAuth(); const transfer = source.getHydration?.(); - return Promise.resolve(credential).then((auth) => ({ auth, transfer, sourceOrigin })); + return Promise.resolve(credential).then((auth) => ({ auth, transfer })); }); const transition = queue.then(async () => { try { - const { auth, transfer, sourceOrigin } = await candidate; + const { auth, transfer } = await candidate; const next = snapshotAuthCredential(auth); if ( current !== undefined && @@ -1144,24 +1056,7 @@ function createAuthorizationFence( const freshTransfer = transfer !== undefined && !seenHydrations.has(transfer.hydration) && !seenAuthorities.has(transfer.authority); - const retained = freshTransfer && refresh(transfer); - recordSessionDiagnostic({ - kind: 'auth-credential-change', - sourceOrigin, - previousCredentialOrdinal: sessionDiagnosticCredentialOrdinal(current), - nextCredentialOrdinal: sessionDiagnosticCredentialOrdinal(next), - hasTransfer: transfer !== undefined, - freshTransfer, - retained, - reason: retained - ? 'fresh-transfer-retained' - : transfer === undefined - ? 'missing-transfer' - : !freshTransfer - ? 'replayed-transfer' - : 'refresh-rejected' - }); - if (!retained) invalidate(); + if (!freshTransfer || !refresh(transfer)) invalidate(); } current = next; if (transfer !== undefined) { diff --git a/tests/e2e-ui/gateway/refresh.mjs b/tests/e2e-ui/gateway/refresh.mjs index 6182477a..c5895a35 100644 --- a/tests/e2e-ui/gateway/refresh.mjs +++ b/tests/e2e-ui/gateway/refresh.mjs @@ -59,10 +59,6 @@ export async function verifySessionRefreshContinuity(page, origin) { const lost = await page.evaluate(()=>globalThis.__refreshContinuity.lost); if (lost) { console.error('Redacted refresh replica diagnostics:', JSON.stringify(await page.evaluate(()=>globalThis.__replicaDiagnosticSnapshot?.()))); - console.error('Redacted refresh lifecycle trace:', JSON.stringify(await page.evaluate(() => ({ - refresh: globalThis.__distributedRefreshTrace ?? [], - session: globalThis.__distributedSessionTrace ?? [] - })))); } assert.equal(lost,false,route+' rows were removed during token refresh: '+JSON.stringify(await page.evaluate(()=>globalThis.__refreshContinuity.events))); } finally { diff --git a/tests/e2e-ui/ui/src/lib/components/shared/AuthRefresh.svelte b/tests/e2e-ui/ui/src/lib/components/shared/AuthRefresh.svelte index e970f89d..0f32da8b 100644 --- a/tests/e2e-ui/ui/src/lib/components/shared/AuthRefresh.svelte +++ b/tests/e2e-ui/ui/src/lib/components/shared/AuthRefresh.svelte @@ -13,48 +13,6 @@ let refreshTimer: number | undefined; let retryTimer: number | undefined; - function traceRefresh(event: Readonly>) { - const maxEvents = 64; - const diagnostics = globalThis as typeof globalThis & { - __captureReplicaDiagnostics?: unknown; - __distributedRefreshTrace?: unknown; - }; - if (diagnostics.__captureReplicaDiagnostics !== true) return; - if (!Array.isArray(diagnostics.__distributedRefreshTrace)) { - diagnostics.__distributedRefreshTrace = []; - } - const trace = diagnostics.__distributedRefreshTrace as unknown[]; - trace.push({ - time: performance.now(), - ...event - }); - if (trace.length > maxEvents) trace.splice(0, trace.length - maxEvents); - } - - function withSessionSource(origin: string, action: () => T): T { - const diagnostics = globalThis as typeof globalThis & { - __captureReplicaDiagnostics?: unknown; - __distributedSessionSourceOrigin?: unknown; - }; - if (diagnostics.__captureReplicaDiagnostics !== true) return action(); - const previous = diagnostics.__distributedSessionSourceOrigin; - diagnostics.__distributedSessionSourceOrigin = origin; - try { - return action(); - } finally { - diagnostics.__distributedSessionSourceOrigin = previous; - } - } - - function credentialOrdinal(value: unknown): number | undefined { - const diagnostics = globalThis as typeof globalThis & { - __distributedSessionCredentialOrdinal?: unknown; - }; - return typeof diagnostics.__distributedSessionCredentialOrdinal === 'function' - ? (diagnostics.__distributedSessionCredentialOrdinal as (value: unknown) => number | undefined)(value) - : undefined; - } - function clearTimers() { if (refreshTimer !== undefined) window.clearTimeout(refreshTimer); if (retryTimer !== undefined) window.clearTimeout(retryTimer); @@ -64,7 +22,6 @@ async function refreshSession() { try { - traceRefresh({ kind: 'refresh-start' }); const response = await fetch('/api/auth/refresh', { method: 'POST', credentials: 'same-origin', @@ -81,24 +38,11 @@ if (response.ok) { const result = await response.json(); - traceRefresh({ - kind: 'refresh-response', - status: response.status, - hasPageData: Boolean(result.pageData), - hasDistributed: result.pageData?.distributed !== undefined, - hasAuthority: result.pageData?.distributedAuthority !== undefined, - credentialOrdinal: credentialOrdinal(result.pageData) - }); - if (result.pageData) { - withSessionSource('refresh-response', () => onRefresh(result.pageData)); - } - if (result.pageData) traceRefresh({ kind: 'refresh-seed-applied' }); + if (result.pageData) onRefresh(result.pageData); } if (response.ok || response.status === 401) { - traceRefresh({ kind: 'invalidate-all-start', status: response.status }); await invalidateAll(); - traceRefresh({ kind: 'invalidate-all-complete', status: response.status }); return; } } catch (error) { diff --git a/tests/e2e-ui/ui/src/routes/+layout.svelte b/tests/e2e-ui/ui/src/routes/+layout.svelte index 828ced00..cd8bb97f 100644 --- a/tests/e2e-ui/ui/src/routes/+layout.svelte +++ b/tests/e2e-ui/ui/src/routes/+layout.svelte @@ -68,16 +68,7 @@ }); function applyPageData(next: SveltekitDistributedPageData) { - const diagnostics = globalThis as typeof globalThis & Record; - const previousSourceOrigin = diagnostics.__distributedSessionSourceOrigin; - if (diagnostics.__captureReplicaDiagnostics === true) { - diagnostics.__distributedSessionSourceOrigin = 'sveltekit-data'; - } - try { - pageData.set(next); - } finally { - diagnostics.__distributedSessionSourceOrigin = previousSourceOrigin; - } + pageData.set(next); if ( next.distributed === undefined || next.distributedAuthority === undefined ||