fix(celld): make command lifecycle durable and fast

.github/workflows/integration-celld.yaml

@@ -5,7 +5,10 @@ name: celld (live + e2e-celld)
 #
 # Local parity:
 #   make -C tests/e2e-celld test
-#   make -C tests/e2e-ui up-celld-nats && make -C tests/e2e-ui test-celld
+#   make -C tests/e2e-ui up && make -C tests/e2e-ui up-celld-nats
+#   WATCH=0 WATCH_WORKER=0 make -C tests/e2e-celld run
+#   E2E_UI_ORIGIN=http://localhost:5180 npx --prefix tests/e2e-ui playwright test \
+#     todos.user.spec.ts chat.user.spec.ts --project chromium-user
 #
 # Default `cargo test` (quality) still runs fixture-only celld checks and
 # skips live HTTP unless CELLD_URL is set. This job sets CELLD_URL / NATS_URL.
@@ -14,8 +17,9 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
-  CELLD_HTTP_PORT: "18080"
-  CELLD_URL: http://127.0.0.1:18080
+  # Zitadel owns :18080 in the browser topology.
+  CELLD_HTTP_PORT: "18880"
+  CELLD_URL: http://127.0.0.1:18880
   NATS_PORT: "14222"
   NATS_URL: nats://127.0.0.1:14222
   AZURE_STORAGE_USE_EMULATOR: "true"
@@ -67,6 +71,14 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: "22"
+          cache: npm
+          cache-dependency-path: |
+            js/package-lock.json
+            tests/e2e-ui/package-lock.json
+            tests/e2e-ui/ui/package-lock.json
+
+      - name: Install host tools
+        run: sudo apt-get update && sudo apt-get install -y jq openssl curl
 
       - name: Install esbuild
         run: npm install -g esbuild
@@ -81,6 +93,9 @@ jobs:
           curl -fsSL https://celld.dev/install.sh | sh
           echo "$HOME/.local/bin" >> "$GITHUB_PATH"
 
+      - name: Bring up Postgres + Zitadel and bootstrap OIDC
+        run: make -C tests/e2e-ui up
+
       - name: Bring up Azurite + celld + NATS
         run: |
           command -v celld
@@ -90,6 +105,95 @@ jobs:
       - name: Live celld HTTP + NATS profile tests
         run: make -C tests/e2e-ui test-celld
 
+      - name: Build e2e-celld API and UI
+        run: |
+          cargo build --manifest-path tests/e2e-celld/Cargo.toml \
+            -p e2e-celld-runner --bin e2e-celld
+          make -C tests/e2e-ui ui-install
+          npm install --prefix tests/e2e-ui
+
+      - name: Start e2e-celld API + UI
+        run: |
+          set -euo pipefail
+          set -a
+          # shellcheck disable=SC1091
+          . tests/e2e-ui/e2e-ui.env
+          set +a
+          export BIND=0.0.0.0:8791
+          export CELLD_URL=http://127.0.0.1:18880
+          export NATS_URL=nats://127.0.0.1:14222
+          export AUTH_URL=http://localhost:5180
+          export AUTH_USE_SECURE_COOKIES=false
+          export AUTH_TRUST_HOST=true
+
+          tests/e2e-celld/target/debug/e2e-celld \
+            > tests/e2e-celld/.ci-runner.log 2>&1 &
+          echo $! > tests/e2e-celld/.ci-runner.pid
+
+          ok=0
+          for i in $(seq 1 120); do
+            code=$(curl -s -o /dev/null -w '%{http_code}' -X POST \
+              "http://127.0.0.1:8791/graphql" \
+              -H 'content-type: application/json' \
+              -d '{"query":"{ __typename }"}' 2>/dev/null || echo 000)
+            if [ "$code" = "200" ] || [ "$code" = "401" ]; then ok=1; break; fi
+            sleep 0.5
+          done
+          if [ "$ok" != "1" ]; then
+            echo "e2e-celld API failed to become ready"
+            tail -120 tests/e2e-celld/.ci-runner.log
+            exit 1
+          fi
+
+          cd tests/e2e-ui/ui
+          PUBLIC_E2E_PROFILE=celld-nats \
+            E2E_API_ORIGIN=http://127.0.0.1:8791 \
+            npm run dev -- --host localhost --port 5180 \
+            > ../.ci-celld-ui.log 2>&1 &
+          echo $! > ../.ci-celld-ui.pid
+          cd ../../..
+
+          ok=0
+          for i in $(seq 1 60); do
+            code=$(curl -s -o /dev/null -w '%{http_code}' \
+              "http://localhost:5180/" 2>/dev/null || echo 000)
+            if [ "$code" = "200" ] || [ "$code" = "302" ] || [ "$code" = "303" ]; then
+              ok=1
+              break
+            fi
+            sleep 0.5
+          done
+          if [ "$ok" != "1" ]; then
+            echo "e2e-celld UI failed to become ready (last HTTP $code)"
+            tail -100 tests/e2e-ui/.ci-celld-ui.log
+            exit 1
+          fi
+
+      - name: Install Playwright + Chromium
+        working-directory: tests/e2e-ui
+        run: npx playwright install chromium --with-deps
+
+      - name: Todo + Chat browser lifecycle through celld
+        working-directory: tests/e2e-ui
+        run: >-
+          npx playwright test todos.user.spec.ts chat.user.spec.ts
+          --project chromium-user
+        env:
+          E2E_UI_ORIGIN: http://localhost:5180
+          E2E_API_ORIGIN: http://127.0.0.1:8791
+          CI: true
+
+      - name: Upload celld Playwright report
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: celld-playwright-report
+          path: |
+            tests/e2e-ui/playwright-report
+            tests/e2e-ui/test-results
+          if-no-files-found: ignore
+          retention-days: 7
+
       - name: Dump logs on failure
         if: failure()
         run: |
@@ -99,9 +203,23 @@ jobs:
           echo '=== NATS profile compose ==='
           docker compose -f tests/e2e-ui/celld-nats-profile/docker-compose.yml ps -a || true
           docker compose -f tests/e2e-ui/celld-nats-profile/docker-compose.yml logs --tail=80 || true
+          echo '=== e2e-celld API ==='
+          tail -180 tests/e2e-celld/.ci-runner.log || true
+          echo '=== e2e-celld UI ==='
+          tail -100 tests/e2e-ui/.ci-celld-ui.log || true
+          echo '=== Postgres + Zitadel ==='
+          docker compose -f tests/e2e-ui/docker/docker-compose.yml ps -a || true
+          docker compose -f tests/e2e-ui/docker/docker-compose.yml logs --tail=100 || true
 
       - name: Tear down
         if: always()
         run: |
+          [ -f tests/e2e-ui/.ci-celld-ui.pid ] && \
+            kill "$(cat tests/e2e-ui/.ci-celld-ui.pid)" 2>/dev/null || true
+          [ -f tests/e2e-celld/.ci-runner.pid ] && \
+            kill "$(cat tests/e2e-celld/.ci-runner.pid)" 2>/dev/null || true
+          lsof -ti:5180 2>/dev/null | xargs -r kill -9 2>/dev/null || true
+          lsof -ti:8791 2>/dev/null | xargs -r kill -9 2>/dev/null || true
           make -C tests/e2e-ui down-celld-nats || true
           make -C tests/e2e-ui down-celld || true
+          docker compose -f tests/e2e-ui/docker/docker-compose.yml down -v || true

distributed_cli/src/client_compiler/manifest/projections.rs

@@ -319,13 +319,6 @@ pub(crate) fn validate_command_projections(
                     .get(&value.slot)
                     .expect("exact slot coverage was validated");
                 validate_preview_source(command, &value.source, expected)?;
-                if matches!(
-                    value.source,
-                    ManifestProjectionPreviewSource::Unknown
-                        | ManifestProjectionPreviewSource::Absent
-                ) {
-                    requiring_revalidation.insert(command.name.clone());
-                }
             }
         }
         if selected_programs.is_empty() {

distributed_cli/src/client_compiler/projection_delta/preview.rs

@@ -165,7 +165,10 @@ impl CompiledCommandProjection {
     }
 
     pub(crate) fn requires_revalidation(&self) -> bool {
-        !self.preview.recoveries.is_empty()
+        self.preview
+            .recoveries
+            .iter()
+            .any(|recovery| recovery.condition == PreviewRecoveryCondition::Always)
     }
 
     pub(crate) fn selected_models(&self) -> &BTreeSet<String> {

distributed_cli/src/client_compiler/tests.rs

@@ -3221,6 +3221,7 @@ fn command_protocol_and_extensions_are_preserved_exactly() {
     assert!(!partial_commands.contains("\"op\": \"upsert\""));
     assert!(partial_commands.contains("\"condition\": \"if_record_missing\""));
     assert!(partial_commands.contains("\"kind\": \"record\""));
+    assert!(partial_commands.contains("\"required\": false"));
 
     let absent = compile_client(ClientCompileInput::new(
         absent_value,
@@ -3290,6 +3291,7 @@ fn command_protocol_and_extensions_are_preserved_exactly() {
         assert!(fallback_commands.contains("\"kind\": \"model\""));
         assert!(!fallback_commands.contains("\"op\": \"upsert\""));
         assert!(!fallback_commands.contains("\"op\": \"patch\""));
+        assert!(fallback_commands.contains("\"required\": true"));
     }
 
     let delete = compile_client(ClientCompileInput::new(

js/scripts/pack-smoke.mjs

@@ -430,6 +430,7 @@ assert.deepEqual(Object.keys(replicaSurface).sort(), [
   'createReplicaGraphqlTransport',
   'createReplicaIndexMaintenanceRegistry',
   'createReplicaIndexedDbPersistence',
+  'createReplicaUuidV7',
   'createWasmJsonPure',
   'decideReplicaPaginationMaintenance',
   'evaluateReplicaFilter',

js/src/replica/command-id.ts

@@ -1,5 +1,5 @@
-/** Create a browser/Node UUIDv7 command identity. */
-export function createReplicaCommandId(): string {
+/** Create a browser/Node UUIDv7 identity for commands or generated record IDs. */
+export function createReplicaUuidV7(): string {
 	const crypto = globalThis.crypto;
 	if (!crypto || typeof crypto.getRandomValues !== 'function') {
 		throw new Error('replica commands require crypto.getRandomValues');
@@ -19,3 +19,6 @@ export function createReplicaCommandId(): string {
 		.slice(6, 8)
 		.join('')}-${hex.slice(8, 10).join('')}-${hex.slice(10).join('')}`;
 }
+
+/** Package-internal semantic alias used while preparing command envelopes. */
+export const createReplicaCommandId = createReplicaUuidV7;

js/src/replica/command-runtime/create.ts

@@ -555,8 +555,7 @@ export function createReplicaCommandRuntime<
 				throw new Error('projection delta changed during command replay');
 			}
 			return Object.freeze({
-				requiresRevalidation:
-					actual.revalidate || actual.obligations.length === 0
+				requiresRevalidation: actual.revalidate
 			});
 		}
 		assertActualProjectionCapabilities(
@@ -572,14 +571,13 @@ export function createReplicaCommandRuntime<
 			canonical,
 			operations,
 			revalidation:
-				actual.revalidate || actual.obligations.length === 0
+				actual.revalidate
 					? actualProjectionRevalidation(
 							prepared.revalidation,
 							actual.delta
 						)
 					: undefined,
-			requiresRevalidation:
-				actual.revalidate || actual.obligations.length === 0
+			requiresRevalidation: actual.revalidate
 		});
 	};
 
@@ -1022,6 +1020,18 @@ export function createReplicaCommandRuntime<
 						if (tracker.pending !== undefined) {
 							settleTrackedProjection(tracker, pending);
 						}
+					} else if (
+						metadata.state === 'atomic' &&
+						!prepared.revalidation.required &&
+						!statusRequiresRevalidation
+					) {
+						/*
+						 * An exact terminal delta proves delivery but carries no
+						 * canonical revision. Keep its accepted overlay until a later
+						 * comparable authoritative result seals it, without racing
+						 * sibling commands with a command-triggered query.
+						 */
+						settleTrackedProjection(tracker, pending);
 					} else if (
 						metadata.state === 'atomic' ||
 						(metadata.state === 'succeeded' &&

js/src/replica/distributed-replica/impl-optimistic.ts

@@ -1,4 +1,5 @@
 import type {
+	BaseCacheWriter,
 	CacheEngine,
 	OptimisticLayerReplacement
 } from '../../internal/cache-engine.js';
@@ -220,9 +221,18 @@ export function confirmOptimisticLayerOn<T>(
 	id: string,
 	update: (writer: ReplicaBaseWriter) => T
 ): T {
-	const result = host.engine.confirmOptimisticLayer(id, (writer) =>
+	return confirmOptimisticLayerWithCacheWriterOn(host, id, (writer) =>
 		update(baseWriter(writer))
 	);
+}
+
+/** Internal confirmation seam for protocol code that must atomically seal indexes. */
+export function confirmOptimisticLayerWithCacheWriterOn<T>(
+	host: OptimisticHost,
+	id: string,
+	update: (writer: BaseCacheWriter) => T
+): T {
+	const result = host.engine.confirmOptimisticLayer(id, update);
 	host.retireDiagnosticLayer(id, 'retired', 'atomic');
 	host.optimisticReceipts.delete(id);
 	host.syncDiagnostics();