diff --git a/.github/workflows/server-perf-soak.yml b/.github/workflows/server-perf-soak.yml index de9608b39..b642e8606 100644 --- a/.github/workflows/server-perf-soak.yml +++ b/.github/workflows/server-perf-soak.yml @@ -85,8 +85,8 @@ jobs: DW_PERF_MAX_SERVER_MEMORY_MB: "1024" DW_PERF_MAX_POLLING_KEYS: "2048" DW_PERF_MAX_FINAL_POLLING_KEYS: "0" - DW_PERF_MAX_SERVER_CACHE_KEYS_BY_POLICY: '{"polling_cache_availability_probe":0,"workflow_task_poll_requests":2048,"activity_task_poll_requests":2048,"query_task_poll_requests":2048,"sqlite_worker_poll_claim_gate":1,"long_poll_signals":2048,"long_poll_wait_slots":64,"workflow_query_tasks":128,"task_queue_admission_locks":256,"task_queue_dispatch_counters":256,"namespace_request_admission":256,"shared_service_boundary_admission":256,"namespace_durable_state_quota_rejections":256,"runtime_external_payload_quota_rejections":256,"workflow_task_expired_lease_recovery":256,"history_retention_inline":128,"worker_compatibility_heartbeat":128,"readiness_probe":32}' - DW_PERF_MAX_FINAL_SERVER_CACHE_KEYS_BY_POLICY: '{"polling_cache_availability_probe":0,"workflow_task_poll_requests":0,"activity_task_poll_requests":0,"query_task_poll_requests":0,"sqlite_worker_poll_claim_gate":0,"long_poll_signals":0,"long_poll_wait_slots":0,"workflow_query_tasks":0,"task_queue_admission_locks":0,"task_queue_dispatch_counters":0,"namespace_request_admission":0,"shared_service_boundary_admission":0,"namespace_durable_state_quota_rejections":0,"runtime_external_payload_quota_rejections":0,"workflow_task_expired_lease_recovery":0,"history_retention_inline":0,"worker_compatibility_heartbeat":0,"readiness_probe":0}' + DW_PERF_MAX_SERVER_CACHE_KEYS_BY_POLICY: '{"worker_poll_diagnostics":0,"polling_cache_availability_probe":0,"workflow_task_poll_requests":2048,"activity_task_poll_requests":2048,"query_task_poll_requests":2048,"sqlite_worker_poll_claim_gate":1,"long_poll_signals":2048,"long_poll_wait_slots":64,"workflow_query_tasks":128,"task_queue_admission_locks":256,"task_queue_dispatch_counters":256,"namespace_request_admission":256,"shared_service_boundary_admission":256,"namespace_durable_state_quota_rejections":256,"runtime_external_payload_quota_rejections":256,"workflow_task_expired_lease_recovery":256,"history_retention_inline":128,"worker_compatibility_heartbeat":128,"readiness_probe":32}' + DW_PERF_MAX_FINAL_SERVER_CACHE_KEYS_BY_POLICY: '{"worker_poll_diagnostics":0,"polling_cache_availability_probe":0,"workflow_task_poll_requests":0,"activity_task_poll_requests":0,"query_task_poll_requests":0,"sqlite_worker_poll_claim_gate":0,"long_poll_signals":0,"long_poll_wait_slots":0,"workflow_query_tasks":0,"task_queue_admission_locks":0,"task_queue_dispatch_counters":0,"namespace_request_admission":0,"shared_service_boundary_admission":0,"namespace_durable_state_quota_rejections":0,"runtime_external_payload_quota_rejections":0,"workflow_task_expired_lease_recovery":0,"history_retention_inline":0,"worker_compatibility_heartbeat":0,"readiness_probe":0}' DW_PERF_MAX_SERVER_MEMORY_SLOPE_MB_HOUR: "128" DW_PERF_REMOTE_WRITE_ENABLED: ${{ github.event_name != 'workflow_dispatch' || inputs.remote_write }} DW_PERF_REMOTE_WRITE_URL: ${{ vars.DW_PERF_REMOTE_WRITE_URL }} diff --git a/.github/workflows/server-perf.yml b/.github/workflows/server-perf.yml index 1f019e704..3a1a41888 100644 --- a/.github/workflows/server-perf.yml +++ b/.github/workflows/server-perf.yml @@ -24,6 +24,7 @@ on: - "app/Support/ServerReadiness.php" - "app/Support/RemoteScheduleStarter.php" - "app/Support/ServerPollingCache.php" + - "app/Support/WorkerPollDiagnostics.php" - "app/Support/SharedServiceBoundaryPolicy.php" - "app/Support/TaskQueueAdmission.php" - "app/Support/NamespaceRequestAdmission.php" @@ -82,6 +83,7 @@ on: - "app/Support/ServerReadiness.php" - "app/Support/RemoteScheduleStarter.php" - "app/Support/ServerPollingCache.php" + - "app/Support/WorkerPollDiagnostics.php" - "app/Support/SharedServiceBoundaryPolicy.php" - "app/Support/TaskQueueAdmission.php" - "app/Support/NamespaceRequestAdmission.php" @@ -272,8 +274,8 @@ jobs: # limit per poll. Allow the 1,000 synthetic starts plus SDK canary; # total keys and zero-after-drain remain independently enforced. DW_PERF_MIN_SAMPLE_COVERAGE: "0.75" - DW_PERF_MAX_SERVER_CACHE_KEYS_BY_POLICY: '{"polling_cache_availability_probe":0,"workflow_task_poll_requests":3072,"activity_task_poll_requests":512,"query_task_poll_requests":512,"sqlite_worker_poll_claim_gate":1,"long_poll_signals":2048,"workflow_query_tasks":64,"task_queue_admission_locks":128,"task_queue_dispatch_counters":128,"namespace_request_admission":128,"shared_service_boundary_admission":128,"namespace_durable_state_quota_rejections":128,"runtime_external_payload_quota_rejections":128,"workflow_task_expired_lease_recovery":1536,"history_retention_inline":64,"worker_compatibility_heartbeat":64,"readiness_probe":16}' - DW_PERF_MAX_FINAL_SERVER_CACHE_KEYS_BY_POLICY: '{"polling_cache_availability_probe":0,"workflow_task_poll_requests":0,"activity_task_poll_requests":0,"query_task_poll_requests":0,"sqlite_worker_poll_claim_gate":0,"long_poll_signals":0,"workflow_query_tasks":0,"task_queue_admission_locks":0,"task_queue_dispatch_counters":0,"namespace_request_admission":0,"shared_service_boundary_admission":0,"namespace_durable_state_quota_rejections":0,"runtime_external_payload_quota_rejections":0,"workflow_task_expired_lease_recovery":0,"history_retention_inline":0,"worker_compatibility_heartbeat":0,"readiness_probe":0}' + DW_PERF_MAX_SERVER_CACHE_KEYS_BY_POLICY: '{"worker_poll_diagnostics":0,"polling_cache_availability_probe":0,"workflow_task_poll_requests":3072,"activity_task_poll_requests":512,"query_task_poll_requests":512,"sqlite_worker_poll_claim_gate":1,"long_poll_signals":2048,"workflow_query_tasks":64,"task_queue_admission_locks":128,"task_queue_dispatch_counters":128,"namespace_request_admission":128,"shared_service_boundary_admission":128,"namespace_durable_state_quota_rejections":128,"runtime_external_payload_quota_rejections":128,"workflow_task_expired_lease_recovery":1536,"history_retention_inline":64,"worker_compatibility_heartbeat":64,"readiness_probe":16}' + DW_PERF_MAX_FINAL_SERVER_CACHE_KEYS_BY_POLICY: '{"worker_poll_diagnostics":0,"polling_cache_availability_probe":0,"workflow_task_poll_requests":0,"activity_task_poll_requests":0,"query_task_poll_requests":0,"sqlite_worker_poll_claim_gate":0,"long_poll_signals":0,"workflow_query_tasks":0,"task_queue_admission_locks":0,"task_queue_dispatch_counters":0,"namespace_request_admission":0,"shared_service_boundary_admission":0,"namespace_durable_state_quota_rejections":0,"runtime_external_payload_quota_rejections":0,"workflow_task_expired_lease_recovery":0,"history_retention_inline":0,"worker_compatibility_heartbeat":0,"readiness_probe":0}' DW_PERF_COMPOSE_PROJECT: server-perf-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} RUNNER_ENVIRONMENT: "github-hosted" if: ${{ always() && success() }} diff --git a/app/Http/Controllers/Api/TaskQueueController.php b/app/Http/Controllers/Api/TaskQueueController.php index 907d18fe7..851f30663 100644 --- a/app/Http/Controllers/Api/TaskQueueController.php +++ b/app/Http/Controllers/Api/TaskQueueController.php @@ -10,6 +10,7 @@ use App\Support\TaskQueueAdmission; use App\Support\TaskQueueBuildIdRolloutSnapshot; use App\Support\TaskQueuePriorityFairnessSurface; +use App\Support\WorkerPollDiagnostics; use App\Support\WorkflowDefinitionRouting; use App\Support\WorkflowQueryTaskBroker; use Illuminate\Http\JsonResponse; @@ -30,6 +31,7 @@ public function __construct( private readonly TaskQueuePriorityFairnessSurface $priorityFairness, private readonly DeploymentLifecycleService $deployments, private readonly WorkflowDefinitionRouting $definitionRouting, + private readonly WorkerPollDiagnostics $pollDiagnostics, ) {} public function index(Request $request): JsonResponse @@ -439,6 +441,12 @@ private function withAdmission(string $namespace, array $payload): array 'query_tasks' => $this->queryTasks->queueAdmission($namespace, $taskQueue), ]; + $payload['poll_transport'] = $this->pollDiagnostics->snapshot($namespace, $taskQueue); + if ($payload['poll_transport']['status'] === 'warning') { + $payload['repair']['needs_attention'] = true; + $payload['repair']['transport_warnings'] = [$payload['poll_transport']['last_disconnect']]; + } + return $payload; } diff --git a/app/Http/Middleware/WorkerPollResponseProgress.php b/app/Http/Middleware/WorkerPollResponseProgress.php new file mode 100644 index 000000000..5a8c06ac2 --- /dev/null +++ b/app/Http/Middleware/WorkerPollResponseProgress.php @@ -0,0 +1,36 @@ +headers->get('Accept', ''))); + $transport = $accept->get(LongPollResponseProgress::MEDIA_TYPE); + + if (! $request->isMethod('POST') || ! in_array($request->path(), self::PATHS, true) + || $transport === null || $transport->getQuality() <= 0 + || $transport->getQuality() < ($accept->get('application/json')?->getQuality() ?? 0)) { + return $next($request); + } + + $progress = app()->make(LongPollResponseProgress::class, ['request' => $request]); + $request->attributes->set(LongPollResponseProgress::REQUEST_ATTRIBUTE, $progress); + + return $progress->response(static fn (): Response => $next($request)); + } +} diff --git a/app/Support/ActivityTaskPollRequestStore.php b/app/Support/ActivityTaskPollRequestStore.php index 89732faf6..bb15d7970 100644 --- a/app/Support/ActivityTaskPollRequestStore.php +++ b/app/Support/ActivityTaskPollRequestStore.php @@ -72,6 +72,7 @@ public function waitForResult( ): array { $pendingKey = $this->pendingKey($namespace, $taskQueue, $buildId, $leaseOwner, $pollRequestId); $deadline = microtime(true) + (($timeoutMilliseconds ?? $this->waitTimeoutMilliseconds()) / 1000); + $progress = LongPollResponseProgress::current(); while (microtime(true) < $deadline) { $result = $this->result($namespace, $taskQueue, $buildId, $leaseOwner, $pollRequestId); @@ -88,6 +89,7 @@ public function waitForResult( ]; } + $progress?->checkpoint(); $this->pause($this->pollIntervalMilliseconds()); } diff --git a/app/Support/LongPollResponseProgress.php b/app/Support/LongPollResponseProgress.php new file mode 100644 index 000000000..2246f4877 --- /dev/null +++ b/app/Support/LongPollResponseProgress.php @@ -0,0 +1,189 @@ +startedAt = hrtime(true) / 1_000_000_000; + } + + public static function current(): ?self + { + if (! app()->bound('request')) { + return null; + } + + $progress = request()->attributes->get(self::REQUEST_ATTRIBUTE); + + return $progress instanceof self ? $progress : null; + } + + public function beginWait(int $waitSeconds): void + { + $this->waitSeconds = $waitSeconds; + $this->checkpoint(); + } + + public function checkpoint(): void + { + if ($this->fiber === null || Fiber::getCurrent() !== $this->fiber + || hrtime(true) / 1_000_000_000 - $this->lastProgressAt < 1) { + return; + } + + $this->waitSeconds ??= max(0, (int) $this->request->input('timeout_seconds', config('server.polling.timeout', 30))); + $this->lastProgressAt = hrtime(true) / 1_000_000_000; + Fiber::suspend(); + } + + public function response(Closure $handler): Response + { + $this->fiber = new Fiber($handler); + $this->fiber->start(); + $status = $this->fiber->isTerminated() ? $this->terminalResponse()->getStatusCode() : 200; + + return new StreamedResponse($this->send(...), $status, [ + 'Content-Type' => self::MEDIA_TYPE, + 'Cache-Control' => 'no-store', + 'X-Accel-Buffering' => 'no', + 'Vary' => 'Accept', + ]); + } + + private function send(): void + { + $previous = ignore_user_abort(true); + + try { + while (! $this->fiber->isTerminated()) { + echo "\n"; + $this->flushOutput(); + + if ($this->clientDisconnected()) { + $observedSeconds = hrtime(true) / 1_000_000_000 - $this->startedAt; + try { + $this->fiber->throw(new PollClientDisconnected); + } finally { + // Logging and optional cache I/O follow exact wait + // cleanup, so a slow observer cannot retain capacity. + $this->recordAbandonment($observedSeconds); + } + + return; + } + + $this->fiber->resume(); + } + + $response = $this->terminalResponse(); + $headers = array_filter([ + 'retry-after' => $response->headers->get('Retry-After'), + strtolower(WorkerProtocol::HEADER) => $response->headers->get(WorkerProtocol::HEADER), + ], static fn (?string $value): bool => $value !== null); + + // Keep the already encoded worker body intact, including numeric + // representation and payload envelopes. Progress is JSON whitespace. + echo '{"schema":"'.self::SCHEMA.'","status":'.$response->getStatusCode().',"headers":'; + echo json_encode((object) $headers, JSON_THROW_ON_ERROR); + echo ',"body":', $response->getContent(), '}'; + $this->flushOutput(); + + if ($this->clientDisconnected()) { + $this->recordAbandonment(); + } + } catch (PollClientDisconnected) { + // The exact wait owner has unwound through its normal finally. + } finally { + try { + if ($this->fiber->isSuspended()) { + try { + $this->fiber->throw(new PollClientDisconnected); + } catch (PollClientDisconnected) { + // Do not leave an admitted wait suspended on output failure. + } + } + } finally { + ignore_user_abort($previous); + } + } + } + + private function terminalResponse(): JsonResponse + { + $response = $this->fiber->getReturn(); + + if ($response instanceof JsonResponse) { + return $response; + } + + logger()->error('Worker poll returned an unsupported progress-transport response.'); + + return new JsonResponse([ + 'task' => null, + 'reason' => 'worker_poll_transport_unavailable', + 'retryable' => true, + ], 500); + } + + private function recordAbandonment(?float $observedSeconds = null): void + { + if ($this->abandonmentRecorded || $this->waitSeconds === null) { + return; + } + + $this->abandonmentRecorded = true; + $observedSeconds ??= hrtime(true) / 1_000_000_000 - $this->startedAt; + logger()->warning('Worker long-poll client disconnected before its response was delivered.', [ + 'namespace' => $this->request->attributes->get('namespace'), + 'task_queue' => $this->request->input('task_queue'), + 'observed_seconds' => round($observedSeconds, 3), + 'configured_wait_seconds' => $this->waitSeconds, + ]); + app(WorkerPollDiagnostics::class)->recordDisconnect( + (string) $this->request->attributes->get('namespace'), + (string) $this->request->input('task_queue'), + basename(dirname($this->request->path())), + $observedSeconds, + $this->waitSeconds, + ); + } + + protected function clientDisconnected(): bool + { + return connection_aborted() !== 0; + } + + protected function flushOutput(): void + { + if (ob_get_level() > 0) { + ob_flush(); + } + + flush(); + } +} diff --git a/app/Support/LongPoller.php b/app/Support/LongPoller.php index d31139aac..7dca3ec91 100644 --- a/app/Support/LongPoller.php +++ b/app/Support/LongPoller.php @@ -87,6 +87,9 @@ public function until( ); try { + $progress = LongPollResponseProgress::current(); + $progress?->beginWait($timeoutSeconds); + while (microtime(true) < $deadline) { $now = microtime(true); $sleepUntil = min( @@ -97,6 +100,7 @@ public function until( $sleepMilliseconds = max(1, (int) ceil(max(0, $sleepUntil - $now) * 1000)); $this->pause($sleepMilliseconds); + $progress?->checkpoint(); $now = microtime(true); $wakeChanged = $wakeChannels !== [] && $this->signals->changed($wakeSnapshot); diff --git a/app/Support/PollClientDisconnected.php b/app/Support/PollClientDisconnected.php new file mode 100644 index 000000000..869467933 --- /dev/null +++ b/app/Support/PollClientDisconnected.php @@ -0,0 +1,7 @@ +pendingKey($namespace, $taskQueue, $buildId, $leaseOwner, $pollRequestId); $deadline = microtime(true) + (($timeoutMilliseconds ?? $this->waitTimeoutMilliseconds()) / 1000); + $progress = LongPollResponseProgress::current(); while (microtime(true) < $deadline) { $result = $this->result($namespace, $taskQueue, $buildId, $leaseOwner, $pollRequestId); @@ -133,6 +134,7 @@ public function waitForResult( ]; } + $progress?->checkpoint(); $this->pause($this->pollIntervalMilliseconds()); } diff --git a/app/Support/WorkerPollDiagnostics.php b/app/Support/WorkerPollDiagnostics.php new file mode 100644 index 000000000..bec53d425 --- /dev/null +++ b/app/Support/WorkerPollDiagnostics.php @@ -0,0 +1,64 @@ += $waitSeconds || ! $this->cache->available()) { + return; + } + + try { + $this->cache->store()->put($this->key($namespace, $taskQueue), [ + 'level' => 'warning', + 'reason' => 'worker_poll_client_disconnected', + 'poll_kind' => $pollKind, + 'observed_seconds' => round($observedSeconds, 3), + 'configured_wait_seconds' => $waitSeconds, + 'observed_at' => now()->toISOString(), + 'remediation' => 'Compare worker HTTP deadlines and proxy idle timeouts with the Server wait. Use workers supporting negotiated poll progress.', + ], self::RETENTION_SECONDS); + } catch (Throwable) { + // The normal disconnect warning still exists. Diagnostic storage + // must not interrupt exact wait cleanup or change task authority. + } + } + + /** @return array{status: string, retention_seconds: int, last_disconnect: array|null} */ + public function snapshot(string $namespace, string $taskQueue): array + { + $result = ['status' => 'unavailable', 'retention_seconds' => self::RETENTION_SECONDS, 'last_disconnect' => null]; + + if (! $this->cache->available()) { + return $result; + } + + try { + $last = $this->cache->store()->get($this->key($namespace, $taskQueue)); + if ($last === null || is_array($last)) { + $result = ['status' => $last === null ? 'no_recent_observation' : 'warning', + 'retention_seconds' => self::RETENTION_SECONDS, 'last_disconnect' => $last]; + } + } catch (Throwable) { + // Retain the unavailable state when a diagnostic read fails. + } + + return $result; + } + + private function key(string $namespace, string $taskQueue): string + { + return self::PREFIX.hash('sha256', json_encode([$namespace, $taskQueue], JSON_THROW_ON_ERROR)); + } +} diff --git a/app/Support/WorkflowTaskPollRequestStore.php b/app/Support/WorkflowTaskPollRequestStore.php index 7010fdb1b..c2142ec34 100644 --- a/app/Support/WorkflowTaskPollRequestStore.php +++ b/app/Support/WorkflowTaskPollRequestStore.php @@ -93,6 +93,7 @@ public function waitForResult( ): array { $pendingKey = $this->pendingKey($namespace, $taskQueue, $buildId, $leaseOwner, $pollRequestId); $deadline = microtime(true) + (($timeoutMilliseconds ?? $this->waitTimeoutMilliseconds()) / 1000); + $progress = LongPollResponseProgress::current(); while (microtime(true) < $deadline) { $result = $this->result( @@ -116,6 +117,7 @@ public function waitForResult( ]; } + $progress?->checkpoint(); $this->pause($this->pollIntervalMilliseconds()); } diff --git a/bootstrap/app.php b/bootstrap/app.php index 5e57be6a1..6d4d85540 100644 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -10,6 +10,7 @@ use App\Support\ControlPlaneProtocol; use App\Support\DatabaseStorageExhausted; use App\Support\NamespaceDurableStateException; +use App\Support\PollClientDisconnected; use App\Support\RuntimeExternalPayloadAudit; use App\Support\RuntimeExternalPayloadException; use App\Support\WorkerProtocol; @@ -42,6 +43,17 @@ ]); }) ->withExceptions(function (Exceptions $exceptions) { + $exceptions->dontReport([PollClientDisconnected::class]); + + // Laravel selects renderers by the first parameter's reflected type. + $exceptions->render(function (PollClientDisconnected $exception) { + return WorkerProtocol::json([ + 'task' => null, + 'reason' => 'poll_client_disconnected', + 'retryable' => true, + ], 499); + }); + $exceptions->render(function (Throwable $exception, Request $request) { return DatabaseStorageExhausted::is($exception) ? DatabaseStorageExhausted::response($request) diff --git a/config/dw-bounded-growth.php b/config/dw-bounded-growth.php index c8498177a..c701b3888 100644 --- a/config/dw-bounded-growth.php +++ b/config/dw-bounded-growth.php @@ -16,6 +16,7 @@ use App\Support\TaskQueueAdmission; use App\Support\WorkerCompatibilityHeartbeatRecorder; use App\Support\WorkerPollClaimGate; +use App\Support\WorkerPollDiagnostics; use App\Support\WorkflowQueryTaskBroker; use App\Support\WorkflowTaskFailureMetrics; use App\Support\WorkflowTaskPoller; @@ -55,6 +56,15 @@ ], 'cache_keys' => [ + 'worker_poll_diagnostics' => [ + 'owner' => WorkerPollDiagnostics::class, + 'prefix' => 'server:worker-poll-diagnostic:', + 'dimensions' => ['namespace', 'task_queue'], + 'ttl' => 'WorkerPollDiagnostics::RETENTION_SECONDS, 900 seconds from the latest observed early disconnect.', + 'bound' => 'One fixed-size last-observation value per namespace and task queue touched within the TTL window. No request, worker, task or attempt index is retained.', + 'admission' => 'Only an observed negotiated poll disconnect before its configured wait writes a value; subsequent observations overwrite that queue value.', + 'eviction' => 'Cache TTL without a sweeper. Diagnostic values never grant or revoke task authority.', + ], 'polling_cache_availability_probe' => [ 'owner' => ServerPollingCache::class, 'prefix' => 'server:polling-cache:', diff --git a/docs/bounded-growth.md b/docs/bounded-growth.md index 2be7aadb3..94f53fd87 100644 --- a/docs/bounded-growth.md +++ b/docs/bounded-growth.md @@ -1,5 +1,10 @@ # Server Bounded-Growth Policy +The `worker_poll_diagnostics` surface uses `server:worker-poll-diagnostic:` with one +last-observation value per namespace and task queue. Each value expires after +15 minutes; later observations overwrite it. There are no request, worker or +task indexes, and the observation is not task authority. + The server owns a few cache-backed coordination surfaces, SQL recovery scans, JSON metric surfaces, conformance evidence fields, and the perf harness metrics that can be remote-written during soaks. diff --git a/docs/worker-poll-progress.md b/docs/worker-poll-progress.md new file mode 100644 index 000000000..54d6dee06 --- /dev/null +++ b/docs/worker-poll-progress.md @@ -0,0 +1,94 @@ +# Worker poll progress transport + +Development status: this transport is not in a published Server image or SDK. +Task-delivery recovery and first-party SDK integration remain part +of [Server #526](https://github.com/durable-workflow/server/issues/526). + +## Negotiation + +Workflow, activity, query and dedicated update-validation polls may explicitly +request: + +```http +Accept: application/vnd.durable-workflow.poll.v1+json, application/json;q=0.9 +``` + +The media type is case-insensitive. A zero quality value declines it, and a +higher preference for `application/json` retains the ordinary response. +Requests without the explicit media type retain the existing JSON contract. +Other routes and HTTP methods retain their existing responses. + +The negotiated response uses that vendor media type and contains one JSON +document. Newlines sent during a wait are leading JSON whitespace, at most +once per second. They keep a proxy's idle timer from expiring and let the +Server observe a closed client connection while waiting. + +```json +{ + "schema": "durable-workflow.poll-transport.v1", + "status": 200, + "headers": {"x-durable-workflow-protocol-version": "1.20"}, + "body": {"task": null, "poll_status": "empty"} +} +``` + +`body` preserves the original encoded worker response. The transport does not +decode and re-encode task payloads or change the worker protocol version. +`headers` carries the original worker protocol header and `Retry-After`, when +present, with lowercase keys. An empty header map is `{}`. + +Before waiting, an error can retain its HTTP status as well as its envelope +status. Once progress bytes have committed HTTP 200, the final `status` and +`body` still preserve the original result, including late 409, 422, 429 and +5xx errors. Clients must unwrap the document and apply the original status, +headers and error/retry rules before treating it as a task response. + +The response disables intermediary buffering with `X-Accel-Buffering: no` +and uses `Cache-Control: no-store`. It bypasses the Server's ordinary JSON +body compression so progress is not combined with a separately compressed +terminal response. + +## Lifetime and client requirements + +The request runs inside a Fiber. Its first actual wait yields before response +progress begins. Closing the client throws a dedicated interruption into that +same Fiber so existing `finally` blocks release its exact wait resources. +Duplicate poll-result waits also yield; abandoning a duplicate does not remove +the live leader's pending key. The previous `ignore_user_abort` setting is +restored after response processing. + +SDKs must accept both ordinary JSON and the negotiated envelope, reject invalid +envelopes, and preserve an absolute HTTP attempt deadline across all progress +bytes. An idle/read timeout alone can be renewed by each byte. Early cutoffs +must produce a warning with the observed duration and requested wait. + +A completed socket write does not prove that an SDK received or started a +task. A failed duplicate response must not revoke a lease delivered through +another response. Authoritative task-delivery confirmation and recovery remain +separate work: qualification must cover exact namespace, task, owner, attempt +and poll-request identity, successful replay and concurrent response loss. +Wait-slot or cache hints do not authorize task revocation. + +## Queue diagnostics + +Task-queue list and detail responses include `poll_transport`. A recent early +disconnect produces a warning with its poll kind, observed duration, configured +wait and observation time, and sets `repair.needs_attention` with the same +warning in `repair.transport_warnings`. This identifies an observed disconnect; +it does not establish that a proxy caused it or that a task was delivered. + +The observation store keeps only the latest cutoff per namespace and queue for +15 minutes under `server:worker-poll-diagnostic:`. There is no per-request, +worker or task index. A cache outage reports `status: unavailable` rather than +claiming that no disconnect occurred. An available store without a retained +observation reports `no_recent_observation`. Neither state changes task leases. + +## Qualification + +The focused PHP tests are in `tests/Feature/WorkerPollResponseProgressTest.php`. +They cover negotiation, raw JSON preservation, late framework errors, +resource cleanup and duplicate-poll ownership. Run them using the repository's +documented container test environment. Owning #526 records the actual PHP, +nginx, database and SDK observations and retains the full published-image +acceptance gate. The Rust owner owns the shared parity case when the contract +and release are ready; existing frozen fixtures are unchanged. diff --git a/routes/api.php b/routes/api.php index ec39b2e87..f9f79ccb5 100644 --- a/routes/api.php +++ b/routes/api.php @@ -40,6 +40,7 @@ use App\Http\Middleware\RequireTopologyRoles; use App\Http\Middleware\RequireWorkflowBootstrapReady; use App\Http\Middleware\RuntimeExternalPayloadTransport; +use App\Http\Middleware\WorkerPollResponseProgress; use App\Http\Middleware\WorkerProtocolVersionResolver; use Illuminate\Support\Facades\Route; @@ -228,7 +229,7 @@ }); // ── Worker Task Polling ────────────────────────────────────────── - Route::prefix('worker')->middleware([$worker, $wpv, $httpWorker, $workflowBootstrap, $ns])->group(function () { + Route::prefix('worker')->middleware([$worker, $wpv, $httpWorker, $workflowBootstrap, $ns, WorkerPollResponseProgress::class])->group(function () { Route::delete('/registrations/{workerId}', [WorkerDeregistrationController::class, 'destroy']); Route::post('/registrations/{workerId}/deregister', [WorkerDeregistrationController::class, 'destroyFenced']); Route::post('/heartbeat', [WorkerController::class, 'heartbeat']); diff --git a/scripts/perf/server_soak.py b/scripts/perf/server_soak.py index cc5905f0c..bc0de0cad 100755 --- a/scripts/perf/server_soak.py +++ b/scripts/perf/server_soak.py @@ -28,6 +28,7 @@ WORKER_PROTOCOL_VERSION = os.environ.get("DW_PERF_WORKER_PROTOCOL_VERSION", "1.2") ERROR_WRITE_LOCK = threading.Lock() SERVER_CACHE_KEY_PATTERNS = { + "worker_poll_diagnostics": "*server:worker-poll-diagnostic:*", "polling_cache_availability_probe": "*server:polling-cache:*", "long_poll_signals": "*server:long-poll-signal:*", "workflow_task_poll_requests": "*server:workflow-task-poll-request:*", diff --git a/tests/Feature/WorkerPollDiagnosticsTest.php b/tests/Feature/WorkerPollDiagnosticsTest.php new file mode 100644 index 000000000..468eb7408 --- /dev/null +++ b/tests/Feature/WorkerPollDiagnosticsTest.php @@ -0,0 +1,91 @@ + 'array']); + } + + public function test_recent_cutoffs_are_isolated_and_overwrite_only_their_queue_observation(): void + { + $diagnostics = app(WorkerPollDiagnostics::class); + $diagnostics->recordDisconnect('tenant-a', 'queue', 'workflow-tasks', 1.2345, 30); + $diagnostics->recordDisconnect('tenant-b', 'queue', 'query-tasks', 3, 45); + $diagnostics->recordDisconnect('tenant-a', 'queue', 'activity-tasks', 2.3456, 60); + + $current = $diagnostics->snapshot('tenant-a', 'queue'); + $this->assertSame('warning', $current['status']); + $this->assertSame('activity-tasks', $current['last_disconnect']['poll_kind']); + $this->assertSame(2.346, $current['last_disconnect']['observed_seconds']); + $this->assertSame(60, $current['last_disconnect']['configured_wait_seconds']); + $this->assertSame('query-tasks', $diagnostics->snapshot('tenant-b', 'queue')['last_disconnect']['poll_kind']); + $this->assertSame('no_recent_observation', $diagnostics->snapshot('tenant-a', 'other')['status']); + } + + public function test_observations_expire_and_full_waits_are_not_mislabeled_as_early_cutoffs(): void + { + $diagnostics = app(WorkerPollDiagnostics::class); + $diagnostics->recordDisconnect('default', 'queue', 'workflow-tasks', 30, 30); + $this->assertSame('no_recent_observation', $diagnostics->snapshot('default', 'queue')['status']); + $diagnostics->recordDisconnect('default', 'queue', 'workflow-tasks', 2, 30); + $this->travel(WorkerPollDiagnostics::RETENTION_SECONDS + 1)->seconds(); + $this->assertSame('no_recent_observation', $diagnostics->snapshot('default', 'queue')['status']); + $this->travelBack(); + } + + public function test_store_failure_reports_unavailable_and_does_not_escape_disconnect_cleanup(): void + { + $store = Mockery::mock(Repository::class); + $store->shouldReceive('get')->with('server:polling-cache:availability-probe')->andReturn(null); + $store->shouldReceive('get')->with(Mockery::pattern('/^server:worker-poll-diagnostic:/'))->andThrow(new RuntimeException('fixture outage')); + $store->shouldReceive('put')->andThrow(new RuntimeException('fixture outage')); + $factory = Mockery::mock(Factory::class); + $factory->shouldReceive('store')->andReturn($store); + $diagnostics = new WorkerPollDiagnostics(new ServerPollingCache($factory, new Filesystem)); + + $diagnostics->recordDisconnect('default', 'queue', 'workflow-tasks', 2, 30); + $result = $diagnostics->snapshot('default', 'queue'); + $this->assertSame('unavailable', $result['status']); + $this->assertNull($result['last_disconnect']); + } + + public function test_queue_detail_and_summary_expose_the_actual_cutoff_warning(): void + { + WorkflowNamespace::query()->updateOrCreate(['name' => 'default'], [ + 'description' => 'Default namespace', 'retention_days' => 30, 'status' => 'active', + ]); + WorkerRegistration::query()->create([ + 'namespace' => 'default', 'worker_id' => 'fixture-cutoff-worker', 'task_queue' => 'fixture-cutoff-queue', + 'runtime' => 'php', 'last_heartbeat_at' => now(), 'status' => 'active', + ]); + app(WorkerPollDiagnostics::class)->recordDisconnect('default', 'fixture-cutoff-queue', 'workflow-tasks', 2.125, 30); + $headers = ['X-Namespace' => 'default', 'X-Durable-Workflow-Control-Plane-Version' => '2']; + $this->withHeaders($headers)->getJson('/api/task-queues/fixture-cutoff-queue') + ->assertOk()->assertJsonPath('repair.needs_attention', true) + ->assertJsonPath('poll_transport.status', 'warning') + ->assertJsonPath('repair.transport_warnings.0.reason', 'worker_poll_client_disconnected') + ->assertJsonPath('repair.transport_warnings.0.observed_seconds', 2.125) + ->assertJsonPath('repair.transport_warnings.0.configured_wait_seconds', 30); + $this->withHeaders($headers)->getJson('/api/task-queues') + ->assertOk()->assertJsonPath('task_queues.0.repair.needs_attention', true) + ->assertJsonPath('task_queues.0.poll_transport.last_disconnect.configured_wait_seconds', 30); + } +} diff --git a/tests/Feature/WorkerPollResponseProgressTest.php b/tests/Feature/WorkerPollResponseProgressTest.php new file mode 100644 index 000000000..3a69982d8 --- /dev/null +++ b/tests/Feature/WorkerPollResponseProgressTest.php @@ -0,0 +1,216 @@ + 'array', 'server.polling.max_concurrent_waits' => 1]); + } + + public function test_legacy_and_declined_transports_preserve_the_original_response(): void + { + foreach (['application/json', LongPollResponseProgress::MEDIA_TYPE.';q=0', + LongPollResponseProgress::MEDIA_TYPE.';q=.5, application/json;q=1'] as $accept) { + $original = new JsonResponse(['reason' => 'fixture'], 422); + $response = $this->throughTransport(static fn () => $original, $accept); + $this->assertSame($original, $response); + } + } + + #[DataProvider('pollKinds')] + public function test_negotiation_preserves_early_status_and_raw_json_body(string $kind): void + { + $raw = '{"task":null,"decimal":1.0,"large":9223372036854775808}'; + $original = JsonResponse::fromJsonString($raw, 429, ['Retry-After' => '2']); + $response = $this->throughTransport(static fn () => $original, strtoupper(LongPollResponseProgress::MEDIA_TYPE).', application/json;q=0.9', '/api/worker/'.$kind.'-tasks/poll'); + $this->assertInstanceOf(StreamedResponse::class, $response); + $this->assertSame(429, $response->getStatusCode()); + $wire = TestResponse::fromBaseResponse($response)->streamedContent(); + $this->assertStringContainsString('"body":'.$raw, $wire); + $frame = json_decode($wire, true, flags: JSON_THROW_ON_ERROR); + $this->assertSame(LongPollResponseProgress::SCHEMA, $frame['schema']); + $this->assertSame(429, $frame['status']); + $this->assertSame('2', $frame['headers']['retry-after']); + } + + public static function pollKinds(): array + { + return ['workflow' => ['workflow'], 'activity' => ['activity'], 'query' => ['query'], 'update-validation' => ['update-validation']]; + } + + public function test_a_real_wait_suspends_and_releases_its_slot_before_successful_delivery(): void + { + $poller = $this->poller(static fn () => usleep(2000)); + $probes = 0; + $response = $this->throughTransport(function () use ($poller, &$probes): JsonResponse { + $task = $poller->until( + static function () use (&$probes): ?string { + return ++$probes > 1 ? 'fixture-task' : null; + }, + static fn (?string $value): bool => $value !== null, + timeoutSeconds: 1, + intervalMilliseconds: 1, + reserveWorkerWaitSlot: true, + waitSlotNamespace: 'fixture', + ); + + return new JsonResponse(['task' => $task]); + }); + $slots = app(LongPollWaitSlotStore::class); + $this->assertSame(1, $probes); + $this->assertNull($slots->tryAcquire(1, 'other')); + $frame = json_decode(TestResponse::fromBaseResponse($response)->streamedContent(), true, flags: JSON_THROW_ON_ERROR); + $this->assertSame(200, $frame['status']); + $this->assertSame('fixture-task', $frame['body']['task']); + $this->assertSame(2, $probes); + $slot = $slots->tryAcquire(1, 'other'); + $this->assertNotNull($slot); + $slot->release(); + } + + #[DataProvider('lateErrorStatuses')] + public function test_late_framework_errors_preserve_status_retry_header_and_resource_cleanup(int $status): void + { + $poller = $this->poller(static function () use ($status): void { + throw new HttpException($status, 'fixture late failure', headers: ['Retry-After' => '3']); + }); + $response = $this->throughTransport(static function () use ($poller): JsonResponse { + $poller->until(static fn () => null, static fn () => false, timeoutSeconds: 1, reserveWorkerWaitSlot: true); + + return new JsonResponse(['unreachable' => true]); + }); + $this->assertSame(200, $response->getStatusCode()); + $frame = json_decode(TestResponse::fromBaseResponse($response)->streamedContent(), true, flags: JSON_THROW_ON_ERROR); + $this->assertSame($status, $frame['status']); + $this->assertSame('3', $frame['headers']['retry-after']); + $this->assertIsArray($frame['body']); + $this->assertNotSame('worker_poll_transport_unavailable', $frame['body']['reason'] ?? null); + $slot = app(LongPollWaitSlotStore::class)->tryAcquire(1, 'other'); + $this->assertNotNull($slot); + $slot->release(); + } + + public static function lateErrorStatuses(): array + { + return ['conflict' => [409], 'unavailable' => [503], 'internal' => [500]]; + } + + public function test_disconnect_unwinds_the_exact_wait_and_records_observed_and_configured_time(): void + { + $this->simulateDisconnect(); + Log::shouldReceive('warning')->once()->andReturnUsing(static function (): void { + $slot = app(LongPollWaitSlotStore::class)->tryAcquire(1, 'diagnostic-observer'); + self::assertNotNull($slot, 'Diagnostic I/O must run after the abandoned wait releases its slot.'); + $slot->release(); + }); + $poller = $this->poller(static function (): void { + self::fail('A disconnected wait must unwind before the next probe.'); + }); + $response = $this->throughTransport(static function () use ($poller): JsonResponse { + $poller->until(static fn () => null, static fn () => false, timeoutSeconds: 30, reserveWorkerWaitSlot: true); + + return new JsonResponse(['unreachable' => true]); + }); + $wire = TestResponse::fromBaseResponse($response)->streamedContent(); + $this->assertSame('', trim($wire)); + $slot = app(LongPollWaitSlotStore::class)->tryAcquire(1, 'other'); + $this->assertNotNull($slot); + $slot->release(); + Log::shouldHaveReceived('warning')->once()->withArgs(static fn (string $message, array $context): bool => str_contains($message, 'disconnected') && $context['configured_wait_seconds'] === 30 + && $context['observed_seconds'] < 1 && $context['namespace'] === 'fixture'); + } + + #[DataProvider('pendingPollStores')] + public function test_disconnected_duplicate_wait_does_not_release_the_live_leaders_pending_key(string $storeClass): void + { + $this->simulateDisconnect(); + $store = app($storeClass); + $arguments = ['fixture', 'fixture-queue', null, 'fixture-worker', 'same-poll']; + $this->assertTrue($store->tryStart(...$arguments)); + $response = $this->throughTransport(static function () use ($store, $arguments): JsonResponse { + $store->waitForResult(...$arguments); + + return new JsonResponse(['unreachable' => true]); + }); + $this->assertSame('', trim(TestResponse::fromBaseResponse($response)->streamedContent())); + $this->assertFalse($store->tryStart(...$arguments)); + $store->forgetPending(...$arguments); + $this->assertTrue($store->tryStart(...$arguments)); + $store->forgetPending(...$arguments); + } + + public static function pendingPollStores(): array + { + return [ + 'workflow' => [WorkflowTaskPollRequestStore::class], + 'activity' => [ActivityTaskPollRequestStore::class], + 'query' => [QueryTaskPollRequestStore::class], + ]; + } + + private function simulateDisconnect(): void + { + $this->app->bind(LongPollResponseProgress::class, static function ($app, array $parameters) { + return new class($parameters['request']) extends LongPollResponseProgress + { + protected function clientDisconnected(): bool + { + return true; + } + }; + }); + } + + private function throughTransport(Closure $handler, string $accept = LongPollResponseProgress::MEDIA_TYPE, string $path = '/api/worker/workflow-tasks/poll'): Response + { + $request = Request::create($path, 'POST', ['task_queue' => 'fixture-queue', 'timeout_seconds' => 30]); + $request->headers->set('Accept', $accept); + $request->attributes->set('namespace', 'fixture'); + $this->app->instance('request', $request); + + return (new Pipeline($this->app))->send($request) + ->through([WorkerPollResponseProgress::class])->then($handler); + } + + private function poller(Closure $pause): LongPoller + { + return new class(app(LongPollSignalStore::class), app(LongPollWaitSlotStore::class), $pause) extends LongPoller + { + public function __construct($signals, $slots, private readonly Closure $pause) + { + parent::__construct($signals, $slots); + } + + protected function pause(int $milliseconds): void + { + ($this->pause)(); + } + }; + } +}