diff --git a/CHANGELOG.md b/CHANGELOG.md index 330c5f0d..86f4f45e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,184 @@ SPDX-License-Identifier: Apache-2.0 ## Unreleased +- Serve bounded run summaries and classified step timelines with fresh scoped + authorization. Timeline output remains optional; legacy timings stay unknown. + Logs remain unavailable with HTTP 501. +- Change `runs.list` to newest-started-first order. Existing v1 cursors remain + supported with explicit `order=id`; chronological orders use v2 cursors. + Update-time pagination is a live view and may require a refresh after changes. + Restored executions reappear in every order. +- `runs.list` accepts the run summary filters and leaves out test runs unless + `include_test=true`. It rejects unknown query parameters, and an out-of-range + `limit` answers `WV-VALIDATION` instead of `WV-PAGE`. Pages stay limited to + 8 MiB of retained run data. +- Step timeline outputs are limited to 8 MiB per page (`429 WV-PAGE-LIMIT`). + Outputs of agent, tool or yield instance keys stay withheld until they have + their own schemas. A run waiting for a platform that runs it answers + `422 WV-IR-UNSUPPORTED` as `runs.read` does. +- The scheduler re-checks stored run listing decisions in the background: runs + recorded while this platform could not run them, which it now runs, and + available or unsupported decisions recorded under an earlier classification + policy. Unavailable decisions stay unavailable. Each minute it visits up to 16 + tenants and gives one environment of each up to one second of database work + and at most 100 runs. Until a run is re-checked, run summaries, step timelines + and incident lists keep withholding it, even when `runs.read` serves it again: + its timeline answers `422 WV-IR-UNSUPPORTED` when this platform runs it again, + and `409 WV-LEGACY-UNAVAILABLE` when it was recorded under an earlier policy. + A retained run larger than 128 MiB is recorded as unavailable without being + read. +- The Python SDK's `list_runs` rejects a `query` combined with keyword filters. +- Request telemetry records authorized `501 WV-UNAVAILABLE` answers with the + status `unavailable` instead of `failed`. +- A new run's `started` history event records how the run started as `origin`, + the value run summaries report. History pages and exports include it, and + replay accepts it. +- Serve incident lists from incident facts without loading runs, newest opened + first. They filter by status, code, workflow, step, opening time, and test + runs (left out unless `include_test=true`). Explicit `order=id` continues + earlier cursors. Unknown query parameters are rejected, and an out-of-range + `limit` answers `WV-VALIDATION` instead of `WV-PAGE`. +- New operation `incidents.read` (`weave runs incidents read`, SDK + `read_incident`) serves direct incident links and sends the revision as `ETag`. +- Incident views add optional `opened_at`, `opened_at_estimated`, `updated_at`, + `workflow`, `run_status`, `test`, and `instance_key`; for a repeated step + instance, `node_id` is the static step ID. Upgrade CLIs and SDKs with the + server: earlier releases reject the new fields and fail with `502 WV-WIRE` on + incident lists. Resolution receipts, `incidents.resolve` answers, and the + earlier plain array of a run's incidents keep their bytes. A resolve answer is + the incident as stored, so read it again with `incidents.read`. +- An incident on a switch or parallel branch output or a join keeps its + control-step `node_id` in incident lists and reads. An incident whose step + does not match its incident facts is withheld as unavailable. +- The Python SDK's `list_incidents` and `run_incidents` accept a typed `query` + and reject one combined with keyword arguments. +- Workers report where they run when they register: an optional, bounded + `identity` with host, runtime, versions, platform, Kubernetes namespace and + node, and up to 8 labels from `WEAVE_WORKER_LABELS` + (`WEAVE_WORKER_REPORT_HOST=false` leaves out the host). The Agentic and Files + workers and the SDK's `register_worker` send it and retry once without it when + an older server answers `422 WV-VALIDATION`; native executors send it inside + the API process. With native executors configured, an invalid + `WEAVE_WORKER_LABELS` stops the API from starting. New SDK helpers: + `worker_identity` and `register_worker_http`. +- Worker reads, lists, and drain or resume answers add optional `identity` and + `registered_at`. Upgrade CLIs and SDKs with the server: earlier releases fail + with `502 WV-WIRE` on workers registered after the upgrade. The registration + answer and stored worker records keep their bytes, and a worker cannot report + a placement. +- Deployment observations list up to 100 live `members` of each worker + component: running or starting Compose containers and Kubernetes pods, with + host, ordinal, state, readiness, start time, restarts and node. Stopped, + terminating, finished, failed and evicted ones are left out. With more than 100 + live members, or when the runner cannot list them all, `members` is left out + (placement unknown) and the observation still succeeds. Kubernetes runners + read members through the deployment's selector, including set-based + `matchExpressions`, and need `list` on the namespace's pods. Observations + without members keep their bytes and digests. +- Upgrade the server before its deployment runners: an older server refuses the + `members` an upgraded runner reports, so observe jobs fail and applies end + `reconciliation_required`. Earlier CLIs and SDKs fail with `502 WV-WIRE` on + observation reads and lists that carry members. +- `workers.list` reads each page with one query and lists current workers by + default: not revoked, and seen (or, never seen, registered) in the last hour; + `state=all` lists every worker. It filters by `presence`, `draining`, + `release_id`, `task_type`, `target_id` and `host_prefix` (a literal, + case-sensitive prefix) and orders by `last_seen_desc` (default) or + `registered_desc`, unknown times last. Cursors issued before the upgrade are + refused, unknown query parameters are rejected, and an out-of-range `limit` + answers `WV-VALIDATION` instead of `WV-PAGE`. Pages stay limited to 8 MiB. +- Studio's Workers page asks the platform for current workers, last contact + first, and for every worker with **Show retired workers**. Earlier Studio + hosts and desktop apps get the + reduced default list from an upgraded server, because the Studio bridge + forwards answers unchanged: their Workers page lists only current workers, and + **Show revoked workers** shows none. Upgrade Studio and the desktop app with + the server. +- Worker reads and lists add optional `placement` (target, deployment and + component), matched from the latest complete, unexpired observation of the + targets the reader may read, and `drained_at`, when the worker started + draining; drain and resume answers add `drained_at`. Upgrade CLIs and SDKs + with the server: earlier releases fail with `502 WV-WIRE` on placed workers + and on workers drained since the upgrade. +- New operation `workers.retire_stale` (`weave workers retire-stale`, SDK + `retire_stale_workers`) revokes up to 100 workers per command that hold no + active lease and were last seen, or registered if never seen, longer ago than + `older_than_seconds` (one hour to 30 days), oldest first. `dry_run` previews + the same batch, and `matched` counts every eligible worker. +- New operation `releases.usage` (`weave workers releases usage`, SDK + `release_usage`) counts a release's online, offline, draining and revoked + workers and the activations that pin it, and pages the deployment components + that run it on targets the reader may read; `deployments_hidden: true` says the + reader may read no target. +- The Python SDK's `list_workers` accepts a typed `query` and rejects one + combined with keyword arguments. +- Kubernetes runners no longer read a pod's free-text status reason, and a pod + whose readiness is unknown is listed as a member that is not ready. +- New operation `operations_summary.read` (`weave operations summary`) answers + an environment's health, load, workers, targets, incidents and up to 20 items + that need attention, in one read that reloads the caller's grants. Each + section and item needs its own capability; one the caller may not read is + `{"state": "forbidden"}` without running its queries or recording a denial, + and one whose queries use up its two seconds of database time between them + is `{"state": "unavailable"}` while the rest are still served. Run figures + leave out test runs, and opening times the upgrade estimated are not counted. + Telemetry export health is degraded while exports are being dropped. +- New operation `operations_telemetry.read` (`weave operations telemetry`) + answers whether telemetry export is on, the collector host names, the + metrics interval and how many items were dropped, never endpoint paths, + credentials or headers. It needs `status.read` for the project. + `WEAVE_TELEMETRY` now refuses a collector host name longer than 253 + characters. +- Background loop health, which the summary reports, ages successes with a + monotonic clock, counts a cycle still within its loop's budget as on time, + and reads degraded after 3 failed cycles in a row or among the last 20. An + outbox cycle that cannot start a delivery, because the platform is not ready + or no delivery slot is free, is neither a success nor a failure. +- Studio's host bridge forwards the environment summary, the telemetry + settings and `releases.usage`. +- Studio's Operate area opens on a new Overview page. It shows the health of + the API, the database, the scheduler, the outbox, compatibility and + telemetry export, each with its details; the current load; workers, + clusters and incidents; and up to 20 items that need attention, most of + them linking to their record. It reads the summary every 15 seconds while + visible. A section you may not read names the permission it needs instead + of showing zeros. +- Operate › Runs reads run summaries, newest first, with time range, workflow, + version, origin, status, test, handled-error, incident, top-level and + archive filters. The address keeps every filter except the business and + correlation keys. Load more continues the same list, Auto-refresh reads the + first page every 10 seconds, and up to 50 selected runs can be canceled or + retried together. A run's detail draws its steps with queue wait and run + time, groups loop iterations, links to called runs, shows a step's output + on request and follows a running run every 2 seconds. Starting, canceling, + retrying, archiving or restoring a run, or deleting its data, from the page + reads the list again at once, and a run started there shows first while + the list puts new runs first (with Oldest first it waits for its place). + The line under the title names the order in use. +- Operate › Incidents filters on the platform by status, code, workflow, + step, opening range and test runs, with a choice of order, so a match + beyond the loaded pages shows. The address keeps the filters and the open + incident (`?incident=ID`), which opens from `incidents.read` even when no + loaded page lists it. The drawer shows the workflow, the full step instance, + the run's status and when the incident opened, marked estimated when the + upgrade inferred it and Unknown when it isn't known; Refresh incident reads + the incident itself. +- Operate › Workers has Fleet and Releases tabs. Fleet filters on the platform + by status, draining, release, task type, target and host prefix (kept on the + page, never in the address) with a choice of order, shows each worker's + host, placement, versions and registration time, drains, resumes or revokes + up to 50 selected workers one by one, and **Retire offline workers…** + previews and then revokes offline workers that hold no work, 100 at a time. + Releases lists the admitted releases and, for one, its workers by state, the + activations that pin it and the deployments on targets you may read; it shows + the CLI command that admits a release and never admits one itself. On a narrow + screen both pages fold their filters behind **Filters (n)**, and every filter + checkbox is a 36 px target with a mouse and 44 px on touch. + A retirement whose answer arrives after its dialog closed, or after the + workspace or person changed, says what happened in a notice; a bulk action + stops sending when the person changes, and the host prefix is cleared for the + next person. + - The compiler checks and compiles **Call a workflow** (`callWorkflow`) against the exact published version it calls: that version must opt in with `spec.callable` (any workflow, or only `allowedCallers`), `with` is checked diff --git a/docs/guides/execution-management.md b/docs/guides/execution-management.md index 6995287a..08b2d68c 100644 --- a/docs/guides/execution-management.md +++ b/docs/guides/execution-management.md @@ -100,12 +100,17 @@ start, the dialog stays open and says "The run didn't start." with the reason. **Find runs later** with the filters above the list in **Runs**. The list updates as soon as you change a filter; there is no button to apply them. -- Type an exact business key in **Search by business key**. -- Choose a status: **All**, **Waiting**, **Failed**, or **Succeeded**. +- Type an exact business key in **Business key**. +- Choose one or more statuses, such as **Waiting** and **Failed**; choose a + status again to remove it. +- Choose a **Time range**: the last 24 hours by default, a longer or shorter + one, or a custom range. - Open **More filters** to enter an exact **Correlation key** or to check **Include archived runs**. -To remove a filter, empty its field, choose **All**, or clear the check box. +To remove a filter, empty its field, choose its status again, or clear the +check box; **Clear filters** removes them all. Business and correlation keys +never appear in the page's address. ### With the CLI @@ -179,10 +184,11 @@ explicitly which run should receive the event. Open a run in Studio's **Runs** view. Its detail shows, from top to bottom: the status, business key, and run ID (**Copy** copies the full ID); a **Now** line that says what the run is doing; its commands; the graph of the workflow -version it runs, with **Now** on the current step; a **Timeline** of its -recorded events; **Technical details**; and **Archive**. The detail does not -update by itself: select **Refresh** at the bottom of the detail to read the -run again. +version it runs, with **Now** on the current step; its **Steps**, each with the +time it waited and the time it ran; its recorded **Events**; **Technical +details**; and **Archive**. While the run is still going, Studio reads it again +every 2 seconds; select **Refresh** at the bottom of the detail to read it at +once. The **Now** line, or the card that replaces it, tells you what the run waits for. STEP stands for the step's ID: diff --git a/docs/guides/studio.md b/docs/guides/studio.md index e56eae48..bd879891 100644 --- a/docs/guides/studio.md +++ b/docs/guides/studio.md @@ -275,7 +275,8 @@ Open the assistant in any of these ways: - While you work locally, select **Connect to a platform** on **Home**, in the platform menu at the top right, in the designer next to "Saving, publishing and runs need a platform.", or on the **Connections**, **My tasks**, - **Email**, **Runs**, **Incidents**, **Workers**, and **Clusters** pages. + **Email**, **Overview**, **Runs**, **Incidents**, **Workers**, and + **Clusters** pages. - In **Settings → Platforms**, select **Add platform** while you use a platform, or **Connect to a platform** while you work locally. @@ -1518,27 +1519,94 @@ Every start creates a new run, even with the same business key; a business key groups related runs. Studio remembers the last input for each version in this browser, leaving out fields the schema marks as secret. -**Follow and manage runs in Runs.** - -- **Find runs** with the filters above the list: type an exact business key in - **Search by business key**, choose **All**, **Waiting**, **Failed**, or - **Succeeded**, or open **More filters** for an exact **Correlation key** and - **Include archived runs**. The list updates as you change a filter. In a - narrow window, the filters fold behind a button such as "Filters (2)". +**See the platform at a glance in Overview.** **Operate** opens on +**Overview**, which needs `status.read`. It reads the environment's summary +every 15 seconds while the page is visible, and after a failed read it waits +longer, up to a minute. + +- **Health** has one tile each for the API, the database, the scheduler, the + outbox, compatibility and telemetry export, with a state (**OK**, + **Degraded**, **Down** or **Off**) and one figure such as "Latency 4 ms" or + "Last success 2 min ago". Each tile's details button lists its facts or its + background loops, with each loop's last success and failures in a row. +- **Load** shows active runs (test runs left out), ready tasks with the oldest + wait, waits with how many are overdue, the outbox backlog and the provider + backlog. +- **Workers and clusters** counts online, offline and draining workers, free + task slots against ready tasks, runners, busy targets, targets that need + reconciliation and plans waiting for approval, and active incidents. +- **Needs attention** lists up to 20 items, most urgent first: jobs that need + reconciliation, plans you may approve, offline workers that still hold work, + active incidents, targets without an online runner, a restricted + compatibility verdict and runs that failed in the last hour. A job, plan, + worker or failed run opens its record; an incident opens **Incidents**, a + target without a runner opens **Runners**, and the compatibility verdict has + no link. + +A section you may not read says which permission it needs, such as "You need +`incident.read` in this environment.", instead of showing zeros; a section the +platform could not compute in time says "This section could not be loaded." +with **Try again**, while the rest of the page stays usable. + +**Follow and manage runs in Runs.** The list shows the newest runs first +(choose another **Order** under **More filters**; the line under the title +says which order is in use): each run's status (a run that succeeded after handling errors reads "Succeeded +with errors"), its workflow and version, how it started (**Retry of** and +**Called by** open the run it came from), when it started, how long it ran +(counting up while it runs), its active incidents, its business key and its +short run ID with **Copy**. On a wide screen each run is one line: a long +workflow name or business key ends in "…", and pointing at it shows the whole +value, as the run's detail does. + +- **Find runs** with the filters above the list: **Time range** (the last 15 + minutes, hour, 24 hours, 7 days or 30 days, or a custom range; the last 24 + hours by default), **Workflow** and **Version**, **Origin**, an exact + **Business key**, one or more statuses, and **Include test runs**. **More + filters** adds an exact **Correlation key**, **Handled errors**, **Order**, + **Has active incident**, **Top-level runs only** and **Include archived + runs**. The list updates as you change a filter, and **Clear filters** resets + them all. The address keeps every filter except the business and correlation + keys, so you can share a view without sharing business data. In a narrow + window, the filters fold behind a button such as "Filters (2)". +- **Keep the list current.** With **Auto-refresh** on (the default), the list + reads its first page again every 10 seconds while the page is visible; a + relative time range then ends at that moment. **Refresh** does the same at + once. **Load more runs** continues exactly the list you are reading, so pages + never shift under you; while more than the first page is open, or runs are + selected, the list waits for **Refresh**. Starting, canceling, retrying, + archiving or restoring a run, or deleting its data, from this page reads the + first page again at once. While the list shows new runs first (**Newest + first** or **Recently updated**), a run you just started shows first, as + "Just started", until the list includes it; with **Oldest first** it + appears in its place once a page reaches it. +- **Act on several runs.** Select runs, up to 50, and choose **Cancel runs** + (one reason for all) or **Retry runs** (each on its own version with its own + input). Studio sends them one at a time and reports the result for each run. - **Open a run** to watch it. The **Now** line says what it is doing, such as "Waiting for a person at review, assigned to reviewers", and offers **Open task** or **Open My tasks** when a person must act ([what each reason means](execution-management.md#3-understand-why-a-run-is-waiting)). The graph of the version it runs marks the current step **Now** and finished - steps with a check. **Timeline** lists the recorded events in plain sentences, - each with **View raw event**, and **Technical details** holds the run ID, the - correlation key, the artifact, and the run data. The view does not update by - itself: select **Refresh** at the bottom of the run's detail. -- **Pause run** and **Resume run** need their own permissions, ask for a - **Reason**, and appear only while the run hasn't finished; only one of them - shows at a time. Studio has no controls to cancel or retry a run or to resolve - an incident; a blocked run shows "This run stopped at" and the step. Use the - CLI or the API, as in [incident operations](../reference/incident-operations.md). + steps with a check. **Steps** draws one bar per step: the time it waited in + the queue and the time it ran. Loop iterations sit under their step, and a + step that called a workflow opens the run it called. **Input/Output** shows a + step's recorded output only when you select **Show output**, and says why + when an output can't be shown. **Logs** aren't available yet. **Events** + lists the recorded events in plain sentences, each with **View raw event**, + and **Technical details** holds the run ID, the correlation key, the + artifact, and the run data. While the run is still going, Studio reads it and + its steps every 2 seconds; **Refresh** at the bottom of the detail reads it at + once. Each of those reads takes every page of steps on screen again: after + **Load more steps**, which stops at 10 pages (2,000 step instances) and says + so, a running run costs up to 10 step requests every 2 seconds, one after + another, and a slow platform slows the reads down instead of overlapping them. +- **Cancel run**, **Retry run**, **Send signal** and **Export history** work as + [Manage runs and business cases](execution-management.md#3-understand-why-a-run-is-waiting) + describes. **Pause run** and **Resume run** need their own permissions, ask + for a **Reason**, and appear only while the run hasn't finished; only one of + them shows at a time. A blocked run shows "This run stopped at" and the step; + resolve its incident in **Incidents**, or as in + [incident operations](../reference/incident-operations.md). - **Archive run**, under **Archive**, hides a finished run from the list and keeps its history. Open **More filters** and check **Include archived runs** to show it again; **Restore run** returns it to the list. @@ -1676,8 +1744,8 @@ security policy, start preferences, and local validation when the repository Python `.venv` is available. Connected API fixtures test request shapes and failure handling; they are separate from database/mail transport integration evidence. `tests/browser/real-platform.spec.ts` drives the connection, sign-in, -session expiry and revocation, switch-account, and quick-integration journeys -against a running local platform and its Keycloak, with the system credential +session expiry and revocation, switch-account, quick-integration and Runs +refresh journeys against a running local platform and its Keycloak, with the system credential store; it is skipped unless `WEAVE_E2E_PLATFORM_DIR` and `WEAVE_E2E_PERSON_FILE` are set, as its header explains. Its quick-integration test also needs `WEAVE_E2E_ACME_ORIGIN`, the Acme fixture origin the platform diff --git a/docs/guides/workers.md b/docs/guides/workers.md index 5525eace..3b0e5ef9 100644 --- a/docs/guides/workers.md +++ b/docs/guides/workers.md @@ -230,14 +230,60 @@ and stops claiming early enough to drain its declared 180-second tasks; a supervisor can restart it with fresh credentials. It is not an indefinitely refreshing token client. +### Report where a worker runs + +A worker can describe where it runs when it registers. Operators then see the +host, runtime, versions and labels on the worker, and the server can match the +host to a container or pod of an observed deployment. The identity is display +metadata: it never grants a worker more authority, and a worker cannot report +its own placement. + +The packaged Agentic and Files workers and the API's native executors send it +automatically. A worker you write sends it by registering with +`register_worker_http(client, environment_url, body)`, with +`"identity": worker_identity().model_dump(mode="json")` in the body, or with +`WeaveClient.register_worker`. Both retry once without the identity when an +older server refuses it. The checked-in example registers without one. + +| Variable | Effect | +| --- | --- | +| `WEAVE_WORKER_REPORT_HOST` | `false` leaves out the hostname (the pod name in Kubernetes); default `true` | +| `WEAVE_WORKER_LABELS` | Up to 8 `key=value` pairs separated by commas, for example `team=payments,tier=gold` | +| `WEAVE_POD_NAMESPACE`, `WEAVE_NODE_NAME` | The pod's namespace and node, set from the Kubernetes downward API | +| `KUBERNETES_SERVICE_HOST` | Set by Kubernetes; the worker then reports a `container` runtime and the `kubernetes` platform | + +Labels are visible to everyone who can read workers. Never put credentials in +them: keys that name a token, password, secret or authorization are refused, +and an invalid label stops the worker before it registers, with a message that +does not repeat the value. On the API host, with native executors configured, +an invalid label stops the API from starting. An invalid hostname or platform value is left out +instead. + ## 4. Observe one task through completion Start a workflow that calls the published Action. Inspect its run history and -worker status through the API. In Studio, **Operate › Workers** lists the -workers in the selected environment with their status (**Online**, -**Offline** or **Not seen yet**, and **Draining**), their load against their -task slots, and their last contact. **Show revoked workers** adds the -**Revoked** ones. Open a worker to **Drain** or **Resume** it. Use the +worker status through the API. In Studio, **Operate › Workers** has two tabs. +**Fleet** lists the current workers in the selected environment (not revoked, +and seen, or registered, in the last hour), last contact first, with their +status (**Online**, **Offline** or **Not seen yet**, and **Draining**), the +host they report, where they run (a target and component, **In API process**, +**Unmanaged**, or **Unknown** when a worker reports nothing), their release, +SDK and runtime versions, their load against their task slots, their last +contact and when they registered. The platform applies the filters (status, +draining, release, task type, target, **Host starts with** and **Order**), so +a match beyond the first page still shows; the address keeps every filter +except the host prefix, which stays on the page. **Show retired workers** adds +revoked and long-offline workers. Select up to 50 workers to **Drain**, +**Resume** or **Revoke** them one after another, each with its own answer, or +open a worker to see what it reports, where the platform places it, and to +**Drain**, **Resume** or **Revoke** it. **Retire offline workers…** first shows +how many workers offline for 24 hours (or 7 or 30 days) hold no work, then +revokes them 100 at a time, oldest first. **Releases** lists the admitted +releases; open one to see its workers by state, the activations that pin it +and the deployments on targets you may read. Studio never admits a release: it +shows the `weave workers releases create --request release.json` command. +**Roll out to a target…** stays unavailable until change requests come to +Studio. Use the [API playground](api-playground.md) to authenticate and select your environment, then the [full API reference](../reference/api-explorer.md) for worker registration and run/history reads. A worker token is for execution; diff --git a/docs/operations/observability.md b/docs/operations/observability.md index 2192eccf..e95c148c 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -142,7 +142,7 @@ Export failure never changes workflow state or grants execution authority. | Field | Default | Rule | | --- | --- | --- | | `enabled` | `false` | JSON `true` or `false`; at least one endpoint is required when `true` | -| `traces_endpoint`, `metrics_endpoint` | Absent | Exact `http` or `https` URL, at most 2048 characters; `http` only for `localhost`, `127.0.0.1`, or `::1`; no user name, password, query, or fragment. Either may be omitted | +| `traces_endpoint`, `metrics_endpoint` | Absent | Exact `http` or `https` URL, at most 2048 characters, whose host name has at most 253 characters; `http` only for `localhost`, `127.0.0.1`, or `::1`; no user name, password, query, or fragment. Either may be omitted | | `authorization` | Absent | The complete `Authorization` header value the collector expects, 1 to 4096 printable ASCII characters | | `timeout_seconds` | `5` | Greater than 0 and at most 5 | | `metrics_interval_seconds` | `30` | From 5 to 300 | @@ -228,7 +228,7 @@ volume needs to be deleted. | What you see | Why | What to do | | --- | --- | --- | | No export traffic at all | The API process never received `WEAVE_TELEMETRY` with `"enabled": true`, or it runs under `weave platform start` | Restart or redeploy the API with the explicit value; `OTEL_*` variables are not enough | -| The API stops with `Invalid WEAVE_TELEMETRY configuration` | The JSON is malformed, has an unknown field, an `http` URL to a non-loopback host, a URL with credentials or a query, or a value out of range | Compare each field with the table above | +| The API stops with `Invalid WEAVE_TELEMETRY configuration` | The JSON is malformed, has an unknown field, an `http` URL to a non-loopback host, a URL with credentials or a query, a host name longer than 253 characters, or a value out of range | Compare each field with the table above | | Connection refused | The collector is not listening where the API looks | A loopback URL works only for an API on the same host; use a reachable trusted URL for a container API | | The collector rejects requests | Receiver type, exact `/v1/traces` or `/v1/metrics` path, TLS, or authorization | Correct the receiver or the configuration; never put credentials in the URL | | Metrics look doubled | Several replicas report the same global inventory | Use max for duplicate inventory observations and check freshness | @@ -242,6 +242,46 @@ payloads to spans to diagnose a credentialed connection. Generic PyFly request logging, global tracing filters, and generic server metrics stay disabled in Weave's packaged configuration so they cannot bypass the attribute policy. +## Operational reads at scale + +Studio's Overview, Runs, Incidents and Workers pages read the platform summary +and the operational lists, not run payloads. The server targets these p95 +latencies for a single read, measured on PostgreSQL: + +| Operation | Page | Target | +| --- | --- | --- | +| `run_summaries.list`, with any single filter | 50 runs | 120 ms | +| `runs.steps`, without outputs | 200 steps | 120 ms | +| `operations_summary.read` | one environment | 200 ms | +| `workers.list` | 100 workers | 100 ms | +| `incidents.list` | 50 incidents | 100 ms | + +`tests/benchmarks/test_operations_reads.py` checks them against one environment +holding 1,000,000 runs, 50,000 incidents, 500 workers, 10,000 ready tasks and +1,000 stored deployment observations of about 250 KB. That is far more than a +project retains under the default operation policy, which the benchmark raises +for its own database only. On an Apple silicon development machine with +PostgreSQL 17 in a 4-CPU, 8 GiB virtual machine and default PostgreSQL memory +settings, the p95 of each read was below 60 ms (the summary was the slowest, at +about 52 ms). These are measurements of that environment, not a guarantee for +yours: rehearse with your own data and hardware before relying on them. + +`tests/integration/test_operations_query_plans.py` checks the plans behind +these reads on 200,000 runs: every chronological page walks a scoped index +without sorting the environment, and no listing statement reads run payloads, +events or step outputs. Studio's real-platform suite checks that Overview shows +live figures within one second of opening it once Home has finished its own reads, +and that every scheduled refresh of Overview, Runs, Workers and Incidents transfers +less than 50,000 bytes. Opened while Home was still reading, Overview took about two +seconds in the same run (1.9 s), because both pages' reads share the Studio host; +that case is recorded, not held to the one-second target. + +Background loop health in the summary counts a loop as `degraded` after 3 +failed cycles in a row or 3 among its last 20 (see the +[API reference](../reference/api.md)). A failure that returns less often than +that, for example once every 10 cycles, leaves the loop `ok`; its +`recent_failures` still shows it. + ## Next steps - Look up the other server settings in [configuration](configuration.md). diff --git a/docs/operations/operations-runner.md b/docs/operations/operations-runner.md index 89e9a0bf..89447d53 100644 --- a/docs/operations/operations-runner.md +++ b/docs/operations/operations-runner.md @@ -57,7 +57,12 @@ substantial host authority: isolate the runner and keep its provider permissions within the destination it operates. For Kubernetes, grant only the namespace and resource permissions needed by the allowed deployments. API-component updates also require reading that namespace's horizontal pod autoscalers so the runner -can reject a conflicting scaling policy. +can reject a conflicting scaling policy. To report where worker pods run, the +runner also lists the namespace's pods (`list` on `pods`); without that read, +observations leave the worker's members out. Upgrade the API server before its +runners: an older server refuses the members a newer runner reports, so its +observe jobs fail and its applies end in reconciliation. Grant the read after +the server upgrade and before the runner upgrade. ## 2. Copy the destination's actual identity @@ -211,6 +216,13 @@ then applies only the named services and waits for container readiness. Add meaningful health checks to your Compose services; without one, a running process is the only readiness evidence Docker provides. +For a worker service, the observation also lists its live containers as +members: the container ID prefix, which is the container's default hostname, +and its Compose number. Exited, dead, and removing containers are left out. With +more than 100 live containers, or a container whose details cannot be read, the +observation leaves the members out (placement unknown) and still succeeds. A +service that sets its own `hostname` gets no placement. + ### Kubernetes The adapter manages **existing deployments**, not arbitrary resources. Each @@ -236,6 +248,20 @@ The adapter changes only the selected container's image/resources, replicas, and its ownership label. A worker scaling plan does not change the image. Objects already labeled as belonging to another Weave target are rejected. +For a worker component, the observation also lists the deployment's live pods +as members, inside the pinned namespace and through the deployment's own +selector: `matchLabels` become `key=value` and `matchExpressions` become +set-based requirements (`key in (a,b)`, `key notin (a,b)`, `key`, `!key`). The +runner reads only the few pod fields it needs. A pod counts only when its +controlling ReplicaSet is named after the deployment and its pod template hash; +terminating, finished, failed, and evicted pods are left out. A pod whose +readiness is unknown, for example on an unreachable node, is listed as not +ready. Members are placement only and never fail an observation or an apply: +with more than 100 live pods, a selector kubectl cannot express exactly, a label +that is not valid Kubernetes syntax, or a listing that fails, the observation +leaves the members out (placement unknown) instead of reporting part of them, +and it never lists the whole namespace. + ### Azure Container Apps Set `context` to the subscription UUID, `boundary` to the resource group, diff --git a/docs/operations/retention.md b/docs/operations/retention.md index 1888f170..933525ed 100644 --- a/docs/operations/retention.md +++ b/docs/operations/retention.md @@ -193,7 +193,10 @@ usage and active work. Ending a run or expiring a debug session can release acti capacity while its retained bytes stay charged. Starting a debug session also reserves its bounded session capacity. The server's admission checks release expired reservations; the data of an expired session stays until eligible -maintenance removes it. +maintenance removes it. The operational facts kept for each run, +step, task, incident and worker count toward retained usage too, and an active run +reserves room for its facts' final updates, so a project near its ceiling reaches +it a little sooner than before they existed. **Logical accounting is not disk usage.** It measures serialized data and allocation metadata. It does not reclaim PostgreSQL files, indexes, WAL, or diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 5211714e..2a254cce 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -45,7 +45,7 @@ the desktop app never migrates a platform. | alpha13 | `0030_worker_presence` | No new schema migration. Workers retry explicit capacity rejections while they read task context or credentials, and an Agentic preparation timeout before provider execution counts as not started. Upgrade the Agentic and Files workers to 0.1.5 together with the server. | | alpha14 | `0030_worker_presence` | No new schema migration. Adds the detached Docker development platform (`weave platform up`); existing foreground installations keep working and are never converted. The Agentic and Files workers 0.1.6 pin this server version. | | alpha15 | `0030_worker_presence` | No new schema migration and no role change. Upgrade CLIs and SDKs together with the server, review native executor `capacity` and the legacy private-network settings before restarting, and finish runs that use text templates before any rollback to alpha14; see [what changes in alpha15](#what-changes-in-alpha15). The Agentic and Files workers 0.1.7 pin this server version. | -| Unreleased | `0031_operations_key_collation` | One new migration that only replaces two internal accounting functions: no table, data, or role changes. Explicitly migrate the server before restarting it. `weave platform up --username` grants the local owner roles (developer, deployer, operator, viewer, task_participant, tenant_admin, lumi_manager, execution_manager, worker_operator, deployment_reader, deployment_planner, deployment_approver, deployment_operator); existing accounts keep their roles until `weave platform user grant` adds more. `weave platform user` is now a command group; `weave platform user --username` works as before. | +| Unreleased | `0032_operations_facts` | Two new migrations. `0031_operations_key_collation` only replaces two internal accounting functions: no table, data, or role changes. `0032_operations_facts` adds [scoped operational fact storage](#operational-fact-storage), a bounded historical backfill, and reserved capacity for terminal fact updates. Explicitly migrate the server before restarting it. Upgrade CLIs and SDKs together with the server, and upgrade the server before its deployment runners; see [incident list compatibility](#incident-list-compatibility), [worker identity compatibility](#worker-identity-compatibility) and [worker fleet compatibility](#worker-fleet-compatibility). `weave platform up --username` grants the local owner roles (developer, deployer, operator, viewer, task_participant, tenant_admin, lumi_manager, execution_manager, worker_operator, deployment_reader, deployment_planner, deployment_approver, deployment_operator); existing accounts keep their roles until `weave platform user grant` adds more. `weave platform user` is now a command group; `weave platform user --username` works as before. | ![Schema, compatibility and execution acceptance gates](../diagrams/operations-upgrade.svg) @@ -148,6 +148,273 @@ policy update, and this release has no in-place policy-change operation: a mismatch leaves the runtime restricted. Never rewrite the fingerprint or counters by hand to force readiness. +## Run list and timeline compatibility + +The server now serves run summaries and step timelines. `runs.list` changes its +default to `started_desc`, with `started_asc` and `updated_desc` also available. +Clients that continue a legacy v1 cursor must send `order=id` and retain its +original filters. New filters bind ID cursors to the normalized query; time +orders use v2 cursors. Restart pagination after changing filters or order. +The `weave runs list` command does not send `order=id` yet, so it cannot +continue a cursor saved before the upgrade; start such a listing again. + +`runs.list` also changes how it reads requests: + +- It accepts the run summary filters and leaves out test runs unless the + request sends `include_test=true`. +- It rejects unknown query parameters with `422 WV-VALIDATION`; the earlier + route ignored them. +- A `limit` outside 1 to 100 answers `422 WV-VALIDATION` instead of + `422 WV-PAGE`. +- An older server reads only `limit`, `cursor`, `business_key`, + `correlation_key`, a single `status` and `include_archived`. It ignores the + other filters, and it keeps only the last of several `status` values. Upgrade + servers before relying on the new filters. + +The Python SDK's `list_runs` takes either a `query` or the keyword filters; +passing both raises `ValueError` instead of silently dropping the keywords. + +Request telemetry records an authorized `501 WV-UNAVAILABLE` answer, such as +`runs.logs`, with the status `unavailable` instead of `failed`. Alerts that +count `weave.requests` with `status=failed` no longer include these answers. + +The update-time order is a live view: a later update can move an item between +pages. Refresh from the first page to see the latest ordering. Unknown historical +times sort last and are never replaced by the fact projection's creation date. +Restored runs reappear even though their archive lifecycle record is retained. + +Old runs may have incomplete timelines. The server reads their missing author +step metadata in batches of 500, retaining only the requested page candidates. +A request stops after 100,000 keys or two seconds with `429 WV-RUNTIME-LIMIT`; +the time budget can be reached first. Synthetic control instances do not count +toward completeness or the key cap. Step outputs remain optional and classified. +`runs.logs` still answers authorized requests with `501 WV-UNAVAILABLE`. + +## Incident list compatibility + +`incidents.list` and the paged `incidents.run_list` now read incident facts +instead of loading each incident's run. Their default order changes to +`opened_desc`, newest opened first, with `opened_asc` also available; incidents +whose opening time is unknown come last. Clients that continue a cursor saved +before the upgrade must send `order=id` without filters. The +`weave runs incidents list` and `run-list` commands do not send `order=id` yet, +so they cannot continue such a cursor; start the listing again. + +The incident lists also change how they read requests: + +- They accept `status` (repeatable), `code`, `workflow`, `node_id`, an opening + time range, and `include_test`. Without `status` they still list every + status. +- Incidents of test runs are left out unless the request sends + `include_test=true`. +- They reject unknown query parameters with `422 WV-VALIDATION`; the earlier + routes ignored them. A `limit` outside 1 to 100 answers `422 WV-VALIDATION` + instead of `422 WV-PAGE`. +- An older server ignores the filters and the order and lists every incident in + ID order. Upgrade servers before relying on them. + +Incident views gain optional fields that are left out while unknown, false, or +empty. **Upgrade CLIs and SDKs with the server.** Earlier releases reject the +new fields: `weave runs incidents list` and `run-list`, and the Python SDK's +`list_incidents` and `run_incidents`, fail with `502 WV-WIRE` on the first page +that lists an available incident. Stored resolution receipts and +`incidents.resolve` answers keep their bytes, and so does the earlier plain +array of a run's incidents. A resolve answer is the incident as stored, so read +it again with `incidents.read` to show it as the lists do. + +For an incident of a repeated step instance, `node_id` becomes the static step +ID and `instance_key` carries the full key; other incidents, including those on +a switch or parallel branch output or a join, keep the same `node_id`. Whether +an incident is withheld now comes from its run's metadata, its policy blocks, +and whether its step matches its incident facts, so a list never loads runs. +The new `incidents.read` operation serves direct links and sends the revision +as `ETag`. The Python SDK adds `read_incident`, and `list_incidents` and +`run_incidents` take either a `query` or the keyword arguments; passing both +raises `ValueError`. + +Historical opening times come from each incident's opening event. When that +event was not retained, the upgrade used the run's last recorded event and +marks the time `opened_at_estimated`; without either, the opening time stays +unknown. + +## Worker identity compatibility + +Workers can now report where they run when they register, and worker reads +return it. Deployment observations can also list the containers or pods of a +worker component. Both changes cross versions: + +- **New workers, older server.** The Agentic and Files workers and the Python + SDK's `register_worker` send an `identity`. A server without it refuses the + field with `422 WV-VALIDATION`; the worker then registers exactly once more + without it, on the same connection and credentials. Other failures are not + retried. Upgrade servers first to keep the identity. Native executors register + inside the API process, so they always use a server of their own version. +- **Older clients, new server.** Worker reads, lists, and drain or resume + answers add `identity` and `registered_at` when they are known. **Upgrade + CLIs and SDKs with the server.** Earlier releases reject the new fields: + `weave workers list`, `read`, `drain`, and `resume`, and the Python SDK's + `read_worker`, `list_workers`, `drain_worker`, and `resume_worker`, fail with + `502 WV-WIRE` for a worker registered after the upgrade. A drain or resume is + still applied; read the worker again with an upgraded client. Workers + registered before the upgrade, the registration answer, and stored worker + records keep their earlier shape. +- **Labels.** `WEAVE_WORKER_LABELS` is read only from the worker's own + environment. An invalid label stops the worker before it registers, so check + the setting before rolling out new worker images. With native executors + configured, an invalid label on the API's own environment stops the API from + starting. + +**Upgrade the server before its runners.** An upgraded runner reports +`members` for worker components, and an older server refuses them: the runner's +observe jobs fail, and an apply that touches a worker component ends +`reconciliation_required` with an ambiguous receipt, even though the change +was applied. Upgrade the API first, then each runner. + +- **Older clients, observations with members.** Observations that an upgraded + runner recorded with `members` fail with `502 WV-WIRE` in earlier releases: + `weave operations observations read` and `list`, and the Python SDK's + `invoke("deployment_observations.read")` and + `invoke("deployment_observations.list")`. Upgrade CLIs and SDKs with the + server before upgrading runners. +- **Unchanged.** Observations without members, their digests, plans reviewed + against them, and Compose and Kubernetes resource versions keep their bytes. +- **Members are placement only.** They list live containers or pods: stopped, + terminating, finished, failed, and evicted ones are left out. When a worker + component has more than 100 live members, or the runner cannot list them all, + `members` is left out and placement is unknown; the observation itself still + succeeds. +- **Kubernetes.** After upgrading the server, and before upgrading a Kubernetes + runner, grant it `list` on `pods` in the namespace. Without that permission + the runner leaves `members` out. See + [the runner guide](operations-runner.md#kubernetes). + +## Worker fleet compatibility + +`workers.list` now reads each page with one query and changes what it lists by +default: + +- **Current workers only.** Without `state=all`, it leaves out revoked workers + and workers neither seen nor registered in the last hour. Workers registered + before the upgrade that were never seen are listed only with `state=all`. +- **Newest contact first.** The default order is `last_seen_desc`; + `registered_desc` is also available, and unknown times come last. +- **Cursors.** Cursors issued before the upgrade are refused with + `422 WV-VALIDATION`; start the listing again. `weave workers list` sends only + `limit` and `cursor`, so it lists current workers. +- **Requests.** Unknown query parameters are refused with `422 WV-VALIDATION` + instead of being ignored, and a `limit` outside 1 to 100 answers + `422 WV-VALIDATION` instead of `422 WV-PAGE`. An older server ignores the new + filters and lists every worker in ID order, so upgrade servers before relying + on them. +- **Earlier Studio hosts and desktop apps.** The Studio bridge forwards answers + unchanged, so an earlier Studio host or desktop app gets this reduced default + list from an upgraded server. Its Workers page lists only current workers, + **Show revoked workers** shows none, and rows reorder as workers check in. + Upgrade Studio and the desktop app with the server. Until then, the Python + SDK's `list_workers(query=WorkerListQuery(state="all"))` lists every worker. + The upgraded Studio asks the platform for current workers and turns on + `state=all` with **Show retired workers**; against an older server its filters + are ignored and it lists every worker. + +Worker reads and lists add optional `placement` and `drained_at`, and drain and +resume answers add `drained_at`. **Upgrade CLIs and SDKs with the server.** +Earlier releases reject these fields: `weave workers list`, `read`, `drain`, and +`resume`, and the Python SDK's `read_worker`, `list_workers`, `drain_worker`, +and `resume_worker`, fail with `502 WV-WIRE` for a worker that is placed, or +drained since the upgrade. A drain or resume is still applied. Stored worker records and the registration +answer keep their bytes, and workers drained before the upgrade show no drain +time. + +Two operations are new: `workers.retire_stale` (`weave workers retire-stale`, +SDK `retire_stale_workers`) revokes up to 100 offline workers without active +leases per command, and `releases.usage` (`weave workers releases usage`, SDK +`release_usage`) reports where a release runs. Neither changes an existing +answer. Earlier servers do not serve them and answer `404` or `405`. + +## Environment summary and telemetry settings + +Two reads are new, and neither changes an existing answer: + +- `operations_summary.read` (`weave operations summary`) answers an + environment's health, load, workers, targets, incidents and the items that + need attention. It needs `status.read`; each section beyond health, load and + workers needs its own capability, and a section you may not read answers + `{"state": "forbidden"}` instead of zeros. Run figures leave out test runs. +- `operations_telemetry.read` (`weave operations telemetry`) answers whether + telemetry export is on and which collector hosts it uses. It needs + `status.read` granted for the project, not only for an environment, and + never returns endpoint paths, credentials or headers. + +Earlier servers do not serve them and answer `404` or `405`. For the first 24 +hours after the upgrade, `opened_24h` leaves out incidents whose opening time +the upgrade could only estimate. A `WEAVE_TELEMETRY` collector URL whose host +name is longer than 253 characters, which no DNS name is, now stops startup with +`Invalid WEAVE_TELEMETRY configuration`. Loop health is +measured in each API process and starts again when the process restarts, so a +freshly started process reads `degraded` until each loop completes a cycle. An +answer from an API-only replica reports every scheduler loop as `disabled`; the +summary's `instance` ID tells you which process answered. The reference explains +how each state is decided; see +[Discovery, debugging, and history](../reference/api.md#discovery-debugging-and-history). + +The Studio host bridge now forwards both reads and `releases.usage`. An earlier +Studio host or desktop app refuses them with `404`, so upgrade Studio and the +desktop app with the server before relying on the pages that use them. + +## Operational fact storage + +The `0032_operations_facts` migration adds five scoped operational projections +and the operations loop's bounded traversal cursors: one for tenants, one per +tenant for environments, and one per environment for re-checking stored listing +decisions. The scheduler login reaches tenants through the new execute-only +`weave_operations_tenants(integer)`, granted to `weave_scheduler`, so it never +moves the recovery traversal's cursors. The migration does not add columns to +stored runs or events. Apply it explicitly with all writers stopped, using the procedure above. + +The migration runs in one transaction and reads retained runs in UUID order, at +most 10,000 IDs per batch. It validates each run's state, activation and request, +classifies each distinct stored artifact once, and aggregates event timestamps per +batch. The indexes it adds to existing tables (`run_deadlines`, `event_deliveries` +and `incidents`) are built inside that transaction, so their locks last until the +whole migration commits: the maintenance window is the migration's duration, which +is one more reason to stop every writer first. The foreign keys of the new fact +tables likewise lock `runs`, `task_intents`, `incidents`, `worker_instances`, +`environments` and `tenants` against writes until the migration commits. Allow temporary disk space for all +five projections, their indexes, and transaction/WAL overhead. + +Rehearse against a restored copy to measure your maintenance window. On a +development machine (PostgreSQL 17 in a 4-CPU, 8 GiB virtual machine with default +memory settings, upgrading from `0031_operations_key_collation`), the migration +took 17 seconds for 10,000 retained runs and 32 seconds for 50,000, and the +database grew by 1.4 to 1.6 KB per retained run (16 MB and 70 MB). Usage counters +equaled an independent recomputation afterwards in both. Larger sizes were not +measured: if the time keeps growing linearly, 1,000,000 runs would take roughly 10 +to 15 minutes on similar hardware, but treat that as an estimate, not a +measurement. + +Historical run times come from accepted events. Missing, malformed or oversized +evidence remains unavailable, and malformed lifecycle values remain unknown +instead of being labeled as queued. A run requiring an unsupported IR version or +language feature keeps that separate classification. Historical origins remain +unknown unless an explicit start, retry or matching start receipt proves them. +Signal delivery receipts do not establish how a run started. Historical step +and task times are not invented, and worker instances registered before the +upgrade keep an unknown registration time and no identity; a worker reports its +identity when it next registers. Each run projection records when its coverage +began. + +The new facts count toward ordinary storage usage. Near a quota ceiling, this +means a claim or completion that an earlier server accepted can be refused with +the usual capacity answer, and it changes nothing when it is refused. Active executions reserve +additional capacity for bounded terminal fact updates, including allocation +provenance. Backfill inventories retained data without granting extra admission +capacity: a platform already above a quota can still migrate, but new work must +satisfy the unchanged quotas. Purging an authorized archived run also removes +its run, step, task and incident facts; deleting a worker removes its facts. +The cursor tables hold a single row for tenants, one row per tenant for +environments, and one row per environment for re-checks. They have no business +payload or retained history. + ## Start and inspect compatibility Remove the migration credentials from the runtime environment, then start the @@ -265,6 +532,26 @@ on the older server records only the cancellation, as for and its state stays withheld after you upgrade. To keep a run's full record, let it finish or cancel it before you roll back. +After the upgrade, `runs.read` serves the run at once, but run summaries, step +timelines and incident lists keep withholding it until the scheduler re-checks +its stored listing decision; its timeline answers `422 WV-IR-UNSUPPORTED` +meanwhile. The scheduler does this in the background: each minute it visits up +to 16 tenants, gives each one's next environment up to one second of database +work, and re-checks at most 100 runs there, fewer when the second runs out +first. A tenant's environments take turns, one per minute. So in an installation +with one tenant and one environment, 10,000 waiting runs take at least 100 +minutes, and a million about a week. More environments in a tenant, or more +than 16 tenants, make each environment wait longer. + +Runs recorded under an earlier classification policy are re-checked the same +way, and their timelines answer `409 WV-LEGACY-UNAVAILABLE` meanwhile. A policy +change re-checks available and unsupported decisions only: unavailable ones, +such as runs whose history the upgrade could not trust, stay unavailable. API +replicas that disable the scheduler rely on the scheduler replica for this. A +run whose re-check fails, for example because its record cannot be locked in +time or ordinary storage is full, stays withheld, is logged by error class, and +is tried again on a later pass while the others continue. + **A server whose capabilities do not list `language_features` predates language features.** It cannot recognize such a run, treats it as legacy evidence, and can block it permanently; a blocked run's tasks are never offered @@ -359,8 +646,8 @@ changes before you reopen traffic: ## What changes after alpha15 -These changes are not yet in a release. They add one schema migration (the head -becomes `0031_operations_key_collation`), no role, and no change to definition schemas or +These changes are not yet in a release. They add two schema migrations (the head +becomes `0032_operations_facts`), no role, and no change to definition schemas or the stored digests of the built-in AI Connector, Action and task. Check them before you reopen traffic on a server that includes them: diff --git a/docs/reference/api.md b/docs/reference/api.md index 51fd7279..cdf0a18e 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -569,7 +569,9 @@ capabilities, bindings, Actions, and connection settings, plus **Lists.** Canonical lists return `{ "items": [...], "next_cursor": null | "opaque" }`. -- `limit` is 1 to 100 (50 by default), and items come in stable ID order. +- `limit` is 1 to 100 (50 by default). Items come in stable ID order unless a + list documents another order: `runs.list` and run summaries are newest + started first, and incident lists newest opened first, by default (see below). - Pass `next_cursor` back as `cursor` for the next page. A cursor is bound to its tenant, project, environment, collection, and any run or schedule filter; a cursor from another scope or filter is rejected. @@ -596,22 +598,277 @@ Replay reports `consistent`, `inconsistent`, or `incomplete`: a history prefix, or a history with unavailable or redacted parts, is never reported as success. See [history and replay](history-and-replay.md). -**Run summaries, steps, and logs.** `run_summaries.list`, `runs.steps`, and -`runs.logs` are published so that clients can build against them. Until this -server serves them, an authorized request answers `501` with `WV-UNAVAILABLE`. - -- These lists are ordered by time, not by ID: run summaries by start time - (newest first by default) or by last update, steps by scheduled time, and log - entries by time. +**Run summaries, steps, and logs.** `run_summaries.list` and `runs.steps` serve +scoped operational metadata after checking both token and current permissions. +`runs.logs` remains a published contract: an authorized request answers `501` +with `WV-UNAVAILABLE`. + +- Run summaries and `runs.list` default to `order=started_desc`. They also accept + `started_asc` and `updated_desc`; `runs.list` additionally accepts `order=id`. + Use explicit `order=id` with legacy v1 cursors. The original business-key, + correlation-key, single-status and archive filters retain their v1 collection; + additional filters bind the cursor to the normalized query. +- Chronological lists use v2 cursors with UUID tie breakers and unknown times + last. `updated_desc` is a live view: updates can move a run between pages, + so refresh from the first page to see its latest position. Steps sort by + scheduled time, static step ID and full instance key; unknown times sort last. - `runs.steps` and `runs.logs` accept a `limit` of 1 to 500 (200 by default). - Each cursor is bound to every filter and to the order. After you change a filter, start again from the first page. - Test runs are left out unless you send `include_test=true`, and archived runs - unless you send `include_archived=true`. + unless you send `include_archived=true`. This applies to `runs.list` too. +- `runs.list` and run summaries reject unknown query parameters, and a `limit` + outside 1 to 100 answers `422` with `WV-VALIDATION`. +- A `runs.list` page is limited to 8 MiB of retained run data. A larger page + answers `429` with `WV-PAGE-LIMIT`; request a smaller `limit`. - A contradictory filter, such as `version` without `workflow`, answers `422` with `WV-FILTER`; the message names the rule that failed. - Send times as RFC 3339 with an offset. Use `Z`, or encode `+` as `%2B` in the query string. +- A run summary's `handled_errors` counts the failed steps whose error a step + setting handled so the run went on, and `has_handled_errors` filters on it; a + step fact's `handled` says how (`continue` or `errorOutput`). Workflows cannot + handle step errors yet, so this server leaves `handled_errors` out (zero) and + never sets `handled`; the fields are reserved so clients need no change when + they can. + +Step timelines do not read payloads unless `include=output` is requested. Only +the displayed outputs are fetched, after checking the bounded run artifact and +classifying each output. The outputs on one page are limited to 8 MiB in total; +a larger page answers `429 WV-PAGE-LIMIT`, so request a smaller `limit`. An +omitted output carries an omission reason; a returned JSON `null` is distinct +from an omitted output. The output of an instance key with an activation +segment (`#`) or a yield count (`~`) is omitted as `classification_unavailable`, +because its step's schema does not describe it. Synthetic control +instances do not appear. `complete=false` means the run has author step +instances without facts, even when they fall outside the current page or filter. +Their unknown timestamps remain null. Legacy metadata is streamed in batches +of 500, up to 100,000 keys and two seconds per request; exceeding either budget +answers `429 WV-RUNTIME-LIMIT`. A run this platform cannot run yet answers +`422 WV-IR-UNSUPPORTED`, as `runs.read` does; a run with unavailable legacy +evidence answers `409 WV-LEGACY-UNAVAILABLE`. Timelines, summaries and incident +lists follow the run's stored listing decision. After an upgrade lets the +platform run such a run again, `runs.read` serves it at once, but its timeline +keeps answering `422 WV-IR-UNSUPPORTED` until the scheduler re-checks the +decision in the background. Each minute the scheduler visits up to 16 tenants +and re-checks at most 100 runs in one environment of each, so a large backlog +takes a long time; see [Upgrades](../operations/upgrades.md#runs-that-need-a-newer-server). +In `runs.list`, only a recorded policy block withholds a run before its own read +classifies it. + +**Incidents.** `incidents.list` and the paged `incidents.run_list` read +incident facts after checking both token and current permissions; they never +load run payloads. `incidents.read` (`GET …/incidents/{identifier}`) reads one +incident for a direct link and sends its revision as the `ETag`, ready for +`incidents.resolve`. + +- Filters: `status` (repeatable: `active`, `closed`, `resolved`; every status + when omitted), `code`, `workflow`, `node_id`, `opened_after`, + `opened_before`, and `include_test`. `node_id` is a static step ID and + matches every instance of that step. Opening times follow the run summary + rules: RFC 3339 with an offset, `opened_after` earlier than `opened_before`, + and a range of at most 400 days. +- Incident lists default to `order=opened_desc` and also accept `opened_asc` + and `order=id`. Incidents whose opening time is unknown come last. ID order + without filters keeps the earlier cursors, so send explicit `order=id` to + continue one; every other cursor is bound to the run, each filter, and the + order. +- Incidents of test runs are left out unless you send `include_test=true`. + Incidents of archived runs stay listed. +- Incident views add `opened_at`, `updated_at`, `workflow` (`name` and + `version`), `run_status`, `test`, and, for an incident of a repeated step + instance, `instance_key`; `node_id` is then the static step ID. Each is left + out while unknown, false, or empty. `opened_at_estimated: true` marks an + opening time taken from the run's last recorded event because the incident's + own opening event was not retained; an unknown opening time is never + estimated. +- An incident on a control step, a switch or parallel branch output + (`@branch:…`) or a join (`@join:…`), keeps that control-step ID as `node_id` + and has no `instance_key`. The `node_id` filter takes author step IDs, so it + does not select these incidents. +- An incident whose run metadata cannot be classified, whose run has a recorded + policy block, or whose step does not match its incident facts is an + unavailable placeholder. `incidents.read` then sends no `ETag`. +- `incidents.resolve` answers with the incident as stored, without the added + fields; for a repeated step instance, its `node_id` is the full instance key. + Read the incident again with `incidents.read` after resolving to show it as + the lists do. +- A page is limited to 8 MiB of retained incident data; a larger page answers + `429` with `WV-PAGE-LIMIT`. Unknown query parameters are rejected, and a + `limit` outside 1 to 100 answers `422` with `WV-VALIDATION`. + +**Worker identity.** `workers.create` accepts an optional `identity` describing +where the worker runs: `host`, `runtime` (`embedded`, `container`, or +`process`), `sdk_version`, `language_version`, `os`, `arch`, `platform_hint` +(`kubernetes`, `docker`, or `none`), `namespace`, `node`, and up to 8 +`labels`. It is display metadata and never grants authority or placement. + +- `host`, `namespace`, and `node` are host or DNS names of at most 253 + characters. Versions, `os`, and `arch` have at most 64 letters, digits, `.`, + `_`, `+`, or `-`. +- Label keys have at most 64 characters and must not name a token, password, + secret, or authorization; values have at most 256 characters without control + characters. The whole identity is at most 4 KiB as stored JSON. +- Unknown fields, a `placement` among them, are refused with `422` + `WV-VALIDATION`; placement is derived by the server, never reported. +- The registration answer keeps its earlier shape and does not repeat the + identity. `workers.read`, `workers.list`, `workers.drain`, and + `workers.resume` return `identity` and `registered_at` (the database time of + registration) when they are known. Workers registered before the server + recorded them have neither. + +An older server refuses `identity` with `422` `WV-VALIDATION`. Current packaged +workers and the Python SDK then register exactly once more without it, using the +same client and credentials; any other failure is not retried. Native executors +register inside the API process and always use a server of their own version. +The registration answer's schema (`WorkerInstance`) has no `identity` field. + +**Worker fleet.** `workers.list` reads every page with one database query, +including each worker's active leases, and checks `status.read` on both the +token and the current principal. + +- By default it lists current workers: not revoked, and seen in the last hour + or, never seen, registered in the last hour. `state=all` lists every worker, + including revoked ones and workers registered before registration times were + recorded. +- Filters: `presence` (`recent`, `stale`, or `unknown`, with the 60-second + presence window), `draining`, `release_id`, `task_type` (a capability as + workers register it, such as `echo@1.2.0`), `target_id`, and `host_prefix`, a + literal, case-sensitive prefix of the reported host in which `%` and `_` + match only themselves. `target_id` needs `deployment.read` on that target. +- Orders: `last_seen_desc` (default) and `registered_desc`; unknown times come + last, and equal times are ordered by ID. Cursors are bound to every filter and + the order. A `limit` outside 1 to 100 or an unknown query parameter answers + `422 WV-VALIDATION`, and a page is limited to 8 MiB of worker data + (`429 WV-PAGE-LIMIT`). +- `placement` names the deployment component a worker runs in: + `target_id`, `target_name`, `deployment_id`, and `component`. The server + matches the worker's reported `host` against the members of the latest + complete observation of each target, taken at the target's current revision + and not yet expired. Only targets that both the token and the current + principal may read with `deployment.read` are considered, so other targets' + names never appear. A host listed by more than one readable target or + component, a native executor (`runtime: embedded`), and a worker without a + reported host have no placement. Because uniqueness is decided among readable + targets, a reader restricted to some targets can see a placement for a host + that a reader of every target sees as ambiguous, and so without placement. +- `drained_at` is the time the worker started draining. A repeated drain keeps + it and resuming clears it; workers drained before the time was recorded show + none. `workers.read` returns the same fields as the list, and `workers.drain` + and `workers.resume` answers include `drained_at` but no placement. + +`workers.retire_stale` (`POST …/workers/retire-stale`, `release.retire`, +Idempotency-Key) revokes workers that hold no active lease and whose last +contact (or registration, if never seen) is older than `older_than_seconds`, +from one hour to 30 days. A worker without either time is never retired. Each +command revokes at most 100 workers, oldest last contact first, and answers +`{matched, revoked, worker_ids}`: `matched` counts every eligible worker, and +`revoked` and `worker_ids` describe this batch. `dry_run: true` returns the same +batch without revoking it. Repeating an Idempotency-Key returns the saved +answer; send a new key for the next batch. Retirement never deletes or settles +leases, and a worker another command is holding is left for the next batch. + +`releases.usage` (`GET …/worker-releases/{identifier}/usage`, `catalog.read`) +answers `{workers, deployments, activations, next_cursor}`: + +- `workers` counts every instance of the release once: `revoked`, else + `draining`, else `online` (seen in the last 60 seconds) or `offline`. +- `activations` counts the activation revisions that pin the release and can + still start runs: neither they, the latest revision of their activation, nor + their workflow version is retired. +- `deployments` lists, at most `limit` (1 to 100, default 100) at a time, the + desired deployment components whose admitted release and image digest are + this release: `target_id`, `target_name`, `deployment_id`, `deployment_name`, + `component`, and `replicas`. Only targets that both the token and the current + principal may read with `deployment.read` are listed; without that capability + the list is empty and `deployments_hidden` is `true` (it is left out + otherwise), so an empty list from a reader who may see no target is not + mistaken for "nothing runs this release". The worker counts need only + `catalog.read`: they are counts per state, never worker identities. `next_cursor` continues the list for the same release and + readable targets. +- A release of another environment answers `404 WV-NOT-FOUND`. + +**Environment summary.** `operations_summary.read` +(`GET …/environments/{environment}/operations/summary`, `weave operations +summary`) answers one environment's platform health in a single read. It needs +`status.read` on both the token and the current principal, and reads every +section in the same transaction that reloads those grants. + +- `observed_at` is the database time of the answer. `api` names the server + `version`, its image `build` digest when the server knows it, the `instance` + ID of the process that answered (new each time it starts), its + `uptime_seconds`, and `compatibility` (`mode` and the number of this + project's `findings`). +- `health` lists `database` (with `latency_ms` of a bounded probe), each + background loop (`recovery_loop`, `schedules`, `outbox_loop`, + `provider_loop`, `kafka_loop`, `operations_loop`, `alert_evaluator`), + `native_executors`, and `telemetry`, each with a `state` of `ok`, `degraded`, + `down`, or `disabled`. Loops add `last_started_at` and `last_ok_at` when + known, `consecutive_failures`, and `recent_failures` (failed cycles among + the last 20). `telemetry` is `disabled` while export is off, `degraded` when + the process dropped telemetry within the last three metrics export intervals + (90 seconds by default), and `ok` otherwise. Health is measured by the + process that answered; no exception text is returned. +- `load`: `runs_active` (active by status, as the run usage counter counts + them, with test runs left out), `tasks_ready` and + `oldest_ready_seconds` (tasks due now; a retry joins when its next attempt is + due), `tasks_leased`, `waits_active`, `waits_overdue` and + `oldest_overdue_seconds`, `outbox_pending` and `outbox_oldest_seconds`, + `provider_pending`, and `sources_enabled`. Task and worker figures count test + runs' work too, because they measure shared capacity. An age is `null` when + nothing waits. +- `workers` counts current workers as `workers.list` lists them by default: + `online` (seen in the last 60 seconds), `offline`, `draining`, the `capacity` + of online workers that are not draining, and the `active_leases` workers hold. +- `targets` needs `deployment.read` and counts only the targets both the token + and the current principal may read: `total`, `runner_online`, `busy` (an + active job), `reconciliation_required`, and `awaiting_approval` (plans that + can still be approved: unexpired, on a fresh observation of the current + target and deployment revisions, and neither approved nor applied). +- `incidents` needs `incident.read`: `active` and `opened_24h`, leaving out + incidents of test runs. `opened_24h` counts only known opening times: an + incident whose opening time the upgrade estimated is never counted. +- `attention` lists at most 20 items, `{kind, id, title}` plus `expires_at` for + approvals, in this order: jobs that need reconciliation, plans you may + approve (soonest expiry first), offline workers that still hold active + leases, active incidents, targets whose runners are all offline, a restricted + compatibility verdict (`id` is `compatibility`), and runs that failed in the + last hour. Within a kind, items come oldest first, then by ID. Each kind + needs the capability that reads its records: `deployment.read` on the target + for jobs and runners, `deployment.read` and `deployment.approve` on the same + target for approvals, `incident.read` for incidents, and `run.read` for + failed runs. Items you may not read are left out; test runs never appear. +- A section you may not read is `{"state": "forbidden"}`; none of its queries + run, and leaving it out records no authorization denial. Each section has two + seconds of database time in all (or a stricter statement timeout already in + force), shared by its queries: each runs under what is left. One that runs + out, or meets a transient database failure, is `{"state": "unavailable"}` + while the others are still served with `200`. Neither state is ever reported + as zero. The answer has no 24-hour run figures yet. + +Loop states follow these rules. A loop is `disabled` when its setting turns it +off (API-only replicas disable every scheduler loop) and `down` while its task +is not running. It is `degraded` before its first successful cycle, after 3 +failed cycles in a row or 3 failed cycles among its last 20, and when its last +success is older than three poll intervals. The window is those 20 cycles only: +a failure that comes back fewer than 3 times in any 20 cycles, for example once +every 10 cycles, leaves the loop `ok`, with only `recent_failures` showing it. A cycle still running within its +loop's budget is on time: 5 seconds for the outbox and broker traversals, 30 +seconds for recovery and schedules, and 60 seconds for provider and mailbox +polling. A cycle that starts no work, such as an outbox cycle while the platform +is not ready or while no delivery slot is free, is neither a success nor a +failure, so a loop that keeps waiting turns `degraded` without counting +failures. Ages use the process's monotonic clock; the times shown come from +its wall clock. + +**Telemetry settings.** `operations_telemetry.read` +(`GET …/projects/{project}/operations/telemetry`, `weave operations +telemetry`) needs `status.read` for the project, on both the token and the +current principal; an environment grant is not enough. It answers whether +telemetry export is `enabled`, whether `traces` and `metrics` have a collector +`configured` and its `host` name, the metrics `interval_seconds`, and how many +telemetry items this process has `dropped`. It never returns endpoint paths, +ports, queries, credentials, or headers. **Answers that say "unavailable".** Cancellation and task completion can return an unavailable acknowledgment when the platform cannot classify the historical @@ -661,6 +918,8 @@ operation in the [full API reference](api-explorer.md); to generate a client, | `lumi.status` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/lumi/status` | lumi.use | | `lumi.ask` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/lumi/ask` | lumi.use | | `compatibility.read` | `GET /api/v1/tenants/{tenant}/projects/{project}/operations/compatibility` | status.read | +| `operations_summary.read` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/operations/summary` | status.read | +| `operations_telemetry.read` | `GET /api/v1/tenants/{tenant}/projects/{project}/operations/telemetry` | status.read | | `compatibility.check` | `POST /api/v1/tenants/{tenant}/projects/{project}/operations/compatibility/check` | compatibility.check | | `retention.plan` | `POST /api/v1/tenants/{tenant}/projects/{project}/operations/retention/plans` | retention.plan | | `retention.read` | `GET /api/v1/tenants/{tenant}/projects/{project}/operations/retention/plans/{identifier}` | retention.plan | @@ -771,14 +1030,17 @@ operation in the [full API reference](api-explorer.md); to generate a client, | `runs.logs` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/runs/{identifier}/logs` | run.read | | `incidents.run_list` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/runs/{identifier}/incidents` | incident.read | | `incidents.list` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/incidents` | incident.read | +| `incidents.read` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/incidents/{identifier}` | incident.read | | `incidents.resolve` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/incidents/{identifier}/resolve` | incident.resolve | | `releases.create` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/worker-releases` | release.activate | | `releases.list` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/worker-releases` | catalog.read | | `releases.read` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/worker-releases/{identifier}` | catalog.read | +| `releases.usage` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/worker-releases/{identifier}/usage` | catalog.read | | `workers.create` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/workers` | worker.register | | `workers.list` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/workers` | status.read | | `workers.read` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/workers/{identifier}` | status.read | | `workers.revoke` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/workers/{identifier}/revoke` | release.retire | +| `workers.retire_stale` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/workers/retire-stale` | release.retire | | `workers.grant` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/worker-connection-grants` | connection.manage | | `tasks.claim` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/tasks/claim` | task.claim | | `tasks.context` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/tasks/context` | task.claim | @@ -860,6 +1122,17 @@ and runner registrations belong to one environment. Child-resource reads check the target grant. Lists filter authorized target IDs before pagination; use `target_id` to select a target, and `deployment_id` to narrow plan history. An observation is a dated snapshot, not a promise of current health. +A worker component's observed resource lists up to 100 live `members`, one per +running or starting container or pod: `host`, `ordinal`, `state`, `ready`, +`started_at`, `restarts`, and `node`. Compose reports the container ID prefix, +which is the container's default hostname, and its Compose number as +`ordinal`; Kubernetes reports the pod name and node. Stopped, terminating, +finished, failed, and evicted containers or pods are left out. Members are +placement only: with more than 100 live members, or when the runner cannot list +them all, the field is left out (placement unknown) rather than truncated, and +the observation still succeeds. A resource without the field therefore has no +members or unknown ones, and its observation keeps its digest. Upgrade the +server before its runners; an older server refuses a report with `members`. The server creates an immutable plan from the desired revision and a fresh, complete observation. Approval records its exact digest. Applying checks the diff --git a/docs/reference/incident-operations.md b/docs/reference/incident-operations.md index a913163f..3a556407 100644 --- a/docs/reference/incident-operations.md +++ b/docs/reference/incident-operations.md @@ -50,8 +50,15 @@ an action or clear an incident. ## Resolve an incident -**In Studio**, open **Operate › Incidents**. It lists active incidents; select -one to see its code, its step and the run's last engine events. Then select +**In Studio**, open **Operate › Incidents**. It lists active incidents, +newest opened first, without test runs. The platform applies the filters +(status, code, workflow, step, an opening range, **Include test runs** and +**Order**), and the address keeps them and the incident you open +(`?incident=ID`), so a link opens that incident even when no loaded page lists +it. Select one to see its code, its step and full step instance, its workflow, +the run and its status, when it opened (marked **estimated** when the upgrade +inferred the time from the run's last recorded event, and **Unknown** when +nobody knows), and the run's last engine events (their types only). Then select **Resolve incident**, choose a decision and give the reason. To accept a verified result, also give its evidence reference and the result as JSON. Studio sends the incident's revision and a new receipt, and reuses that receipt @@ -211,6 +218,7 @@ repeated. Platform administration alone grants no execution rights. | --- | --- | --- | --- | | `GET /runs/{run}/incidents` | `weave runs incidents run-list` | `incident.read` | Incident views, including closed history | | `GET /incidents` | `weave runs incidents list` | `incident.read` | Incidents across the environment | +| `GET /incidents/{incident}` | `weave runs incidents read` | `incident.read` | One incident, with its revision as `ETag` | | `POST /incidents/{incident}/resolve` | `weave runs incidents resolve` | `incident.resolve` | An immutable, revision-fenced resolution | | `POST /runs/{run}/cancel` | `weave runs cancel` | `run.cancel` | Terminal cancellation with the external-effects flag | | `POST /runs/{run}/retry` | `weave runs retry` | `run.retry` and normal start admission | A new run linked by `parent_run_id` | diff --git a/docs/reference/sdk.md b/docs/reference/sdk.md index dccab849..a6279fc4 100644 --- a/docs/reference/sdk.md +++ b/docs/reference/sdk.md @@ -351,11 +351,11 @@ capability. | `create_connection`, `read_connection`, `test_connection`, `list_connections` | Immutable integration connections that hold only secret handles, and explicit checks | | `start_run`, `read_run`, `signal`, `cancel`, `retry`, `list_runs` | Durable runs | | `pause_run`, `resume_run`, `run_lifecycle`, `archive_run`, `restore_run`, `purge_run` | Run lifecycle management | -| `history`, `export_run`, `replay`, `run_incidents`, `list_incidents`, `resolve_incident` | Bounded evidence and audited recovery | +| `history`, `export_run`, `replay`, `run_incidents`, `list_incidents`, `read_incident`, `resolve_incident` | Bounded evidence and audited recovery | | `human_tasks`, `read_human_task`, `claim_human_task`, `release_human_task`, `complete_human_task`, `reassign_human_task` | Human task inbox and decisions | | `assignment_bindings`, `put_assignment_binding`, `put_human_group` | Who may work on which human task | | `register_release`, `read_release`, `list_releases`, `register_worker`, `read_worker`, `list_workers`, `revoke_worker`, `grant_connection` | Trusted worker releases and instances | -| `drain_worker`, `resume_worker` | Revision-checked worker controls; existing leases can settle while drained | +| `drain_worker`, `resume_worker`, `retire_stale_workers`, `release_usage` | Revision-checked worker controls, bounded retirement of offline workers, and where a release runs; existing leases can settle while drained | | `invoke("deployment_targets.…")`, `invoke("deployments.…")`, `invoke("deployment_observations.…")`, `invoke("deployment_plans.…")`, `invoke("deployment_jobs.…")`, `invoke("deployment_runners.…")` | Typed, environment-scoped Operations contracts; operation names and required authority are in the [API inventory](api.md#operation-inventory) | | `invoke("ai_endpoints.list")`, `invoke("ai_models.list", …)`, `invoke("ai_readiness.read")`, `invoke("ai_setup.publish", …)`, `invoke("ai_setup.grant", …)`, `invoke("ai_connections.test", …)`, `invoke("bindable_connections.list")` | AI model discovery, readiness, setup and connection choices; see [AI operations through the SDK](#ai-operations-through-the-sdk) | | `claim_tasks`, `heartbeat`, `complete_task`, `fail_task`, `request_credentials` | The typed [worker protocol](worker-protocol.md); `WorkerTransport` remains supported | @@ -378,6 +378,27 @@ code. Paginated methods return a typed `Page`, except run history, whose belong to the client's scope. Secret leases use the `CredentialLease` model, whose ordinary serialization leaves out the resolved value. +**Worker registration.** `register_worker` sends the request's optional +`identity`, usually from `worker_identity()` in `firefly_weave.sdk.worker_identity`. +When an older server refuses it with `422` `WV-VALIDATION`, the method registers +exactly once more without it; other errors are raised unchanged. A worker that +registers with its own HTTP client calls +`register_worker_http(client, environment_url, body)` from +`firefly_weave.sdk.worker_registration` for the same behavior on that client. +`worker_identity()` reads only `WEAVE_WORKER_REPORT_HOST`, `WEAVE_WORKER_LABELS`, +`WEAVE_POD_NAMESPACE`, `WEAVE_NODE_NAME`, and `KUBERNETES_SERVICE_HOST`; see +[worker identity](../guides/workers.md#report-where-a-worker-runs). + +**Worker fleet.** `list_workers` takes either the page keywords or a typed +`query=WorkerListQuery(...)` from `firefly_weave.contracts.worker_queries`; +passing both raises `ValueError`. Without a query the server lists current +workers, most recently seen first. `retire_stale_workers(RetireStaleRequest(...), +idempotency_key=...)` revokes, or with `dry_run=True` previews, up to 100 +offline workers; repeat it with a new key for the next batch. +`release_usage(release_id, query=ReleaseUsageQuery(...))` returns worker counts, +pinning activations and a page of readable deployments. See +[Discovery, debugging, and history](api.md#discovery-debugging-and-history) in the API reference for the rules. + ## Deployment Operations through the SDK Use `WeaveClient.invoke` with models from diff --git a/migrations/versions/0032_operations_facts.py b/migrations/versions/0032_operations_facts.py new file mode 100644 index 00000000..b290a5e2 --- /dev/null +++ b/migrations/versions/0032_operations_facts.py @@ -0,0 +1,683 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Scoped operational projections, bounded retained-history backfill and accounted control capacity.""" + +import json +from typing import Any +from uuid import UUID + +import sqlalchemy as sa +from alembic import op +from pydantic import TypeAdapter + +from firefly_weave.compiler.api import import_artifact +from firefly_weave.contracts.catalog import Activation +from firefly_weave.contracts.definitions import ResourceName +from firefly_weave.contracts.instance_keys import InstanceKeyText, instance_view +from firefly_weave.contracts.runtime import StartRunRequest +from firefly_weave.runtime.admission import POLICY, admission +from firefly_weave.runtime.models import RunState + +revision = "0032_operations_facts" +down_revision = "0031_operations_key_collation" + +SCOPE_COLUMNS = "tenant_id uuid NOT NULL,project_id uuid NOT NULL,environment_id uuid NOT NULL" +SCOPE_KEY = "tenant_id,project_id,environment_id" +SCOPE = "tenant_id=:tenant AND project_id=:project AND environment_id=:environment" +EXACT = " AND ".join( + f"{name}=nullif(current_setting('weave.{name}',true),'')::uuid" + for name in ("tenant_id", "project_id", "environment_id") +) +TABLE_KEYS = { + "run_facts": ("run_id",), + "step_facts": ("run_id", "node_id", "instance_key"), + "task_facts": ("task_id",), + "incident_facts": ("incident_id",), + "worker_facts": ("worker_id",), +} +STATUSES = "'queued','running','waiting','suspended','succeeded','failed','cancelled','timed_out'" +TERMINAL = "'succeeded','failed','cancelled','timed_out'" +ORIGINS = "'manual','webhook','schedule','broker','email','provider','retry','call','test'" +BATCH_SIZE = 10000 +FETCH_BYTES = 128 * 1024 * 1024 + + +def secure(table: str, key: tuple[str, ...]) -> None: + op.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY") + op.execute(f"CREATE POLICY exact_scope ON {table} TO weave_app USING ({EXACT}) WITH CHECK ({EXACT})") + op.execute(f"GRANT SELECT,INSERT,UPDATE ON {table} TO weave_app") + op.execute(f"GRANT SELECT,DELETE ON {table} TO weave_retention_owner") + op.execute(f"CREATE POLICY operations_inventory ON {table} TO weave_retention_owner USING(true)") + arguments = ",".join("'" + name + "'" for name in key) + op.execute( + f"CREATE TRIGGER operation_usage AFTER INSERT OR UPDATE OR DELETE ON {table} " + f"FOR EACH ROW EXECUTE FUNCTION weave_usage_charge({arguments})" + ) + + +def tables() -> None: + run_parent = f"FOREIGN KEY({SCOPE_KEY},run_id) REFERENCES runs({SCOPE_KEY},id) ON DELETE CASCADE" + op.execute(f"""CREATE TABLE run_facts( + {SCOPE_COLUMNS},run_id uuid PRIMARY KEY,definition_version_id uuid NOT NULL, + workflow_name text NOT NULL,workflow_version text NOT NULL,activation_id uuid NOT NULL, + activation_name text NOT NULL,activation_revision integer NOT NULL CHECK(activation_revision>0), + status text,paused boolean NOT NULL DEFAULT false, + test boolean NOT NULL DEFAULT false,origin text CHECK(origin IN ({ORIGINS})), + caller_run_id uuid,caller_node_id text,caller_instance_key text,retried_from_run_id uuid, + business_key text,correlation_key text,started_at timestamptz,updated_at timestamptz,ended_at timestamptz, + failed_node_id text,failed_instance_key text,failed_error_code text, + active_incidents bigint NOT NULL DEFAULT 0 CHECK(active_incidents>=0), + handled_errors bigint NOT NULL DEFAULT 0 CHECK(handled_errors>=0), + classification_state text NOT NULL CHECK(classification_state IN ('available','unavailable','unsupported')), + classification_policy text NOT NULL,pinned_ir_version text, + CHECK((status IS NULL AND classification_state='unavailable') OR + (status IS NOT NULL AND status IN ({STATUSES}))), + pinned_features jsonb NOT NULL DEFAULT '[]',node_kinds jsonb NOT NULL DEFAULT '{{}}', + last_event_sequence bigint NOT NULL DEFAULT 0,facts_started_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CHECK(CASE WHEN jsonb_typeof(pinned_features)='array' THEN + jsonb_array_length(pinned_features)<=256 AND octet_length(pinned_features::text)<=16384 AND + NOT jsonb_path_exists(pinned_features,'$[*] ? (@.type() != "string")') ELSE false END), + CHECK(CASE WHEN jsonb_typeof(node_kinds)='object' THEN octet_length(node_kinds::text)<=1048576 AND + jsonb_array_length(jsonb_path_query_array(node_kinds,'$.keyvalue()'))<=10000 AND + NOT jsonb_path_exists(node_kinds,'$.keyvalue().value ? (@.type() != "string")') ELSE false END), + {run_parent})""") + op.execute(f"""CREATE TABLE step_facts( + {SCOPE_COLUMNS},run_id uuid NOT NULL,node_id text NOT NULL,instance_key text NOT NULL DEFAULT '', + kind text NOT NULL,status text NOT NULL,scheduled_at timestamptz,started_at timestamptz,ended_at timestamptz, + attempts bigint NOT NULL DEFAULT 0 CHECK(attempts>=0),worker_id uuid,error_code text, + handled text CHECK(handled IN ('continue','errorOutput')),child_run_id uuid, + PRIMARY KEY(run_id,node_id,instance_key),CHECK(handled IS NULL OR status='failed'),{run_parent})""") + op.execute(f"""CREATE TABLE task_facts( + {SCOPE_COLUMNS},task_id uuid PRIMARY KEY,run_id uuid NOT NULL,node_id text NOT NULL, + instance_key text NOT NULL DEFAULT '',task_type text NOT NULL,status text NOT NULL, + created_at timestamptz NOT NULL,ready_since timestamptz,first_claimed_at timestamptz, + FOREIGN KEY({SCOPE_KEY},task_id) REFERENCES task_intents({SCOPE_KEY},id) ON DELETE CASCADE,{run_parent})""") + op.execute(f"""CREATE TABLE incident_facts( + {SCOPE_COLUMNS},incident_id uuid PRIMARY KEY,node_id text,instance_key text NOT NULL DEFAULT '', + opened_at timestamptz,updated_at timestamptz,workflow_name text NOT NULL,workflow_version text NOT NULL, + test boolean NOT NULL DEFAULT false,opened_at_estimated boolean NOT NULL DEFAULT false, + FOREIGN KEY({SCOPE_KEY},incident_id) REFERENCES incidents({SCOPE_KEY},id) ON DELETE CASCADE)""") + op.execute(f"""CREATE TABLE worker_facts( + {SCOPE_COLUMNS},worker_id uuid PRIMARY KEY,registered_at timestamptz,identity jsonb NOT NULL DEFAULT '{{}}', + drained_at timestamptz,CHECK(octet_length(identity::text)<=4096), + FOREIGN KEY({SCOPE_KEY},worker_id) REFERENCES worker_instances({SCOPE_KEY},id) ON DELETE CASCADE)""") + op.execute( + "CREATE TABLE operations_environment_cursors(tenant_id uuid PRIMARY KEY REFERENCES " + "tenants(id),environment_id uuid)" + ) + tenant = "tenant_id=nullif(current_setting('weave.tenant_id',true),'')::uuid" + op.execute("ALTER TABLE operations_environment_cursors ENABLE ROW LEVEL SECURITY") + op.execute("ALTER TABLE operations_environment_cursors FORCE ROW LEVEL SECURITY") + op.execute( + f"CREATE POLICY tenant_scope ON operations_environment_cursors TO weave_app USING " + f"({tenant}) WITH CHECK ({tenant})" + ) + op.execute("GRANT SELECT,INSERT,UPDATE ON operations_environment_cursors TO weave_app") + # The operations traversal pages tenants with its own cursor, so it never moves the recovery traversal's. + op.execute("CREATE TABLE operations_tenant_cursor(singleton boolean PRIMARY KEY CHECK(singleton),tenant_id uuid)") + op.execute("INSERT INTO operations_tenant_cursor VALUES(true,NULL)") + op.execute("GRANT SELECT,UPDATE ON operations_tenant_cursor TO weave_catalog_reader") + op.execute("""CREATE FUNCTION weave_operations_tenants(page_size integer) RETURNS SETOF uuid + LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,public AS $$ + DECLARE prior uuid; candidate uuid; visited uuid[] := ARRAY[]::uuid[]; + BEGIN + IF page_size IS NULL OR page_size<1 OR page_size>16 THEN RAISE EXCEPTION 'Bounded catalog page required'; END IF; + SELECT tenant_id INTO prior FROM public.operations_tenant_cursor WHERE singleton FOR UPDATE; + FOR candidate IN SELECT id FROM public.tenants WHERE prior IS NULL OR id>prior ORDER BY id LIMIT page_size LOOP + visited := array_append(visited,candidate); RETURN NEXT candidate; + END LOOP; + IF array_length(visited,1) IS NULL OR array_length(visited,1)0 THEN + UPDATE public.operations_tenant_cursor SET tenant_id=visited[array_length(visited,1)] WHERE singleton; + END IF; + END $$""") + op.execute("GRANT CREATE ON SCHEMA public TO weave_catalog_reader") + op.execute("ALTER FUNCTION weave_operations_tenants(integer) OWNER TO weave_catalog_reader") + op.execute("REVOKE CREATE ON SCHEMA public FROM weave_catalog_reader") + op.execute("REVOKE ALL ON FUNCTION weave_operations_tenants(integer) FROM PUBLIC") + op.execute("GRANT EXECUTE ON FUNCTION weave_operations_tenants(integer) TO weave_scheduler") + op.execute(f"""CREATE TABLE operations_fact_cursors({SCOPE_COLUMNS},after_run_id uuid, + PRIMARY KEY({SCOPE_KEY}),FOREIGN KEY({SCOPE_KEY}) REFERENCES environments(tenant_id,project_id,id) + ON DELETE CASCADE)""") + op.execute("ALTER TABLE operations_fact_cursors ENABLE ROW LEVEL SECURITY") + op.execute("ALTER TABLE operations_fact_cursors FORCE ROW LEVEL SECURITY") + op.execute( + f"CREATE POLICY exact_scope ON operations_fact_cursors TO weave_app USING ({EXACT}) WITH CHECK ({EXACT})" + ) + op.execute("GRANT SELECT,INSERT,UPDATE ON operations_fact_cursors TO weave_app") + + +def indexes() -> None: + for name, columns, condition in ( + ("started", "started_at DESC,run_id DESC", ""), + ("workflow", "workflow_name,started_at DESC,run_id DESC", ""), + ("status", "status,started_at DESC,run_id DESC", ""), + ("activation", "activation_id,started_at DESC,run_id DESC", ""), + ("updated", "updated_at DESC,run_id DESC", ""), + ("business_key", "business_key", ""), + ("correlation_key", "correlation_key", ""), + ("caller", "caller_run_id", "WHERE caller_run_id IS NOT NULL"), + ("retry", "retried_from_run_id", "WHERE retried_from_run_id IS NOT NULL"), + ("ended", "ended_at", "WHERE ended_at IS NOT NULL"), + # Active test runs as the run usage counter counts runs: by status, never by a missing end time. + ( + "active_test", + "", + "WHERE test AND (status IS NULL OR status NOT IN ('succeeded','failed','cancelled','timed_out'))", + ), + # Rows a listing-metadata refresh can need: another policy's decisions and transient unsupported ones. + ( + "revalidation", + "run_id", + "WHERE classification_state='unsupported' OR classification_policy<>'classified-v1'", + ), + ): + op.execute( + f"CREATE INDEX run_facts_{name} ON run_facts({SCOPE_KEY}{',' + columns if columns else ''}) {condition}" + ) + op.execute(f"CREATE INDEX step_facts_ended ON step_facts({SCOPE_KEY},ended_at)") + op.execute( + f"CREATE INDEX task_facts_ready ON task_facts({SCOPE_KEY},task_type,ready_since) WHERE ready_since IS NOT NULL" + ) + op.execute(f"CREATE INDEX incident_facts_node_opened ON incident_facts({SCOPE_KEY},node_id,opened_at,incident_id)") + # A run's reservation refresh counts its task facts by run. + op.execute(f"CREATE INDEX task_facts_run ON task_facts({SCOPE_KEY},run_id)") + # Incident pages newest (or oldest) opened first: known opening times walk this index in either direction. + op.execute( + f"CREATE INDEX incident_facts_opened ON incident_facts({SCOPE_KEY},opened_at DESC,incident_id DESC) " + "WHERE opened_at IS NOT NULL" + ) + op.execute(f"CREATE INDEX run_deadlines_operations_due ON run_deadlines({SCOPE_KEY},deadline) WHERE NOT consumed") + op.execute( + f"CREATE INDEX event_deliveries_operations_due ON event_deliveries({SCOPE_KEY},next_at) " + f"WHERE status IN ('pending','retry')" + ) + op.execute(f"CREATE INDEX incidents_operations_active ON incidents({SCOPE_KEY},status) WHERE status='active'") + + +def patch_function(signature: str, anchor: str, replacement: str) -> None: + connection = op.get_bind() + body = connection.scalar( + sa.text("SELECT pg_get_functiondef(cast(:signature AS regprocedure))"), {"signature": signature} + ) + if not isinstance(body, str) or body.count(anchor) != 1: + raise RuntimeError(f"Unexpected definition of {signature}; reconcile the control-accounting migration") + connection.exec_driver_sql(body.replace(anchor, replacement)) + + +def control_accounting() -> None: + anchor = "IF relation_name='runs' THEN RETURN (item->>'id')::uuid; END IF;" + patch_function( + "public.weave_control_target(text,jsonb)", + anchor, + """ + IF relation_name IN ('run_facts','step_facts','task_facts') THEN + RETURN (item->>'run_id')::uuid; + END IF; + IF relation_name='incident_facts' THEN + SELECT run_id INTO identifier FROM public.incidents WHERE id=(item->>'incident_id')::uuid; + RETURN identifier; + END IF; + """ + + anchor, + ) + anchor = "+(item->>'_weave_activation_bytes')::numeric+16777216+64*2048+65536+576*affected;" + predicate = "tenant_id=tenant AND project_id=project AND environment_id=(item->>'environment_id')::uuid" + patch_function( + "public.weave_reserve_refresh(text,uuid,boolean)", + anchor, + anchor[:-1] + + f""" + +2048+2048*((SELECT count(*) FROM public.step_facts WHERE {predicate} AND run_id=identifier) + +(SELECT count(*) FROM public.task_facts WHERE {predicate} AND run_id=identifier) + +(SELECT count(*) FROM public.incident_facts f JOIN public.incidents i ON + i.tenant_id=f.tenant_id AND i.project_id=f.project_id AND i.environment_id=f.environment_id + AND i.id=f.incident_id WHERE f.tenant_id=tenant AND f.project_id=project + AND f.environment_id=(item->>'environment_id')::uuid AND i.run_id=identifier));""", + ) + # Every fact write is charged to usage, but only a step fact insert or delete changes what the run's reservation + # counts on its own: run facts are not counted, updates change no count, and task and incident facts are written + # and removed in the same statement as their task or incident, whose own refresh sees them. + anchor = "PERFORM public.weave_reservation_touch(TG_TABLE_NAME,item);" + patch_function( + "public.weave_usage_charge()", + anchor, + "IF NOT (TG_TABLE_NAME IN ('run_facts','task_facts','incident_facts') OR " + "(TG_TABLE_NAME='step_facts' AND TG_OP='UPDATE')) THEN " + anchor + " END IF;", + ) + anchor = "ELSIF relation_name IN ('task_intents','task_leases','incidents','run_deadlines') THEN" + patch_function( + "public.weave_reservation_touch(text,jsonb)", + anchor, + "ELSIF relation_name IN ('task_intents','task_leases','incidents','run_deadlines'," + "'run_facts','step_facts','task_facts','incident_facts') THEN", + ) + + +def node_kinds(artifact: Any) -> dict[str, str]: + """Freeze author-step eligibility for the retained projection independently of live writers.""" + result = {} + for node in artifact.executable["graph"]["nodes"]: + key = node["id"] + if isinstance(key, str) and key.startswith("@"): + continue + TypeAdapter(InstanceKeyText).validate_python(key) + view = instance_view(key) + TypeAdapter(ResourceName).validate_python(view.node_id) + result[view.node_id] = node["kind"] + if len(result) > 10000 or len(json.dumps(result, ensure_ascii=False).encode()) > 1048576: + raise ValueError("Node-kind projection exceeds its bound") + return result + + +def classification(row: dict[str, Any]) -> dict[str, Any]: + result = { + "classification_state": "unavailable", + "classification_policy": POLICY, + "pinned_ir_version": None, + "pinned_features": "[]", + "node_kinds": "{}", + } + try: + RunState.model_validate_json(json.dumps(row["state"])) + Activation.model_validate_json(json.dumps(row["activation"])) + StartRunRequest.model_validate_json(json.dumps(row["request"])) + decision = admission(row) + executable = row["artifact"].get("executable", {}) + version, features = executable.get("irVersion"), executable.get("features", []) + if not isinstance(version, str) or len(version) > 256 or not isinstance(features, list): + return result + if len(features) > 256 or any(type(feature) is not str for feature in features): + return result + encoded = json.dumps(features, ensure_ascii=False) + if len(encoded.encode()) > 16384: + return result + result.update(classification_state=decision, pinned_ir_version=version, pinned_features=encoded) + if decision == "available": + result["node_kinds"] = json.dumps(node_kinds(import_artifact(row["artifact"])), ensure_ascii=False) + except (ValueError, TypeError, KeyError, AttributeError, RecursionError): + result.update(classification_state="unavailable", node_kinds="{}") + return result + + +MEMO_ENTRIES = 1024 + + +def classify(connection, params, run_id, row: dict[str, Any], memo: dict) -> dict[str, Any]: + """`classification` of one run, reading its artifact only for an artifact this scope has not classified yet. + + Once the run's own state, activation and request validate, the result depends only on the stored artifact, the + state's `unavailable` flag and whether the state names the current admission policy, so runs that share those + share the result. A row that does not validate is unavailable without reading its artifact. + """ + try: + RunState.model_validate_json(json.dumps(row["state"])) + Activation.model_validate_json(json.dumps(row["activation"])) + StartRunRequest.model_validate_json(json.dumps(row["request"])) + key = ( + bytes(row["artifact_key"]), + bool(row["state"].get("unavailable")), + row["state"].get("admission_policy") == POLICY, + ) + except (ValueError, TypeError, KeyError, AttributeError, RecursionError): + return classification({}) + if key not in memo: + artifact = connection.scalar( + sa.text(f"SELECT artifact FROM runs WHERE {SCOPE} AND id=:id"), {**params, "id": run_id} + ) + if len(memo) >= MEMO_ENTRIES: + memo.pop(next(iter(memo))) + memo[key] = classification({**row, "artifact": artifact}) + return dict(memo[key]) + + +EVENTS = f"""WITH ordered AS ( + SELECT e.*,lag(created_at) OVER(PARTITION BY run_id ORDER BY sequence) AS previous_at, + max(sequence) OVER(PARTITION BY run_id) AS final_sequence, + public.weave_run_events_bytes(e)<={FETCH_BYTES} AS bounded + FROM run_events e WHERE {SCOPE} AND run_id=ANY(:ids) +) SELECT run_id,min(created_at) AS started_at,max(created_at) AS updated_at, + max(created_at) FILTER(WHERE transition->'state'->>'status' IN ({TERMINAL})) AS ended_at, + min(sequence) AS first_sequence,max(sequence) AS last_event_sequence,count(*) AS event_count, + max(transition->'state'->>'status') FILTER(WHERE sequence=final_sequence) AS last_status, + bool_and(coalesce(bounded AND jsonb_typeof(data)='object' AND jsonb_typeof(transition->'state')='object' + AND transition->'state'->>'status' IN ({STATUSES}) + AND transition->'state'->>'accepted_sequence'=sequence::text + AND (previous_at IS NULL OR created_at>=previous_at),false)) AS coherent, + min(created_at) FILTER(WHERE sequence=1 AND type='started') AS first_started, + min(data->>'origin') FILTER(WHERE sequence=1 AND type='started' AND data->>'origin' IN ({ORIGINS})) AS + explicit_origin + FROM ordered GROUP BY run_id""" + + +def receipt_origins(connection, params): + query = [] + for receipt, source, foreign, extra, origin in ( + ("trigger_receipts", "trigger_routes", "trigger_id", "", "webhook"), + ("broker_events", "broker_routes", "trigger_id", "", "broker"), + ("email_receipts", "email_sources", "source_id", "AND r.state='dispatched' AND r.signal_id IS NULL", "email"), + ): + query.append( + f"SELECT r.run_id,'{origin}' AS origin FROM {receipt} r JOIN {source} s " + "ON s.tenant_id=r.tenant_id AND s.project_id=r.project_id AND s.environment_id=r.environment_id " + f"AND s.id=r.{foreign} WHERE r.tenant_id=:tenant AND r.project_id=:project " + "AND r.environment_id=:environment AND r.run_id=ANY(:ids) AND s.activation_id IS NOT NULL " + extra + ) + query.append( + "SELECT run_id,'schedule' AS origin FROM schedule_occurrences WHERE " + + SCOPE + + " AND run_id=ANY(:ids) AND kind='started'" + ) + query.append( + "SELECT cast(r.receipt->>'run_id' AS uuid),'provider' AS origin FROM provider_receipts r " + "JOIN provider_sources s ON s.tenant_id=r.tenant_id AND s.project_id=r.project_id " + "AND s.environment_id=r.environment_id AND s.id=r.source_id " + "WHERE r.tenant_id=:tenant AND r.project_id=:project AND r.environment_id=:environment " + "AND r.receipt->>'run_id'=ANY(:text_ids) AND r.state='dispatched' " + "AND s.activation_id IS NOT NULL AND r.receipt->>'signal_id' IS NULL" + ) + result: dict[UUID, set[str]] = {} + for row in connection.execute( + sa.text(" UNION ALL ".join(query)), {**params, "text_ids": [str(v) for v in params["ids"]]} + ).mappings(): + result.setdefault(row["run_id"], set()).add(row["origin"]) + return result + + +def backfill_incidents(connection, params, events) -> None: + scoped = "i.tenant_id=:tenant AND i.project_id=:project AND i.environment_id=:environment" + rows = connection.execute( + sa.text(f"""WITH evidence AS ( + SELECT run_id,type,data->>'node_id' AS node_id,coalesce(data->>'generation','0') AS generation, + data->>'code' AS code,CASE WHEN type='incident_resolved' THEN data->>'incident_key' + ELSE (data->>'node_id')||':'||coalesce(data->>'generation','0') END AS incident_key, + min(created_at) AS first_at,max(created_at) AS last_at + FROM run_events e WHERE {SCOPE} AND run_id=ANY(:ids) + AND type IN ('incident_opened','incident_resolved') + AND public.weave_run_events_bytes(e)<={FETCH_BYTES} + GROUP BY run_id,type,data->>'node_id',coalesce(data->>'generation','0'),data->>'code', + CASE WHEN type='incident_resolved' THEN data->>'incident_key' + ELSE (data->>'node_id')||':'||coalesce(data->>'generation','0') END + ) SELECT i.*,f.workflow_name,f.workflow_version,f.test, + min(e.first_at) FILTER(WHERE e.type='incident_opened') AS exact_opened, + max(e.last_at) AS last_transition FROM incidents i JOIN run_facts f + ON f.tenant_id=i.tenant_id AND f.project_id=i.project_id + AND f.environment_id=i.environment_id AND f.run_id=i.run_id + LEFT JOIN evidence e ON e.run_id=i.run_id AND e.node_id=i.node_id + AND e.generation=coalesce(i.generation,0)::text AND e.incident_key=i.incident_key + AND (e.type='incident_resolved' OR e.code=i.origin_code) + WHERE {scoped} AND i.run_id=ANY(:ids) + GROUP BY i.id,f.workflow_name,f.workflow_version,f.test"""), + params, + ).mappings() + for row in rows: + node, instance = None, "" + try: + if row["node_id"] is not None and not row["node_id"].startswith("@"): + TypeAdapter(InstanceKeyText).validate_python(row["node_id"]) + view = instance_view(row["node_id"]) + TypeAdapter(ResourceName).validate_python(view.node_id) + node, instance = view.node_id, view.instance_key + except (ValueError, TypeError): + pass + event = events.get(row["run_id"], {}) + opened = row["exact_opened"] or event.get("updated_at") + times = [value for value in (opened, row["last_transition"], row["resolved_at"]) if value is not None] + connection.execute( + sa.text( + "INSERT INTO incident_facts(tenant_id,project_id,environment_id,incident_id," + "node_id,instance_key,opened_at,updated_at,workflow_name,workflow_version,test,opened_at_estimated) " + "VALUES(:tenant,:project,:environment,:incident,:node,:instance,:opened,:updated," + ":name,:version,:test,:estimated)" + ), + { + **params, + "incident": row["id"], + "node": node, + "instance": instance, + "opened": opened, + "updated": max(times) if times else None, + "name": row["workflow_name"], + "version": row["workflow_version"], + "test": row["test"], + "estimated": row["exact_opened"] is None, + }, + ) + + +def backfill_scope(connection, params) -> None: + after = None + # Classifications of the artifacts already seen in this scope (see classify). + memo: dict[tuple[bytes, bool, bool], dict[str, Any]] = {} + while True: + ids = list( + connection.execute( + sa.text( + f"SELECT id FROM runs WHERE {SCOPE} " + "AND (cast(:after AS uuid) IS NULL OR id>:after) ORDER BY id LIMIT 10000" + ), + {**params, "after": after}, + ).scalars() + ) + if not ids: + break + batch = {**params, "ids": ids} + events = {row["run_id"]: dict(row) for row in connection.execute(sa.text(EVENTS), batch).mappings()} + origins = receipt_origins(connection, batch) + identities = connection.execute( + sa.text("""SELECT r.id,a.id AS activation_id,a.version_id AS definition_version_id, + a.name AS activation_name,a.revision AS activation_revision,v.name AS workflow_name,v.version AS + workflow_version, + CASE WHEN jsonb_typeof(r.state->'status')='string' THEN r.state->>'status' END AS status, + r.state->>'manual_paused' AS paused, + CASE WHEN jsonb_typeof(r.request->'business_key')='string' AND + char_length(r.request->>'business_key')<=200 THEN r.request->>'business_key' END AS business_key, + CASE WHEN jsonb_typeof(r.request->'correlation_key')='string' AND + char_length(r.request->>'correlation_key')<=200 THEN r.request->>'correlation_key' + END AS correlation_key, + l.parent_run_id,public.weave_runs_bytes(r) AS bytes, + (SELECT count(*) FROM incidents i WHERE i.tenant_id=r.tenant_id AND i.project_id=r.project_id + AND i.environment_id=r.environment_id AND i.run_id=r.id AND i.status='active') AS active_incidents + FROM runs r JOIN activation_revisions a ON a.tenant_id=r.tenant_id AND a.project_id=r.project_id + AND a.environment_id=r.environment_id AND a.id=r.activation_id + JOIN definition_versions v ON v.tenant_id=a.tenant_id AND v.project_id=a.project_id AND v.id=a.version_id + LEFT JOIN run_retry_links l ON l.tenant_id=r.tenant_id AND l.project_id=r.project_id + AND l.environment_id=r.environment_id AND l.run_id=r.id + WHERE r.tenant_id=:tenant AND r.project_id=:project AND r.environment_id=:environment AND r.id=ANY(:ids) + ORDER BY r.id"""), + batch, + ).mappings() + facts = [] + for identity in identities: + item = dict(identity) + event = events.get(item["id"], {}) + cached = classification({}) + state = None + if item["bytes"] <= FETCH_BYTES: + row = ( + connection.execute( + sa.text( + "SELECT state,activation,request,sha256(convert_to(artifact::text,'UTF8')) AS artifact_key " + f"FROM runs WHERE {SCOPE} AND id=:id" + ), + {**params, "id": item["id"]}, + ) + .mappings() + .one() + ) + cached = classify(connection, params, item["id"], dict(row), memo) + state = row["state"] + if not ( + event.get("coherent") + and event["first_sequence"] == 1 + and event["last_event_sequence"] == event["event_count"] + and event["first_started"] == event["started_at"] + and isinstance(state, dict) + and state.get("accepted_sequence") == event["last_event_sequence"] + and item["status"] == event["last_status"] + and ( + item["status"] not in ("succeeded", "failed", "cancelled", "timed_out") + or event["ended_at"] is not None + ) + ): + cached.update(classification_state="unavailable", node_kinds="{}") + if item["status"] not in ( + "queued", + "running", + "waiting", + "suspended", + "succeeded", + "failed", + "cancelled", + "timed_out", + ): + item["status"] = None + cached.update(classification_state="unavailable", node_kinds="{}") + event = {**event, "started_at": None, "updated_at": None, "ended_at": None} + matching = origins.get(item["id"], set()) + origin = "retry" if item["parent_run_id"] else event.get("explicit_origin") + if origin is None and len(matching) == 1: + origin = next(iter(matching)) + # A call without its scoped parent identity is not supported historical evidence. + if origin == "call": + origin = None + facts.append( + { + **params, + **item, + **cached, + "origin": origin, + "test": origin == "test", + "paused": item["paused"] == "true", + "started_at": event.get("started_at"), + "updated_at": event.get("updated_at"), + "ended_at": event.get("ended_at") + if item["status"] in ("succeeded", "failed", "cancelled", "timed_out") + else None, + "last_event_sequence": event.get("last_event_sequence", 0), + } + ) + connection.execute( + sa.text("""INSERT INTO run_facts(tenant_id,project_id,environment_id,run_id,definition_version_id, + workflow_name,workflow_version,activation_id,activation_name,activation_revision,status,paused,test,origin, + retried_from_run_id,business_key,correlation_key,started_at,updated_at,ended_at,active_incidents, + classification_state,classification_policy,pinned_ir_version,pinned_features,node_kinds,last_event_sequence) + VALUES(:tenant,:project,:environment,:id,:definition_version_id,:workflow_name,:workflow_version, + :activation_id,:activation_name,:activation_revision,:status,:paused,:test,:origin,:parent_run_id, + :business_key,:correlation_key,:started_at,:updated_at,:ended_at,:active_incidents,:classification_state, + :classification_policy,:pinned_ir_version,cast(:pinned_features AS jsonb),cast(:node_kinds AS + jsonb),:last_event_sequence)"""), + facts, + ) + # The facts table is new in this transaction and has no statistics: without them the next statements of + # this batch and every later batch plan as if it were empty and read every fact inserted so far. + connection.execute(sa.text("ANALYZE run_facts")) + backfill_incidents(connection, batch, events) + after = ids[-1] + connection.execute( + sa.text( + f"INSERT INTO worker_facts(tenant_id,project_id,environment_id,worker_id) " + f"SELECT tenant_id,project_id,environment_id,id FROM worker_instances WHERE {SCOPE}" + ), + params, + ) + + +def scoped_work(connection, *, refresh: bool = False) -> None: + for tenant in connection.execute(sa.text("SELECT weave_tenant_ids()")).scalars(): + connection.execute(sa.text("SELECT set_config('weave.tenant_id',:value,true)"), {"value": str(tenant)}) + for project in connection.execute( + sa.text("SELECT id FROM projects WHERE tenant_id=:tenant ORDER BY id"), {"tenant": tenant} + ).scalars(): + connection.execute(sa.text("SELECT set_config('weave.project_id',:value,true)"), {"value": str(project)}) + for environment in connection.execute( + sa.text("SELECT id FROM environments WHERE tenant_id=:tenant AND project_id=:project ORDER BY id"), + {"tenant": tenant, "project": project}, + ).scalars(): + connection.execute( + sa.text("SELECT set_config('weave.environment_id',:value,true)"), {"value": str(environment)} + ) + params = {"tenant": tenant, "project": project, "environment": environment} + if not refresh: + backfill_scope(connection, params) + continue + after = None + while True: + ids = list( + connection.execute( + sa.text( + f"SELECT id FROM runs WHERE {SCOPE} AND state->>'status' NOT IN ({TERMINAL}) " + "AND (cast(:after AS uuid) IS NULL OR id>:after) ORDER BY id LIMIT 10000" + ), + {**params, "after": after}, + ).scalars() + ) + if not ids: + break + connection.execute( + sa.text("SELECT weave_reserve_refresh('run',id,false) FROM unnest(cast(:ids AS uuid[])) id"), + {"ids": ids}, + ) + after = ids[-1] + + +def upgrade() -> None: + connection = op.get_bind() + if not connection.scalar(sa.text("SELECT has_function_privilege(current_user,'weave_tenant_ids()','EXECUTE')")): + raise RuntimeError("Migration requires explicit EXECUTE on weave_tenant_ids() for tenant-scoped backfill") + if connection.scalar(sa.text("SELECT to_regclass('public.run_call_links')")) is not None or connection.scalar( + sa.text( + "SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND " + "table_name='step_instances' AND column_name='handled')" + ) + ): + raise RuntimeError("Reconcile call and handled-failure history with the operational backfill before migrating") + previous = { + name: connection.scalar(sa.text("SELECT current_setting(:name,true)"), {"name": "weave." + name}) + for name in ("tenant_id", "project_id", "environment_id") + } + try: + with connection.begin_nested(): + tables() + indexes() + control_accounting() + scoped_work(connection) + for table, key in TABLE_KEYS.items(): + op.execute(f"""INSERT INTO operation_usage(tenant_id,project_id,environment_id,metric,value) + SELECT tenant_id,project_id,'00000000-0000-0000-0000-000000000000'::uuid,'ordinary_bytes', + sum((public.weave_row_metrics('{table}',to_jsonb(f))->>'ordinary_bytes')::bigint) + FROM {table} f GROUP BY tenant_id,project_id + ON CONFLICT(tenant_id,project_id,environment_id,metric) + DO UPDATE SET value=operation_usage.value+excluded.value""") + secure(table, key) + scoped_work(connection, refresh=True) + op.execute("UPDATE weave_schema_version SET version='0032_operations_facts'") + finally: + for name, value in previous.items(): + connection.execute( + sa.text("SELECT set_config(:name,:value,true)"), {"name": "weave." + name, "value": value or ""} + ) + + +def downgrade() -> None: + raise RuntimeError("Destructive downgrades are not supported") diff --git a/scripts/acceptance.py b/scripts/acceptance.py index 64418b5d..6ed7ab3b 100644 --- a/scripts/acceptance.py +++ b/scripts/acceptance.py @@ -73,13 +73,19 @@ IMAGE_CONTENT_FORMAT = "{{json .RootFS.Layers}}{{json .Config}}" CANARY_HANDLES = ("acme-api-key", "order-review-webhook") # The real-platform journeys move these tests to playwright.acceptance.config.ts; until then the quick-integration -# test runs against the platform this harness starts. +# test runs against the platform this harness starts. A suite with a capability runs once that capability has landed +# (journeys.toml). SUITES: dict[str, dict[str, Any]] = { "platform-session": { "spec": "tests/browser/real-platform.spec.ts", "grep": "quick integration", "profiles": ("pr", "nightly", "release"), }, + "operations": { + "spec": "tests/browser/operate-real.spec.ts", + "profiles": ("pr", "nightly", "release"), + "capability": "run-views", + }, } EGRESS_CHAIN = "WEAVE-ACCEPTANCE" SUBNET_POOL = ipaddress.IPv4Network("10.231.0.0/16") @@ -422,13 +428,18 @@ def run(plan: Plan, recorder: Recorder) -> None: ) if others: _journeys(plan, recorder, others, allow_empty=True) + enablement = acceptance_journeys.Enablement.load(JOURNEYS) for name, suite in SUITES.items(): if plan.profile not in suite["profiles"]: continue + if "capability" in suite and not enablement.capabilities.get(suite["capability"]): + continue env = { **os.environ, "WEAVE_E2E_PLATFORM_DIR": str(plan.platform_dir), "WEAVE_E2E_PERSON_FILE": str(plan.private / "people" / "builder.json"), + "WEAVE_E2E_VIEWER_FILE": str(plan.private / "people" / "viewer.json"), + "WEAVE_E2E_OPERATIONS_EVIDENCE": str(plan.evidence / "operations"), "WEAVE_E2E_ACME_ORIGIN": plan.state["origins"][0], "WEAVE_E2E_ACME_CONTROL": f"http://127.0.0.1:{plan.state['acme_control_port']}", "WEAVE_E2E_SHOTS": str(plan.evidence / "screens" / name), @@ -440,8 +451,7 @@ def run(plan: Plan, recorder: Recorder) -> None: "playwright", "test", suite["spec"], - "--grep", - suite["grep"], + *(["--grep", suite["grep"]] if "grep" in suite else []), "--retries", "0", "--reporter", diff --git a/scripts/check.py b/scripts/check.py index 1d5c7369..fe51cf86 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -126,8 +126,9 @@ def run(root: Path, evidence: Path, *, release: bool, context: str | None, integ ( "unit-contracts", [python, "-m", "pytest", "tests/unit", "tests/contracts", "-q", "--tb=short", "--show-capture=no"], - # More than 3,000 tests: about 4 minutes locally, up to 10 on a shared CI runner. - 1800, + # More than 7,000 tests: about 15 minutes locally and 15 to 30 on a shared CI + # runner, so the limit leaves room for a slow runner without hiding a hang. + 2700, ), *worker_stages(), ("prepare", [python, "scripts/prepare_release.py", "--output", str(evidence / "release")], 300), diff --git a/src/firefly_weave/access/authorization.py b/src/firefly_weave/access/authorization.py index 14ca3fac..3c36fb56 100644 --- a/src/firefly_weave/access/authorization.py +++ b/src/firefly_weave/access/authorization.py @@ -40,6 +40,22 @@ def contains(parent: Scope | None, child: Scope | None) -> bool: @service class AuthorizationService: + def allows( + self, principal: Principal, scope: Scope | None, capability: str, *, resource: str | None = None + ) -> bool: + """Whether `require` would allow this, without recording anything. + + For reads that only shape what they return, such as which sections of a summary to fill; the access itself + is still checked with `require`. + """ + allowed = principal.active and any( + capability in ROLE_CAPABILITIES[grant.role] + and contains(grant.scope, scope) + and (not grant.resources or resource in grant.resources) + for grant in principal.grants + ) + return allowed and not (principal.kind == "worker" and capability not in ROLE_CAPABILITIES["worker"]) + def require( self, principal: Principal, @@ -49,14 +65,7 @@ def require( resource: str | None = None, context: AuditContext | None = None, ) -> None: - allowed = principal.active and any( - capability in ROLE_CAPABILITIES[grant.role] - and contains(grant.scope, scope) - and (not grant.resources or resource in grant.resources) - for grant in principal.grants - ) - if principal.kind == "worker" and capability not in ROLE_CAPABILITIES["worker"]: - allowed = False + allowed = self.allows(principal, scope, capability, resource=resource) emit(event(principal, scope, capability, "allow" if allowed else "deny", context or AuditContext())) if not allowed: raise AccessDenied() diff --git a/src/firefly_weave/access/scheduler.py b/src/firefly_weave/access/scheduler.py index 3fc6b108..2e9a04c5 100644 --- a/src/firefly_weave/access/scheduler.py +++ b/src/firefly_weave/access/scheduler.py @@ -64,22 +64,32 @@ async def scopes(engine: AsyncEngine, uow: UnitOfWork) -> list[_SchedulerScope]: return result -async def tenant_page(engine: AsyncEngine, limit: int = 16) -> list[UUID]: - """Reserve a persistent bounded catalog turn; this grants no business capability.""" +async def tenant_page(engine: AsyncEngine, limit: int = 16, *, operations: bool = False) -> list[UUID]: + """Reserve a persistent bounded catalog turn; this grants no business capability. + + The operations traversal keeps its own tenant cursor, so it never moves the recovery traversal's. + """ + function = "weave_operations_tenants" if operations else "weave_scheduler_tenants" async with engine.begin() as connection: await connection.execute(text("SET LOCAL statement_timeout='4000ms'")) await connection.execute(text("SET LOCAL lock_timeout='1000ms'")) - return list( - (await connection.execute(text("SELECT weave_scheduler_tenants(:limit)"), {"limit": limit})).scalars() - ) + return list((await connection.execute(text(f"SELECT {function}(:limit)"), {"limit": limit})).scalars()) async def next_scope( - uow: UnitOfWork, tenant: UUID, *, broker: bool = False, outbox: bool = False, provider: bool = False + uow: UnitOfWork, + tenant: UUID, + *, + broker: bool = False, + outbox: bool = False, + provider: bool = False, + operations: bool = False, ) -> _SchedulerScope | None: """One environment per tenant turn, advanced before work so failures cannot pin traversal.""" table = ( - "provider_environment_cursors" + "operations_environment_cursors" + if operations + else "provider_environment_cursors" if provider else "outbox_environment_cursors" if outbox diff --git a/src/firefly_weave/api/operations.py b/src/firefly_weave/api/operations.py index 250e8b18..66d5d981 100644 --- a/src/firefly_weave/api/operations.py +++ b/src/firefly_weave/api/operations.py @@ -25,10 +25,21 @@ from firefly_weave.api.access import request_scope from firefly_weave.api.surface import operation -from firefly_weave.api.transport import canonical_path, etag, page_request, page_response, parse_revision +from firefly_weave.api.transport import ( + canonical_path, + decode_cursor, + decode_cursor_v2, + etag, + parse_query, + parse_revision, +) +from firefly_weave.contracts.access import Scope +from firefly_weave.contracts.incident_queries import IncidentQuery from firefly_weave.contracts.operations import CancelRunRequest, HistoryExport, IncidentResolution +from firefly_weave.contracts.public import UnavailableResource from firefly_weave.contracts.runtime import StartRunRequest from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.fact_positions import RunPosition, run_position from firefly_weave.operations.history import HistoryService from firefly_weave.operations.incidents import IncidentService from firefly_weave.runtime.service import RuntimeService @@ -48,18 +59,8 @@ def __init__(self, incidents: IncidentService, runtime: RuntimeService, history: async def list(self, request: Request) -> JSONResponse: scope = request_scope(request, environment=True) if canonical_path(request.url.path): - identifier = UUID(request.path_params["identifier"]) - collection = "incidents.run/" + str(identifier) - limit, cursor = page_request(request, scope, collection) - page = await self.incidents.list_all( - request.state.principal, - scope, - run_id=identifier, - limit=limit, - cursor=cursor, - context=request.state.audit_context, - ) - return JSONResponse(page_response(page, scope, collection, request.url.path)) + return await self._page(request, scope, UUID(request.path_params["identifier"])) + # The earlier route keeps its plain array of the run's incidents. async with self.runtime.definitions.transaction(scope, None, mutation=False) as tx: result = await self.incidents.list( tx, @@ -168,9 +169,45 @@ def _limit(request: Request, default: int) -> int: @operation("incidents.list") async def discover(self, request: Request) -> JSONResponse: + return await self._page(request, request_scope(request, environment=True), None) + + @operation("incidents.read") + async def read_incident(self, request: Request) -> JSONResponse: scope = request_scope(request, environment=True) - limit, cursor = page_request(request, scope, "incidents") + result = await self.incidents.read( + request.state.principal, + scope, + UUID(request.path_params["identifier"]), + context=request.state.audit_context, + ) + # The revision tag lets a reader resolve what it read; a withheld incident has no revision to offer. + headers = {} if isinstance(result, UnavailableResource) else {"ETag": etag(result.revision, request.url.path)} + return JSONResponse(result.model_dump(mode="json"), headers=headers) + + async def _page(self, request: Request, scope: Scope, run_id: UUID | None) -> JSONResponse: + query = parse_query(request.query_params, IncidentQuery) + canonical = canonical_path(request.url.path) + collection = query.cursor_collection(run_id) + position: RunPosition | UUID | None + if query.order == "id": + # Explicit ID order continues the earlier cursors: scope-bound here, raw IDs on the earlier route. + position = ( + decode_cursor(query.cursor, scope, collection) + if canonical + else UUID(query.cursor) + if query.cursor + else None + ) + else: + position = run_position(decode_cursor_v2(query.cursor, scope, collection)) result = await self.incidents.list_all( - request.state.principal, scope, limit=limit, cursor=cursor, context=request.state.audit_context + request.state.principal, + scope, + run_id=run_id, + query=query, + position=position, + context=request.state.audit_context, ) - return JSONResponse(page_response(result, scope, "incidents", request.url.path)) + if query.order == "id" and not canonical and result["next_cursor"] is not None: + result["next_cursor"] = str(decode_cursor(result["next_cursor"], scope, collection)) + return JSONResponse(result) diff --git a/src/firefly_weave/api/operations_summary.py b/src/firefly_weave/api/operations_summary.py new file mode 100644 index 00000000..adcdde07 --- /dev/null +++ b/src/firefly_weave/api/operations_summary.py @@ -0,0 +1,53 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Native routes for the environment summary and the project's telemetry settings.""" + +from pyfly.container.stereotypes import rest_controller +from pyfly.web import request_mapping +from starlette.requests import Request +from starlette.responses import JSONResponse + +from firefly_weave.api.access import request_scope +from firefly_weave.api.surface import operation +from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.summary import OperationsSummaryService + + +def _service(request: Request) -> OperationsSummaryService: + # The service belongs to the process lifespan: it carries this process's instance ID and loop health. + service = getattr(request.app.state, "operations_summary", None) + if not isinstance(service, OperationsSummaryService): + raise CatalogError(503, "WV-UNAVAILABLE", "The platform summary is not available yet") + return service + + +@rest_controller +@request_mapping("") +class OperationsSummaryController: + @operation("operations_summary.read") + async def summary(self, request: Request) -> JSONResponse: + result = await _service(request).read( + request.state.principal, request_scope(request, environment=True), context=request.state.audit_context + ) + return JSONResponse(result.model_dump(mode="json")) + + @operation("operations_telemetry.read") + async def telemetry(self, request: Request) -> JSONResponse: + result = await _service(request).telemetry( + request.state.principal, request_scope(request), context=request.state.audit_context + ) + return JSONResponse(result.model_dump(mode="json")) diff --git a/src/firefly_weave/api/runs.py b/src/firefly_weave/api/runs.py index 8384b0e9..dee83422 100644 --- a/src/firefly_weave/api/runs.py +++ b/src/firefly_weave/api/runs.py @@ -25,9 +25,11 @@ from firefly_weave.api.access import request_scope from firefly_weave.api.surface import operation -from firefly_weave.api.transport import page_request, page_response +from firefly_weave.api.transport import canonical_path, decode_cursor, decode_cursor_v2, parse_query from firefly_weave.contracts.human_tasks import ManualControlRequest -from firefly_weave.contracts.runtime import RunListFilters, SignalRequest, StartRunRequest +from firefly_weave.contracts.run_views import RunListQuery +from firefly_weave.contracts.runtime import SignalRequest, StartRunRequest +from firefly_weave.operations.fact_positions import RunPosition, run_position from firefly_weave.runtime.service import RuntimeService from firefly_weave.runtime.signals import SignalService @@ -82,25 +84,24 @@ async def signal(self, request: Request) -> JSONResponse: @operation("runs.list") async def discover(self, request: Request) -> JSONResponse: scope = request_scope(request, environment=True) - values: dict[str, str | bool] = { - key: request.query_params[key] for key in RunListFilters.model_fields if key in request.query_params - } - if "include_archived" in values: - if values["include_archived"] not in {"true", "false"}: - raise ValueError("include_archived must be true or false") - values["include_archived"] = values["include_archived"] == "true" - filters = RunListFilters.model_validate(values) - collection = filters.cursor_collection() - limit, cursor = page_request(request, scope, collection) + query = parse_query(request.query_params, RunListQuery) + position: RunPosition | UUID | None = None + if query.order == "id": + position = ( + decode_cursor(query.cursor, scope, query.cursor_collection()) + if canonical_path(request.url.path) + else UUID(query.cursor) + if query.cursor + else None + ) + else: + position = run_position(decode_cursor_v2(query.cursor, scope, query.cursor_collection())) result = await self.service.list( - request.state.principal, - scope, - limit=limit, - cursor=cursor, - filters=filters, - context=request.state.audit_context, + request.state.principal, scope, query=query, position=position, context=request.state.audit_context ) - return JSONResponse(page_response(result, scope, collection, request.url.path)) + if query.order == "id" and not canonical_path(request.url.path) and result["next_cursor"] is not None: + result["next_cursor"] = str(decode_cursor(result["next_cursor"], scope, query.cursor_collection())) + return JSONResponse(result) @operation("runs.pause") async def pause(self, request: Request) -> JSONResponse: diff --git a/src/firefly_weave/api/transport.py b/src/firefly_weave/api/transport.py index 4c72b17e..fc07d1b4 100644 --- a/src/firefly_weave/api/transport.py +++ b/src/firefly_weave/api/transport.py @@ -117,7 +117,10 @@ def decode_sequence_cursor(value: str | None, scope: Scope, collection: str) -> def encode_cursor_v2(scope: Scope, collection: str, sort_value: JsonValue, identifier: str) -> str: """Time-ordered cursor: base64url of [2, scope, collection, sort value, identifier].""" raw = json.dumps([2, scope.model_dump(mode="json"), collection, sort_value, identifier], separators=(",", ":")) - return base64.urlsafe_b64encode(raw.encode()).decode().rstrip("=") + cursor = base64.urlsafe_b64encode(raw.encode()).decode().rstrip("=") + if len(cursor) > CURSOR_V2_LIMIT: + raise ValueError("Invalid scope-bound cursor") + return cursor def decode_cursor_v2(value: str | None, scope: Scope, collection: str) -> tuple[JsonValue, str] | None: @@ -138,7 +141,7 @@ def decode_cursor_v2(value: str | None, scope: Scope, collection: str) -> tuple[ if not isinstance(identifier, str) or encode_cursor_v2(scope, collection, sort_value, identifier) != value: raise ValueError() return sort_value, identifier - except (ValueError, TypeError, KeyError): + except (ValueError, TypeError, KeyError, RecursionError): raise ValueError("Invalid scope-bound cursor") from None diff --git a/src/firefly_weave/api/workers.py b/src/firefly_weave/api/workers.py index 2d0347dc..0f8d2391 100644 --- a/src/firefly_weave/api/workers.py +++ b/src/firefly_weave/api/workers.py @@ -26,7 +26,8 @@ from firefly_weave.api.access import request_scope from firefly_weave.api.surface import operation -from firefly_weave.api.transport import page_request, page_response +from firefly_weave.api.transport import decode_cursor_v2, page_request, page_response, parse_query +from firefly_weave.contracts.worker_queries import ReleaseUsageQuery, RetireStaleRequest, WorkerListQuery from firefly_weave.contracts.workers import ( ClaimRequest, CompleteRequest, @@ -38,6 +39,7 @@ ReleaseRequest, WorkerControlRequest, ) +from firefly_weave.operations.fact_positions import run_position from firefly_weave.workers.leases import TaskService from firefly_weave.workers.service import WorkerService @@ -87,6 +89,17 @@ async def revoke(self, request: Request) -> JSONResponse: ) return JSONResponse({"revoked": True}) + @operation("workers.retire_stale") + async def retire_stale(self, request: Request) -> JSONResponse: + result = await self.service.retire_stale( + request.state.principal, + request_scope(request, environment=True), + RetireStaleRequest.model_validate_json(await request.body()), + request.headers.get("Idempotency-Key", ""), + context=request.state.audit_context, + ) + return JSONResponse(result.model_dump(mode="json"), headers={"Cache-Control": "no-store"}) + @operation("workers.drain") async def drain(self, request: Request) -> JSONResponse: return await self.control(request, draining=True) @@ -171,11 +184,12 @@ async def credentials(self, request: Request) -> JSONResponse: @operation("workers.list") async def discover(self, request: Request) -> JSONResponse: scope = request_scope(request, environment=True) - limit, cursor = page_request(request, scope, "workers") + query = parse_query(request.query_params, WorkerListQuery) + position = run_position(decode_cursor_v2(query.cursor, scope, query.cursor_collection())) result = await self.service.list( - request.state.principal, scope, limit=limit, cursor=cursor, context=request.state.audit_context + request.state.principal, scope, query=query, position=position, context=request.state.audit_context ) - return JSONResponse(page_response(result, scope, "workers", request.url.path)) + return JSONResponse(result) @operation("releases.list") async def releases(self, request: Request) -> JSONResponse: @@ -191,6 +205,17 @@ async def releases(self, request: Request) -> JSONResponse: ) return JSONResponse(page_response(result, scope, "worker-releases", request.url.path)) + @operation("releases.usage") + async def release_usage(self, request: Request) -> JSONResponse: + result = await self.service.release_usage( + request.state.principal, + request_scope(request, environment=True), + UUID(request.path_params["identifier"]), + parse_query(request.query_params, ReleaseUsageQuery), + context=request.state.audit_context, + ) + return JSONResponse(result.model_dump(mode="json")) + @operation("workers.read") async def read(self, request: Request) -> JSONResponse: result = await self.service.read( diff --git a/src/firefly_weave/app.py b/src/firefly_weave/app.py index 4be74071..fd40b411 100644 --- a/src/firefly_weave/app.py +++ b/src/firefly_weave/app.py @@ -19,10 +19,11 @@ import asyncio import logging import os +import time from collections.abc import AsyncIterator from contextlib import asynccontextmanager from pathlib import Path -from uuid import UUID +from uuid import UUID, uuid4 from pyfly.client.ports.outbound import BoundedHttpClientPort from pyfly.container.bean import bean @@ -56,6 +57,7 @@ from firefly_weave.api.human_files import HumanFileController from firefly_weave.api.lumi import LumiController from firefly_weave.api.operations import OperationsController +from firefly_weave.api.operations_summary import OperationsSummaryController from firefly_weave.api.providers import ProviderController from firefly_weave.api.run_views import RunViewController from firefly_weave.api.runs import RunController @@ -102,10 +104,13 @@ from firefly_weave.operations.event_delivery import OutboxDispatcher from firefly_weave.operations.history import HistoryService from firefly_weave.operations.incidents import IncidentService +from firefly_weave.operations.loop import OperationsLoop +from firefly_weave.operations.loop_health import LoopHealth, register_background_loops from firefly_weave.operations.lumi import LumiService from firefly_weave.operations.lumi_gateway import LumiGatewayClient from firefly_weave.operations.outbox_loop import OutboxLoop from firefly_weave.operations.run_views import RunViewService +from firefly_weave.operations.summary import OperationsSummaryService from firefly_weave.operations.telemetry import TelemetryService from firefly_weave.persistence.resources import DatabaseResources from firefly_weave.persistence.uow import UnitOfWork @@ -263,8 +268,15 @@ async def lifespan(app: Starlette) -> AsyncIterator[None]: dispatcher = None kafka_loop = None provider_loop = None + operations_loop = None effects_open = False compatibility = None + # One registry for this process's background loops; every loop reports into it. + health = LoopHealth() + register_background_loops(health, settings) + app.state.loop_health = health + # The summary names the process that answered and how long it has been up. + instance, started = uuid4(), time.monotonic() try: if settings.broker.enabled: require_driver() @@ -357,12 +369,24 @@ async def refresh_models( RunViewService, RunViewController, WorkerController, + OperationsSummaryController, ): pyfly.context.get_bean(required) app.state.authentication = pyfly.context.get_bean(AuthenticationService) await resources.check_startup() compatibility = pyfly.context.get_bean(CompatibilityService) app.state.compatibility = compatibility + app.state.operations_summary = OperationsSummaryService( + pyfly.context.get_bean(UnitOfWork), + pyfly.context.get_bean(AuthorizationService), + settings, + health, + compatibility, + drops=telemetry_drops, + instance=instance, + started_monotonic=started, + native=lambda: getattr(app.state, "dispatcher", None), + ) app.state.telemetry_service = pyfly.context.get_bean(TelemetryService) registry.set_operational_guard(lambda: compatibility.ready, retry_after=compatibility.retry_after_seconds) @@ -376,6 +400,7 @@ async def open_control_loop(*, required: bool = False) -> None: pyfly.context.get_bean(RecoveryService), pyfly.context.get_bean(Scheduler), operational=lambda: compatibility.ready, + health=health, ) try: await candidate.open() @@ -389,15 +414,20 @@ async def open_control_loop(*, required: bool = False) -> None: app.state.recovery_loop = recovery_loop async def open_effects() -> None: - nonlocal outbox_loop, dispatcher, kafka_loop, provider_loop, effects_open + nonlocal outbox_loop, dispatcher, kafka_loop, provider_loop, operations_loop, effects_open if effects_open: return - if any(loop is not None for loop in (outbox_loop, dispatcher, kafka_loop, provider_loop)): + if any( + loop is not None for loop in (outbox_loop, dispatcher, kafka_loop, provider_loop, operations_loop) + ): await close_effects() await open_control_loop(required=True) try: outbox_loop = OutboxLoop( - settings, pyfly.context.get_bean(UnitOfWork), pyfly.context.get_bean(OutboxDispatcher) + settings, + pyfly.context.get_bean(UnitOfWork), + pyfly.context.get_bean(OutboxDispatcher), + health=health, ) await outbox_loop.open() app.state.outbox_loop = outbox_loop @@ -414,6 +444,7 @@ async def open_effects() -> None: pyfly.context.get_bean(UnitOfWork), pyfly.context.get_bean(KafkaTrigger), pyfly.context.get_bean(BrokerClients), + health=health, ) await kafka_loop.open() app.state.kafka_loop = kafka_loop @@ -422,9 +453,13 @@ async def open_effects() -> None: pyfly.context.get_bean(UnitOfWork), pyfly.context.get_bean(ProviderDispatcher), pyfly.context.get_bean(EmailSourceService), + health=health, ) await provider_loop.open() app.state.provider_loop = provider_loop + operations_loop = OperationsLoop(settings, pyfly.context.get_bean(UnitOfWork), health) + await operations_loop.open() + app.state.operations_loop = operations_loop except BaseException: try: await close_effects() @@ -434,11 +469,14 @@ async def open_effects() -> None: effects_open = True async def close_effects() -> None: - nonlocal outbox_loop, dispatcher, kafka_loop, provider_loop, effects_open + nonlocal outbox_loop, dispatcher, kafka_loop, provider_loop, operations_loop, effects_open effects_open = False - remaining = [provider_loop, kafka_loop, dispatcher, outbox_loop] + # The operations loop opened last, so it closes first. + remaining = [operations_loop, provider_loop, kafka_loop, dispatcher, outbox_loop] cleanup_error: BaseException | None = None - for index, name in enumerate(("provider_loop", "kafka_loop", "dispatcher", "outbox_loop")): + for index, name in enumerate( + ("operations_loop", "provider_loop", "kafka_loop", "dispatcher", "outbox_loop") + ): loop = remaining[index] if loop is not None: try: @@ -449,10 +487,11 @@ async def close_effects() -> None: remaining[index] = None setattr(app.state, name, remaining[index]) # Failed owners remain reachable for the next refresh or final cleanup. - provider_loop = None if remaining[0] is None else provider_loop - kafka_loop = None if remaining[1] is None else kafka_loop - dispatcher = None if remaining[2] is None else dispatcher - outbox_loop = None if remaining[3] is None else outbox_loop + operations_loop = None if remaining[0] is None else operations_loop + provider_loop = None if remaining[1] is None else provider_loop + kafka_loop = None if remaining[2] is None else kafka_loop + dispatcher = None if remaining[3] is None else dispatcher + outbox_loop = None if remaining[4] is None else outbox_loop if cleanup_error is not None: raise cleanup_error await open_control_loop() @@ -472,6 +511,11 @@ async def close_effects() -> None: await compatibility.close() except BaseException as error: cleanup_error = error + if operations_loop is not None: + try: + await operations_loop.close() + except BaseException as error: + cleanup_error = error if provider_loop is not None: try: await provider_loop.close() diff --git a/src/firefly_weave/cli/cluster_operations.py b/src/firefly_weave/cli/cluster_operations.py index 785178ad..5e508b16 100644 --- a/src/firefly_weave/cli/cluster_operations.py +++ b/src/firefly_weave/cli/cluster_operations.py @@ -26,7 +26,7 @@ import click -from firefly_weave.cli.remote import family, machine_result +from firefly_weave.cli.remote import command, family, machine_result @click.group("operations") @@ -35,6 +35,8 @@ def operations() -> None: Observe a target, set desired components, create a plan, review and approve its digest, then apply it. The runner keeps provider credentials locally. + `summary` and `telemetry` read the environment's platform health and the + project's telemetry settings. """ @@ -47,6 +49,8 @@ def operations() -> None: ("runners", "deployment_runners"), ): operations.add_command(family(name, prefix)) +operations.add_command(command("operations_summary.read", "summary")) +operations.add_command(command("operations_telemetry.read", "telemetry")) @operations.group("runner") diff --git a/src/firefly_weave/connectors/dispatcher.py b/src/firefly_weave/connectors/dispatcher.py index 93b7cff0..2df5034e 100644 --- a/src/firefly_weave/connectors/dispatcher.py +++ b/src/firefly_weave/connectors/dispatcher.py @@ -36,6 +36,7 @@ from firefly_weave.contracts.workers import InstanceRequest from firefly_weave.operations.execution import ReservedSlots from firefly_weave.sdk.worker import Worker +from firefly_weave.sdk.worker_identity import worker_identity from firefly_weave.workers.repository import WorkerRepository LOCAL_BUILD_FORMAT = "weave/local-build-v1" @@ -147,6 +148,8 @@ async def open(self) -> None: raise RuntimeError("Native executors must share one build attestation") if self.image_digest is None: raise RuntimeError("Native dispatcher requires its operator-provisioned immutable image identity") + # Native executors run inside the API process; each registration reports that once. + identity = worker_identity(runtime="embedded") for entry in self.entries: async with self.service.uow.open(entry.scope) as tx: actor = await self.service.access.load_principal(entry.principal_id, tx=tx) @@ -170,7 +173,12 @@ async def open(self) -> None: instance = await self.service.tasks.workers.register_instance( actor, entry.scope, - InstanceRequest(release_id=entry.release_id, task_types=entry.task_types, capacity=entry.capacity), + InstanceRequest( + release_id=entry.release_id, + task_types=entry.task_types, + capacity=entry.capacity, + identity=identity, + ), context=AuditContext(), tx=tx, ) diff --git a/src/firefly_weave/contracts/deployments.py b/src/firefly_weave/contracts/deployments.py index b49552c6..669959f3 100644 --- a/src/firefly_weave/contracts/deployments.py +++ b/src/firefly_weave/contracts/deployments.py @@ -24,6 +24,7 @@ from firefly_weave.contracts.access import Scope from firefly_weave.contracts.catalog import Digest from firefly_weave.contracts.definitions import ContractModel +from firefly_weave.contracts.workers import WorkerHost # Names are data identifiers, never paths, shell fragments, or executable arguments. type DeploymentName = Annotated[str, Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,99}$")] @@ -110,6 +111,24 @@ class Deployment(DeploymentRequest): created_at: AwareDatetime +class ObservedMember(ContractModel): + """One container or pod of an observed worker component, as the runner saw it. + + `host` is the hostname the member's worker reports: the container ID prefix for Compose + and the pod name for Kubernetes. Members describe placement only; they never identify or + authorize a worker. + """ + + host: WorkerHost + ordinal: int | None = Field(ge=0, le=1000000) + # The adapter's own lowercase state, such as running, exited, pending or terminating. + state: Annotated[str, Field(pattern=r"^[a-z][a-z0-9_-]{0,31}$")] + ready: bool + started_at: AwareDatetime | None + restarts: int = Field(ge=0, le=2147483647) + node: WorkerHost | None + + class ObservedResource(ContractModel): name: DeploymentName external_identity: ExternalIdentity @@ -120,6 +139,8 @@ class ObservedResource(ContractModel): version: ExternalIdentity ownership: Literal["managed", "imported", "unknown"] state: Literal["ready", "progressing", "stopped", "failed", "unknown"] + # Left out when empty, so observations without members keep their bytes and digests. + members: list[ObservedMember] = Field(default_factory=list, max_length=100, exclude_if=lambda v: not v) class ObservationReport(ContractModel): diff --git a/src/firefly_weave/contracts/incident_queries.py b/src/firefly_weave/contracts/incident_queries.py new file mode 100644 index 00000000..326fc5c4 --- /dev/null +++ b/src/firefly_weave/contracts/incident_queries.py @@ -0,0 +1,89 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Incident list queries for `incidents.list` and `incidents.run_list`. + +Lists are newest opened first by default; incidents with an unknown opening time come last. Explicit ID order without +filters keeps the cursors issued before incidents had filters. `node_id` is a static step ID and matches every +instance of that step. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Annotated, Literal +from uuid import UUID + +from pydantic import AwareDatetime, Field, field_validator, model_validator + +from firefly_weave.contracts.definitions import ContractModel, ResourceName +from firefly_weave.contracts.run_views import ( + MAX_TIME_RANGE, + Cursor, + ErrorCode, + SummaryLimit, + _digest, + _filter_error, + utc_time, +) + +type IncidentStatus = Literal["active", "closed", "resolved"] +type IncidentOrder = Literal["opened_desc", "opened_asc", "id"] +type IncidentStatusFilter = Annotated[list[IncidentStatus], Field(max_length=3)] + + +class IncidentQuery(ContractModel): + """Incident list query parameters; the cursor is bound to every filter, the order and the run.""" + + status: IncidentStatusFilter = Field(default_factory=list) + code: ErrorCode | None = None + workflow: ResourceName | None = None + node_id: ResourceName | None = None + opened_after: AwareDatetime | None = None + opened_before: AwareDatetime | None = None + include_test: bool = False + order: IncidentOrder = "opened_desc" + limit: SummaryLimit = 50 + cursor: Cursor | None = None + + @field_validator("status") + @classmethod + def canonical_status(cls, value: list[IncidentStatus]) -> list[IncidentStatus]: + return sorted(set(value)) + + @field_validator("opened_after", "opened_before", mode="before") + @classmethod + def utc(cls, value: object) -> datetime | None: + return utc_time(value) + + @model_validator(mode="after") + def coherent(self) -> IncidentQuery: + if self.opened_after is not None and self.opened_before is not None: + if self.opened_after >= self.opened_before: + raise _filter_error("opened_after must be earlier than opened_before") + if self.opened_before - self.opened_after > MAX_TIME_RANGE: + raise _filter_error("The time range is longer than 400 days") + return self + + def unfiltered(self) -> bool: + return not self.model_dump(exclude={"order", "limit", "cursor"}, exclude_defaults=True) + + def cursor_collection(self, run_id: UUID | None = None) -> str: + collection = "incidents" if run_id is None else f"incidents.run/{run_id}" + if self.order == "id" and self.unfiltered(): + return collection + filters = self.model_dump(mode="json", exclude={"order", "limit", "cursor"}) + return f"{collection}:{_digest(filters)}:{self.order}" diff --git a/src/firefly_weave/contracts/operations.py b/src/firefly_weave/contracts/operations.py index d5731076..48b789eb 100644 --- a/src/firefly_weave/contracts/operations.py +++ b/src/firefly_weave/contracts/operations.py @@ -16,15 +16,17 @@ """Operator requests and immutable incident resolution receipts.""" -from typing import Literal +from typing import Literal, Self from uuid import UUID -from pydantic import AwareDatetime, Field, field_validator +from pydantic import AwareDatetime, Field, field_validator, model_validator from firefly_weave.contracts.access import Scope from firefly_weave.contracts.catalog import Digest -from firefly_weave.contracts.definitions import ContractModel +from firefly_weave.contracts.definitions import ContractModel, ResourceName, SemVer from firefly_weave.contracts.diagnostics import SourceRange +from firefly_weave.contracts.instance_keys import InstanceKeyTextOrEmpty +from firefly_weave.contracts.run_views import RunStatus, _same_step from firefly_weave.contracts.values import JsonData from firefly_weave.operations.redaction import Omission, SafeProjection from firefly_weave.runtime.models import RunState, RuntimeEvent @@ -45,7 +47,20 @@ def nonblank(cls, value: str | None) -> str | None: return value +class IncidentWorkflow(ContractModel): + name: ResourceName + version: SemVer + + class IncidentView(ContractModel): + """An incident. Read from operational facts, `node_id` is the static step ID and `instance_key` the full key. + + An incident on a control step (a branch output or a join) keeps its stored `@` step ID and has no instance key. + The fields after `external_effects_may_continue` are additive: each is left out while it is unknown, false or + empty, so earlier payloads keep their bytes. `opened_at_estimated` marks an opening time inferred from the run's + last recorded event because the incident's own opening event was not retained. + """ + id: UUID run_id: UUID incident_key: str @@ -59,6 +74,23 @@ class IncidentView(ContractModel): resolved_at: AwareDatetime | None = None resolution: IncidentResolution | None = None external_effects_may_continue: bool = False + instance_key: InstanceKeyTextOrEmpty = Field(default="", exclude_if=lambda value: not value) + opened_at: AwareDatetime | None = Field(default=None, exclude_if=lambda value: value is None) + opened_at_estimated: bool = Field(default=False, exclude_if=lambda value: not value) + updated_at: AwareDatetime | None = Field(default=None, exclude_if=lambda value: value is None) + workflow: IncidentWorkflow | None = Field(default=None, exclude_if=lambda value: value is None) + run_status: RunStatus | None = Field(default=None, exclude_if=lambda value: value is None) + test: bool = Field(default=False, exclude_if=lambda value: not value) + + @model_validator(mode="after") + def coherent(self) -> Self: + if self.instance_key: + if self.node_id is None: + raise ValueError("A run-level incident has no instance_key") + _same_step(self.node_id, self.instance_key) + if self.opened_at_estimated and self.opened_at is None: + raise ValueError("Only a known opening time can be estimated") + return self class CancelRunRequest(ContractModel): diff --git a/src/firefly_weave/contracts/operations_summary.py b/src/firefly_weave/contracts/operations_summary.py new file mode 100644 index 00000000..9a18e0d1 --- /dev/null +++ b/src/firefly_weave/contracts/operations_summary.py @@ -0,0 +1,153 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Bounded environment summary and telemetry configuration answers. + +A section the caller may not read is ``{"state": "forbidden"}`` and a section the server could not compute in time +is ``{"state": "unavailable"}``; neither is ever reported as zero. Run figures leave out test runs. +""" + +from typing import Annotated, Literal +from uuid import UUID + +from pydantic import AwareDatetime, Field + +from firefly_weave.contracts.definitions import ContractModel + +type Count = Annotated[int, Field(ge=0)] +type Seconds = Annotated[float, Field(ge=0)] +type HealthState = Literal["ok", "degraded", "down", "disabled"] +type HealthComponentName = Literal[ + "database", + "recovery_loop", + "schedules", + "outbox_loop", + "provider_loop", + "kafka_loop", + "operations_loop", + "alert_evaluator", + "native_executors", + "telemetry", +] +type AttentionKind = Literal["job", "approval", "worker", "incident", "runner", "compatibility", "run"] +type AttentionTitle = Annotated[str, Field(min_length=1, max_length=200)] +type Host = Annotated[str, Field(min_length=1, max_length=253)] + +ATTENTION_LIMIT = 20 +HEALTH_LIMIT = 16 +# The one attention item that is not a record: the platform's compatibility verdict. +COMPATIBILITY_ITEM: Literal["compatibility"] = "compatibility" + + +class ForbiddenSection(ContractModel): + state: Literal["forbidden"] + + +class UnavailableSection(ContractModel): + state: Literal["unavailable"] + + +class CompatibilitySummary(ContractModel): + mode: Literal["ready", "restricted"] + findings: Count + + +class ApiSection(ContractModel): + version: str = Field(min_length=1, max_length=64) + build: str | None = Field(pattern=r"^sha256:[a-f0-9]{64}$") + instance: UUID + uptime_seconds: Seconds + compatibility: CompatibilitySummary + + +class HealthComponent(ContractModel): + component: HealthComponentName + state: HealthState + last_started_at: AwareDatetime | None = Field(default=None, exclude_if=lambda value: value is None) + last_ok_at: AwareDatetime | None = Field(default=None, exclude_if=lambda value: value is None) + consecutive_failures: Count | None = Field(default=None, exclude_if=lambda value: value is None) + recent_failures: Count | None = Field(default=None, exclude_if=lambda value: value is None) + complete_until: AwareDatetime | None = Field(default=None, exclude_if=lambda value: value is None) + latency_ms: Seconds | None = Field(default=None, exclude_if=lambda value: value is None) + + +class LoadSection(ContractModel): + runs_active: Count + tasks_ready: Count + tasks_leased: Count + oldest_ready_seconds: Seconds | None + waits_active: Count + waits_overdue: Count + oldest_overdue_seconds: Seconds | None + outbox_pending: Count + outbox_oldest_seconds: Seconds | None + provider_pending: Count + sources_enabled: Count + + +class WorkersSection(ContractModel): + online: Count + offline: Count + draining: Count + capacity: Count + active_leases: Count + + +class TargetsSection(ContractModel): + total: Count + runner_online: Count + busy: Count + reconciliation_required: Count + awaiting_approval: Count + + +class IncidentsSection(ContractModel): + active: Count + opened_24h: Count + + +class AttentionItem(ContractModel): + kind: AttentionKind + id: UUID | Literal["compatibility"] + title: AttentionTitle + expires_at: AwareDatetime | None = Field(default=None, exclude_if=lambda value: value is None) + + +class OperationsSummary(ContractModel): + observed_at: AwareDatetime + api: ApiSection + health: list[HealthComponent] = Field(max_length=HEALTH_LIMIT) + load: LoadSection | UnavailableSection + workers: WorkersSection | UnavailableSection + targets: TargetsSection | ForbiddenSection | UnavailableSection + incidents: IncidentsSection | ForbiddenSection | UnavailableSection + attention: Annotated[list[AttentionItem], Field(max_length=ATTENTION_LIMIT)] | UnavailableSection + + +class TelemetryEndpoint(ContractModel): + configured: bool + host: Host | None + + +class TelemetryMetricEndpoint(TelemetryEndpoint): + interval_seconds: Seconds | None + + +class OperationsTelemetry(ContractModel): + enabled: bool + traces: TelemetryEndpoint + metrics: TelemetryMetricEndpoint + dropped: Count diff --git a/src/firefly_weave/contracts/run_views.py b/src/firefly_weave/contracts/run_views.py index 63ebbfbf..d6e92072 100644 --- a/src/firefly_weave/contracts/run_views.py +++ b/src/firefly_weave/contracts/run_views.py @@ -26,8 +26,9 @@ import hashlib import json +import re from datetime import UTC, datetime, timedelta -from typing import Annotated, Literal +from typing import TYPE_CHECKING, Annotated, Literal from uuid import UUID from pydantic import AwareDatetime, Field, field_validator, model_validator @@ -39,9 +40,15 @@ from firefly_weave.contracts.values import FiniteFloat, JsonData, SafeInteger from firefly_weave.operations.redaction import Omission +if TYPE_CHECKING: + from firefly_weave.contracts.runtime import RunListFilters + type RunStatus = Literal["queued", "running", "waiting", "suspended", "succeeded", "failed", "cancelled", "timed_out"] type RunOrigin = Literal["manual", "webhook", "schedule", "broker", "email", "provider", "retry", "call", "test"] type RunSummaryOrder = Literal["started_desc", "started_asc", "updated_desc"] +type RunListOrder = Literal["id", "started_desc", "started_asc", "updated_desc"] +type HandledErrorCount = Annotated[int, Field(ge=0)] +type StepHandledFailure = Literal["continue", "errorOutput"] type StepKind = Literal[ "action", "llm", @@ -78,6 +85,26 @@ type LogFieldValue = Annotated[str, Field(max_length=256)] | SafeInteger | FiniteFloat | bool | None +_TIMESTAMP = re.compile( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}[Tt][0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?(?:[Zz]|[+-][0-9]{2}:[0-9]{2})" +) + + +def utc_time(value: object) -> datetime | None: + """Parse a bounded RFC 3339 timestamp into a representable UTC instant.""" + if value is None: + return None + if type(value) is not str or len(value) > 64 or _TIMESTAMP.fullmatch(value) is None: + raise ValueError("Expected a timestamp with an offset") + if value[-1] not in "Zz" and (int(value[-5:-3]) > 23 or int(value[-2:]) > 59): + raise ValueError("Expected a valid timestamp") + try: + parsed = datetime.fromisoformat(value.upper().replace("Z", "+00:00")) + return parsed.astimezone(UTC) + except (ValueError, OverflowError): + raise ValueError("Expected a valid timestamp") from None + + def _same_step(node_id: str, instance_key: str) -> None: if instance_key and (instance_key == node_id or node_of(instance_key) != node_id): raise ValueError("instance_key must be empty or a longer key of the same step") @@ -147,6 +174,7 @@ class RunSummary(ContractModel): failed_step: FailedStep | None active_incidents: Count archived: bool + handled_errors: HandledErrorCount = Field(default=0, exclude_if=lambda value: value == 0) @model_validator(mode="after") def coherent(self) -> RunSummary: @@ -191,6 +219,7 @@ class StepFact(ContractModel): child_run_id: UUID | None log_entries: Count omissions: list[Omission] = Field(default_factory=list, max_length=8, exclude_if=lambda value: not value) + handled: StepHandledFailure | None = Field(default=None, exclude_if=lambda value: value is None) @model_validator(mode="after") def coherent(self) -> StepFact: @@ -202,6 +231,12 @@ def coherent(self) -> StepFact: raise ValueError("duration_ms is set exactly when the step started and ended") if (self.queue_wait_ms is not None) != (self.scheduled_at is not None and self.started_at is not None): raise ValueError("queue_wait_ms is set exactly when the step was scheduled and started") + if self.handled is not None and self.status != "failed": + raise ValueError("Only a failed step can have a handled failure") + if self.started_at and self.ended_at and self.ended_at < self.started_at: + raise ValueError("A step cannot end before it started") + if self.scheduled_at and self.started_at and self.started_at < self.scheduled_at: + raise ValueError("A step cannot start before it was scheduled") return self @@ -279,6 +314,7 @@ class RunSummaryQuery(ContractModel): business_key: BusinessKey | None = None correlation_key: BusinessKey | None = None has_active_incident: bool | None = None + has_handled_errors: bool | None = None include_archived: bool = False activation_id: UUID | None = None order: RunSummaryOrder = "started_desc" @@ -290,16 +326,10 @@ class RunSummaryQuery(ContractModel): def canonical_status(cls, value: list[RunStatus]) -> list[RunStatus]: return sorted(set(value)) - @field_validator("started_after", "started_before") + @field_validator("started_after", "started_before", mode="before") @classmethod - def utc(cls, value: datetime | None) -> datetime | None: - if value is None: - return None - try: - return value.astimezone(UTC) - except OverflowError: - # An offset can push the instant past year 9999 or before year 1; that is a bad value, not a server error. - raise ValueError("The time is outside the supported range") from None + def utc(cls, value: object) -> datetime | None: + return utc_time(value) @model_validator(mode="after") def coherent(self) -> RunSummaryQuery: @@ -321,6 +351,33 @@ def cursor_collection(self) -> str: return f"run_summaries:{_digest(filters)}:{self.order}" +class RunListQuery(RunSummaryQuery): + """Chronological `runs.list` filters, with explicit ID order for legacy cursors.""" + + # The list schema also permits ID order; the summary schema keeps chronological orders only. + order: RunListOrder = "started_desc" # type: ignore[assignment] + + def cursor_collection(self) -> str: + legacy = self.legacy_filters() + if legacy is not None: + return legacy.cursor_collection() + filters = self.model_dump(mode="json", exclude={"order", "limit", "cursor"}) + return f"runs:{_digest(filters)}:{self.order}" + + def legacy_filters(self) -> RunListFilters | None: + from firefly_weave.contracts.runtime import RunListFilters + + excluded = {"order", "limit", "cursor", "business_key", "correlation_key", "status", "include_archived"} + if self.order != "id" or len(self.status) > 1 or self.model_dump(exclude=excluded, exclude_defaults=True): + return None + return RunListFilters( + business_key=self.business_key, + correlation_key=self.correlation_key, + status=self.status[0] if self.status else None, + include_archived=self.include_archived, + ) + + class RunStepQuery(ContractModel): """`runs.steps` query parameters; `step` is a static step ID and matches every instance.""" diff --git a/src/firefly_weave/contracts/schema_export.py b/src/firefly_weave/contracts/schema_export.py index b31c2e08..1d5118d0 100644 --- a/src/firefly_weave/contracts/schema_export.py +++ b/src/firefly_weave/contracts/schema_export.py @@ -104,6 +104,7 @@ def contract_models() -> dict[str, SchemaModel]: DeploymentTarget, JobCancelRequest, ObservationReport, + ObservedMember, ObservedResource, ObserveRequest, PlanApproval, @@ -152,6 +153,7 @@ def contract_models() -> dict[str, SchemaModel]: TaskGroupRequest, ) from firefly_weave.contracts.identity import IdentityView + from firefly_weave.contracts.incident_queries import IncidentQuery from firefly_weave.contracts.language import LanguageManifest from firefly_weave.contracts.lumi import ( LumiAskRequest, @@ -169,7 +171,14 @@ def contract_models() -> dict[str, SchemaModel]: PrincipalRecord, PrincipalStatusRequest, ) - from firefly_weave.contracts.operations import EventPage, HistoryExport, RecordedEvidence, ReplayReport + from firefly_weave.contracts.operations import ( + EventPage, + HistoryExport, + IncidentView, + RecordedEvidence, + ReplayReport, + ) + from firefly_weave.contracts.operations_summary import OperationsSummary, OperationsTelemetry from firefly_weave.contracts.providers import ( ProviderEvent, ProviderIngressResponse, @@ -192,6 +201,7 @@ def contract_models() -> dict[str, SchemaModel]: ) from firefly_weave.contracts.run_lifecycle import RunLifecycle, RunLifecycleRequest, RunPurgeRequest from firefly_weave.contracts.run_views import ( + RunListQuery, RunLogPage, RunLogQuery, RunStepQuery, @@ -208,6 +218,13 @@ def contract_models() -> dict[str, SchemaModel]: from firefly_weave.contracts.schedules import ScheduleOccurrence, ScheduleRequest, ScheduleView from firefly_weave.contracts.teams import TeamsReactivateRequest, TeamsReference, TeamsRevokeRequest from firefly_weave.contracts.whatsapp import WhatsAppDeliveryState, WhatsAppStatusFact + from firefly_weave.contracts.worker_queries import ( + ReleaseUsage, + ReleaseUsageQuery, + RetireStaleRequest, + RetireStaleResult, + WorkerListQuery, + ) from firefly_weave.contracts.workers import ( ClaimRequest, CompleteRequest, @@ -224,7 +241,9 @@ def contract_models() -> dict[str, SchemaModel]: TaskLease, UnavailableCompletionReceipt, WorkerControlRequest, + WorkerIdentity, WorkerInstance, + WorkerPlacement, WorkerRelease, WorkerStatus, ) @@ -240,6 +259,7 @@ def contract_models() -> dict[str, SchemaModel]: "operations-component-spec": ComponentSpec, "operations-deployment-request": DeploymentRequest, "operations-deployment": Deployment, + "operations-observed-member": ObservedMember, "operations-observed-resource": ObservedResource, "operations-observation-report": ObservationReport, "operations-deployment-observation": DeploymentObservation, @@ -296,10 +316,13 @@ def contract_models() -> dict[str, SchemaModel]: "run-purge-request": RunPurgeRequest, "run-summary-page": RunSummaryPage, "run-summary-query": RunSummaryQuery, + "run-list-query": RunListQuery, "step-fact-page": StepFactPage, "run-step-query": RunStepQuery, "run-log-page": RunLogPage, "run-log-query": RunLogQuery, + "incident-view": IncidentView, + "incident-query": IncidentQuery, "human-task": HumanTask, "human-task-command": HumanTaskCommand, "complete-human-task": CompleteHumanTask, @@ -380,10 +403,19 @@ def contract_models() -> dict[str, SchemaModel]: "start-run-request": StartRunRequest, "worker-release-request": ReleaseRequest, "worker-release": WorkerRelease, + "worker-identity": WorkerIdentity, "worker-instance-request": InstanceRequest, "worker-instance": WorkerInstance, "worker-status": WorkerStatus, "worker-control-request": WorkerControlRequest, + "worker-placement": WorkerPlacement, + "worker-list-query": WorkerListQuery, + "retire-stale-request": RetireStaleRequest, + "retire-stale-result": RetireStaleResult, + "release-usage-query": ReleaseUsageQuery, + "release-usage": ReleaseUsage, + "operations-summary": OperationsSummary, + "operations-telemetry": OperationsTelemetry, "lease-proof": LeaseProof, "task-lease": TaskLease, "completion-receipt": CompletionReceipt, diff --git a/src/firefly_weave/contracts/surface.py b/src/firefly_weave/contracts/surface.py index cd2fd5a9..8cc234f3 100644 --- a/src/firefly_weave/contracts/surface.py +++ b/src/firefly_weave/contracts/surface.py @@ -106,6 +106,7 @@ TaskGroupRequest, ) from firefly_weave.contracts.identity import IdentityView +from firefly_weave.contracts.incident_queries import IncidentQuery from firefly_weave.contracts.integration_events import DeliveryAttempt, DeliveryView, Subscription, SubscriptionRequest from firefly_weave.contracts.language import LanguageManifest from firefly_weave.contracts.lumi import ( @@ -133,6 +134,7 @@ IncidentView, ReplayReport, ) +from firefly_weave.contracts.operations_summary import OperationsSummary, OperationsTelemetry from firefly_weave.contracts.providers import ( ProviderIngressResponse, ProviderReceipt, @@ -168,6 +170,7 @@ ) from firefly_weave.contracts.run_lifecycle import RunLifecycle, RunLifecycleRequest, RunPurgeRequest from firefly_weave.contracts.run_views import ( + RunListQuery, RunLogPage, RunLogQuery, RunStepQuery, @@ -187,6 +190,13 @@ from firefly_weave.contracts.teams import TeamsReactivateRequest, TeamsReference, TeamsRevokeRequest from firefly_weave.contracts.values import JsonObjectData from firefly_weave.contracts.whatsapp import MessageId, WhatsAppDeliveryState, WhatsAppStatusFact +from firefly_weave.contracts.worker_queries import ( + ReleaseUsage, + ReleaseUsageQuery, + RetireStaleRequest, + RetireStaleResult, + WorkerListQuery, +) from firefly_weave.contracts.workers import ( ClaimRequest, CompleteRequest, @@ -292,20 +302,6 @@ def native(self) -> OpenAPIOperation: name, "query", _present(field.annotation), required=field.is_required(), **documented ) ) - if self.id == "runs.list": - params += [ - OpenAPIParameter("business_key", "query", Annotated[str, Field(max_length=200)], required=False), - OpenAPIParameter("correlation_key", "query", Annotated[str, Field(max_length=200)], required=False), - OpenAPIParameter( - "status", - "query", - Literal[ - "queued", "running", "waiting", "suspended", "succeeded", "failed", "cancelled", "timed_out" - ], - required=False, - ), - OpenAPIParameter("include_archived", "query", bool, required=False, default=False), - ] if self.id == "human_tasks.list": params.append( OpenAPIParameter( @@ -332,7 +328,7 @@ def native(self) -> OpenAPIOperation: params.append(OpenAPIParameter("target_id", "query", UUID, required=False)) if self.id == "deployment_plans.list": params.append(OpenAPIParameter("deployment_id", "query", UUID, required=False)) - if self.page: + if self.page and self.query is None: params += [ OpenAPIParameter("limit", "query", PageLimit, required=False, default=50), OpenAPIParameter("cursor", "query", str, required=False), @@ -799,6 +795,12 @@ def native(self) -> OpenAPIOperation: CompatibilityReport, "compatibility.check", ), + Operation( + "operations_summary.read", ENVIRONMENT + "/operations/summary", "GET", OperationsSummary, "status.read" + ), + Operation( + "operations_telemetry.read", PROJECT + "/operations/telemetry", "GET", OperationsTelemetry, "status.read" + ), Operation( "retention.plan", PROJECT + "/operations/retention/plans", @@ -1427,7 +1429,13 @@ def native(self) -> OpenAPIOperation: "runs.start", ENVIRONMENT + "/runs", "POST", RunView, "run.start", StartRunRequest, (201,), idempotency=True ), Operation( - "runs.list", ENVIRONMENT + "/runs", "GET", Page[RunView | UnavailableResource], "run.read", page=True + "runs.list", + ENVIRONMENT + "/runs", + "GET", + Page[RunView | UnavailableResource], + "run.read", + page=True, + query=RunListQuery, ), Operation("runs.read", ENVIRONMENT + "/runs/{identifier}", "GET", RunView, "run.read"), Operation( @@ -1467,7 +1475,6 @@ def native(self) -> OpenAPIOperation: RunSummaryPage, "run.read", query=RunSummaryQuery, - served=False, ), Operation( "runs.steps", @@ -1476,7 +1483,6 @@ def native(self) -> OpenAPIOperation: StepFactPage, "run.read", query=RunStepQuery, - served=False, ), Operation( "runs.logs", @@ -1494,6 +1500,7 @@ def native(self) -> OpenAPIOperation: Page[IncidentView | UnavailableResource], "incident.read", page=True, + query=IncidentQuery, ), Operation( "incidents.list", @@ -1502,6 +1509,15 @@ def native(self) -> OpenAPIOperation: Page[IncidentView | UnavailableResource], "incident.read", page=True, + query=IncidentQuery, + ), + Operation( + "incidents.read", + ENVIRONMENT + "/incidents/{identifier}", + "GET", + IncidentView | UnavailableResource, + "incident.read", + etag=True, ), Operation( "incidents.resolve", @@ -1526,6 +1542,14 @@ def native(self) -> OpenAPIOperation: "releases.list", ENVIRONMENT + "/worker-releases", "GET", Page[WorkerRelease], "catalog.read", page=True ), Operation("releases.read", ENVIRONMENT + "/worker-releases/{identifier}", "GET", WorkerRelease, "catalog.read"), + Operation( + "releases.usage", + ENVIRONMENT + "/worker-releases/{identifier}/usage", + "GET", + ReleaseUsage, + "catalog.read", + query=ReleaseUsageQuery, + ), Operation( "workers.create", ENVIRONMENT + "/workers", @@ -1535,7 +1559,15 @@ def native(self) -> OpenAPIOperation: InstanceRequest, (201,), ), - Operation("workers.list", ENVIRONMENT + "/workers", "GET", Page[WorkerStatus], "status.read", page=True), + Operation( + "workers.list", + ENVIRONMENT + "/workers", + "GET", + Page[WorkerStatus], + "status.read", + page=True, + query=WorkerListQuery, + ), Operation("workers.read", ENVIRONMENT + "/workers/{identifier}", "GET", WorkerStatus, "status.read"), *( Operation( @@ -1550,6 +1582,15 @@ def native(self) -> OpenAPIOperation: for action in ("drain", "resume") ), Operation("workers.revoke", ENVIRONMENT + "/workers/{identifier}/revoke", "POST", Revoked, "release.retire"), + Operation( + "workers.retire_stale", + ENVIRONMENT + "/workers/retire-stale", + "POST", + RetireStaleResult, + "release.retire", + RetireStaleRequest, + idempotency=True, + ), Operation( "workers.grant", ENVIRONMENT + "/worker-connection-grants", diff --git a/src/firefly_weave/contracts/telemetry.py b/src/firefly_weave/contracts/telemetry.py index f0b0fac1..b86d8124 100644 --- a/src/firefly_weave/contracts/telemetry.py +++ b/src/firefly_weave/contracts/telemetry.py @@ -42,6 +42,8 @@ def endpoint(cls, value: str | None) -> str | None: valid = ( url.scheme in {"http", "https"} and url.hostname + # A DNS name has at most 253 characters, and the host is all that is ever shown. + and len(url.hostname) <= 253 and url.port != 0 and not url.username and not url.password diff --git a/src/firefly_weave/contracts/worker_queries.py b/src/firefly_weave/contracts/worker_queries.py new file mode 100644 index 00000000..1b7f24bc --- /dev/null +++ b/src/firefly_weave/contracts/worker_queries.py @@ -0,0 +1,139 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Worker fleet queries, stale-worker retirement and release usage. + +`workers.list` lists current workers by default: not revoked, and seen (or, never seen, registered) in the last +hour. Cursors are bound to every filter and the order. `releases.usage` pages the deployments that run a release, +bound to the release and to the targets the reader may see. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Annotated, Literal +from uuid import UUID + +from pydantic import Field, TypeAdapter + +from firefly_weave.contracts.definitions import ContractModel +from firefly_weave.contracts.deployments import DeploymentName +from firefly_weave.contracts.run_views import Count, Cursor, SummaryLimit, _digest +from firefly_weave.contracts.values import JsonValue + +type WorkerPresence = Literal["unknown", "recent", "stale"] +type WorkerListState = Literal["current", "all"] +type WorkerOrder = Literal["last_seen_desc", "registered_desc"] +# A task capability exactly as workers register it, such as echo@1.2.0. +type TaskTypeFilter = Annotated[str, Field(min_length=1, max_length=200, pattern=r"^[^\x00-\x1f\x7f-\x9f]+$")] +# A literal, case-sensitive prefix of a reported host; `%` and `_` match only themselves. +type HostPrefix = Annotated[str, Field(min_length=1, max_length=253, pattern=r"^[^\x00-\x1f\x7f-\x9f]+$")] +# One retirement command revokes at most this many workers. +RETIREMENT_BATCH = 100 + + +class WorkerListQuery(ContractModel): + """`workers.list` query parameters; every combination is a valid filter.""" + + presence: WorkerPresence | None = None + draining: bool | None = None + release_id: UUID | None = None + target_id: UUID | None = None + task_type: TaskTypeFilter | None = None + host_prefix: HostPrefix | None = None + state: WorkerListState = "current" + order: WorkerOrder = "last_seen_desc" + limit: SummaryLimit = 50 + cursor: Cursor | None = None + + def cursor_collection(self) -> str: + filters = self.model_dump(mode="json", exclude={"order", "limit", "cursor"}) + return f"workers:{_digest(filters)}:{self.order}" + + +class RetireStaleRequest(ContractModel): + """Revoke workers without an active lease whose last contact is older than the threshold.""" + + older_than_seconds: int = Field(ge=3600, le=2592000) + dry_run: bool + + +class RetireStaleResult(ContractModel): + """`matched` counts every eligible worker; `revoked` and `worker_ids` describe this command's batch.""" + + matched: Count + revoked: int = Field(ge=0, le=RETIREMENT_BATCH) + worker_ids: list[UUID] = Field(max_length=RETIREMENT_BATCH) + + +class ReleaseUsageQuery(ContractModel): + """`releases.usage` query parameters; a cursor continues only for the same release and readable targets.""" + + limit: SummaryLimit = 100 + cursor: Cursor | None = None + + def cursor_collection(self, release_id: UUID, authorized_targets: tuple[UUID, ...] | None) -> str: + targets = "*" if authorized_targets is None else sorted(str(target) for target in authorized_targets) + return f"releases.usage:{release_id}:{_digest({'targets': targets})}" + + +class ReleaseWorkerCounts(ContractModel): + """Every instance of the release once: revoked, else draining, else online (seen recently) or offline.""" + + online: Count + offline: Count + draining: Count + revoked: Count + + +class ReleaseDeploymentUsage(ContractModel): + target_id: UUID + target_name: DeploymentName + deployment_id: UUID + deployment_name: DeploymentName + component: DeploymentName + replicas: int = Field(ge=0, le=100) + + +class ReleaseUsage(ContractModel): + workers: ReleaseWorkerCounts + deployments: list[ReleaseDeploymentUsage] = Field(max_length=100) + activations: Count + next_cursor: Cursor | None = None + # True when the reader may read no deployment target, so an empty `deployments` says nothing about where the + # release runs; left out otherwise. + deployments_hidden: bool = Field(default=False, exclude_if=lambda value: not value) + + +@dataclass(frozen=True) +class ReleaseUsagePosition: + """The last deployment component of a release usage page.""" + + deployment_id: UUID + component: str + + +_COMPONENT: TypeAdapter[str] = TypeAdapter(DeploymentName) + + +def release_usage_position(position: tuple[JsonValue, str] | None) -> ReleaseUsagePosition | None: + """Validate a decoded release usage cursor before using its values in SQL.""" + if position is None: + return None + component, deployment = position + if type(component) is not str or type(deployment) is not str: + raise ValueError("Invalid release usage position") + return ReleaseUsagePosition(UUID(deployment), _COMPONENT.validate_python(component, strict=True)) diff --git a/src/firefly_weave/contracts/workers.py b/src/firefly_weave/contracts/workers.py index a9dec108..dd20148d 100644 --- a/src/firefly_weave/contracts/workers.py +++ b/src/firefly_weave/contracts/workers.py @@ -16,7 +16,8 @@ """Remote worker protocol. No persistence, startup or secret-provider dependencies.""" -from typing import Literal +import json +from typing import Annotated, Literal, Self from uuid import UUID from pydantic import AwareDatetime, Field, model_validator @@ -63,13 +64,82 @@ class WorkerRelease(ReleaseRequest): id: UUID -class InstanceRequest(ContractModel): +# A hostname, pod name, or Kubernetes namespace or node name: never a path, URL or shell text. +type WorkerHost = Annotated[str, Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9.-]{0,252}$")] +# A reported software or platform value, such as 0.1.0a15, 3.12.4, Linux or x86_64. +type WorkerVersion = Annotated[str, Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$")] +type WorkerLabelKey = Annotated[str, Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,63}$")] +type WorkerLabelValue = Annotated[str, Field(max_length=256, pattern=r"^[^\x00-\x1f\x7f-\x9f]*$")] + +# Label keys that would invite operators to put credentials into worker metadata. +CREDENTIAL_LABEL_WORDS = ("token", "password", "secret", "authorization") +# The worker_facts.identity column stores at most this many bytes of jsonb text. +WORKER_IDENTITY_BYTES = 4096 + + +def identity_text_bytes(value: object) -> int: + """The size of a JSON value as PostgreSQL renders it from jsonb, in UTF-8 bytes.""" + return len(json.dumps(value, ensure_ascii=False, separators=(", ", ": ")).encode()) + + +class WorkerIdentity(ContractModel): + """Where a worker says it runs. Display metadata only: it never grants authority or placement.""" + + host: WorkerHost | None = Field(default=None, exclude_if=lambda v: v is None) + runtime: Literal["embedded", "container", "process"] + sdk_version: WorkerVersion | None = Field(default=None, exclude_if=lambda v: v is None) + language_version: WorkerVersion | None = Field(default=None, exclude_if=lambda v: v is None) + os: WorkerVersion | None = Field(default=None, exclude_if=lambda v: v is None) + arch: WorkerVersion | None = Field(default=None, exclude_if=lambda v: v is None) + platform_hint: Literal["kubernetes", "docker", "none"] = "none" + namespace: WorkerHost | None = Field(default=None, exclude_if=lambda v: v is None) + node: WorkerHost | None = Field(default=None, exclude_if=lambda v: v is None) + labels: dict[WorkerLabelKey, WorkerLabelValue] = Field( + default_factory=dict, max_length=8, exclude_if=lambda v: not v + ) + + @model_validator(mode="after") + def bounded(self) -> Self: + if any(word in key.lower() for key in self.labels for word in CREDENTIAL_LABEL_WORDS): + raise ValueError("Worker labels cannot name credentials") + if identity_text_bytes(self.model_dump(mode="json")) > WORKER_IDENTITY_BYTES: + raise ValueError("Worker identity exceeds 4 KiB") + return self + + +# A target, deployment or component name as deployment records store it. +type PlacementName = Annotated[str, Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,99}$")] + + +class WorkerPlacement(ContractModel): + """Where the latest complete, current observation of a target places a worker, matched by its host. + + The server derives it for targets the reader may see; a worker never reports it. + """ + + target_id: UUID + target_name: PlacementName + deployment_id: UUID + component: PlacementName + + +class InstanceFields(ContractModel): + """What a registration request and the stored instance share.""" + release_id: UUID task_types: list[str] = Field(min_length=1, max_length=100) capacity: int = Field(ge=1, le=100) -class WorkerInstance(InstanceRequest): +class InstanceRequest(InstanceFields): + # Sent by current workers; a server that predates it refuses the field, and workers then + # register once more without it. + identity: WorkerIdentity | None = Field(default=None, exclude_if=lambda v: v is None) + + +class WorkerInstance(InstanceFields): + """The stable registration answer and stored instance. Identity lives in the worker's facts.""" + id: UUID principal_id: UUID revoked: bool = False @@ -80,7 +150,12 @@ class WorkerControlRequest(ContractModel): class WorkerStatus(WorkerInstance): - """A scoped observation, separate from the stable worker registration protocol.""" + """A scoped observation, separate from the stable worker registration protocol. + + `identity`, `registered_at` and `drained_at` come from the worker's facts and are left out when unknown, + as they are for workers registered or drained before they were recorded. `placement` is left out when + no observation the reader may see places the worker. + """ revision: int = Field(ge=1) draining: bool @@ -91,6 +166,10 @@ class WorkerStatus(WorkerInstance): observed_at: AwareDatetime active_leases: int = Field(ge=0) available_capacity: int | None = Field(ge=0, le=100) + identity: WorkerIdentity | None = Field(default=None, exclude_if=lambda v: v is None) + registered_at: AwareDatetime | None = Field(default=None, exclude_if=lambda v: v is None) + placement: WorkerPlacement | None = Field(default=None, exclude_if=lambda v: v is None) + drained_at: AwareDatetime | None = Field(default=None, exclude_if=lambda v: v is None) class LeaseProof(ContractModel): diff --git a/src/firefly_weave/deployment_runner/compose.py b/src/firefly_weave/deployment_runner/compose.py index 48da3386..38e2d819 100644 --- a/src/firefly_weave/deployment_runner/compose.py +++ b/src/firefly_weave/deployment_runner/compose.py @@ -21,29 +21,91 @@ import fcntl import json import os +import re import stat import tempfile from collections.abc import Awaitable, Callable from pathlib import Path from typing import Any, cast +from pydantic import ValidationError + from firefly_weave.compiler.canonical import canonical_digest -from firefly_weave.contracts.deployments import DeploymentPlan, ObservationReport, ObservedResource +from firefly_weave.contracts.deployments import DeploymentPlan, ObservationReport, ObservedMember, ObservedResource from firefly_weave.contracts.values import JsonObject from firefly_weave.deployment_runner.command import run_command from firefly_weave.deployment_runner.policy import DestinationPolicy -from firefly_weave.deployment_runner.runtime import RunnerPolicyError +from firefly_weave.deployment_runner.runtime import RunnerPolicyError, observed_time _INSPECT = ( '{"id":{{json .Id}},"image":{{json .Config.Image}},"state":{{json .State.Status}},' '"started":{{json .State.StartedAt}},"restart":{{json .RestartCount}},' '"project":{{json (index .Config.Labels "com.docker.compose.project")}},' '"service":{{json (index .Config.Labels "com.docker.compose.service")}},' + '"number":{{json (index .Config.Labels "com.docker.compose.container-number")}},' '"target":{{json (index .Config.Labels "firefly.weave.target")}},' '"health":{{with (index .State "Health")}}{{json .Status}}{{else}}""{{end}}}' ) +def _ready(fact: dict[str, Any]) -> bool: + return fact.get("state") == "running" and fact.get("health") in {"", "healthy"} + + +def _versioned(fact: dict[str, Any]) -> dict[str, Any]: + # The container number is read for members only. Leaving it out keeps each resource + # version, and the plans reviewed against it, as they were before members existed. + return {key: value for key, value in fact.items() if key != "number"} + + +def _ordinal(number: object) -> int | None: + """The Compose container number; only ASCII digits, as Compose writes it.""" + if number in (None, ""): + return None + if not isinstance(number, str) or not re.fullmatch(r"[0-9]{1,7}", number): + raise RunnerPolicyError() + return int(number) + + +# Containers in these states run no worker that placement could describe. +_STOPPED = frozenset({"exited", "dead", "removing"}) + + +def _state(fact: dict[str, Any]) -> str: + """The container state as Docker writes it; anything but text makes the container unreadable.""" + state = fact["state"] + if not isinstance(state, str): + raise RunnerPolicyError() + return state + + +def _members(facts: list[dict[str, Any]]) -> list[ObservedMember] | None: + """Live worker containers as members, or None when they cannot all be reported. + + Members are placement only, so they never fail an observation or an apply: more than 100 live + containers or one unreadable container leaves them out (unknown) rather than reporting part of them. + """ + try: + live = [fact for fact in facts if _state(fact) not in _STOPPED] + if len(live) > 100: + return None + return [ + ObservedMember( + # Docker's default hostname, which the worker reports, is the ID prefix. + host=fact["id"][:12], + ordinal=_ordinal(fact.get("number")), + state=fact["state"], + ready=_ready(fact), + started_at=observed_time(fact["started"]), + restarts=fact["restart"], + node=None, + ) + for fact in live + ] + except (KeyError, RunnerPolicyError, ValidationError): + return None + + class ComposeAdapter: def __init__(self, policy: DestinationPolicy, *, command: Callable[..., Awaitable[bytes]] = run_command) -> None: if policy.adapter != "docker-compose" or not policy.compose_file or not policy.lock_file: @@ -132,7 +194,7 @@ async def observe(self) -> ObservationReport: image = None if not isinstance(image, str) or not self.policy.allows_image(image): image = None - ready = sum(item.get("state") == "running" and item.get("health") in {"", "healthy"} for item in members) + ready = sum(_ready(item) for item in members) desired = spec.get("deploy", {}).get("replicas", 1) stopped = not members and desired == 0 stable = stopped or (ready == len(members) == desired) @@ -148,9 +210,13 @@ async def observe(self) -> ObservationReport: image=image, replicas=len(members), ready_replicas=ready, - version=canonical_digest(cast(JsonObject, {"configuration": configuration, "members": members})), + version=canonical_digest( + cast(JsonObject, {"configuration": configuration, "members": [_versioned(m) for m in members]}) + ), ownership="managed" if managed else "imported", state="stopped" if stopped else "ready" if stable else "progressing", + # Placement only concerns workers; other components keep their observation bytes. + members=(_members(members) or []) if binding.kind == "worker" else [], ) ) return ObservationReport(resources=resources, complete=True, settled=settled) diff --git a/src/firefly_weave/deployment_runner/kubernetes.py b/src/firefly_weave/deployment_runner/kubernetes.py index 8efd12ce..9bd594e2 100644 --- a/src/firefly_weave/deployment_runner/kubernetes.py +++ b/src/firefly_weave/deployment_runner/kubernetes.py @@ -25,13 +25,84 @@ from decimal import Decimal from typing import Any -from firefly_weave.contracts.deployments import ComponentSpec, DeploymentPlan, ObservationReport, ObservedResource +from firefly_weave.contracts.deployments import ( + ComponentSpec, + DeploymentPlan, + ObservationReport, + ObservedMember, + ObservedResource, +) from firefly_weave.deployment_runner.command import run_command from firefly_weave.deployment_runner.policy import ComponentBinding, DestinationPolicy -from firefly_weave.deployment_runner.runtime import RunnerPolicyError +from firefly_weave.deployment_runner.runtime import RunnerPolicyError, observed_time TARGET_LABEL = "firefly.weave/target" CONFIGURATION_ANNOTATION = "firefly.weave/configuration" +# Kubernetes label syntax, so a selector built from a deployment never carries selector operators of its own. +_LABEL_KEY = re.compile(r"(?:[a-z0-9](?:[-a-z0-9.]{0,251}[a-z0-9])?/)?[A-Za-z0-9](?:[-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?") +_LABEL_VALUE = re.compile(r"(?:[A-Za-z0-9](?:[-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?)?") +# The only pod fields read for members, so the listing stays small: about 250 bytes a pod. Free-text status +# fields stay out, so no value can carry a line break into the listing; evicted pods always have phase Failed. +_POD_FIELDS = ( + ".metadata.name", + ".metadata.namespace", + ".metadata.deletionTimestamp", + ".metadata.labels.pod-template-hash", + ".metadata.ownerReferences[?(@.controller==true)].apiVersion", + ".metadata.ownerReferences[?(@.controller==true)].kind", + ".metadata.ownerReferences[?(@.controller==true)].name", + ".status.phase", + '.status.conditions[?(@.type=="Ready")].status', + '.status.containerStatuses[?(@.name=="{container}")].restartCount', + '.status.containerStatuses[?(@.name=="{container}")].state.running.startedAt', + ".spec.nodeName", +) + + +def _pod_fields(container: str) -> str: + """A kubectl JSONPath printing one tab-separated line per pod; `container` is a policy name.""" + fields = '{"\\t"}'.join("{" + field.replace("{container}", container) + "}" for field in _POD_FIELDS) + return "{range .items[*]}" + fields + '{"\\n"}{end}' + + +def _label(key: object, value: object = "") -> bool: + return ( + isinstance(key, str) + and isinstance(value, str) + and _LABEL_KEY.fullmatch(key) is not None + and _LABEL_VALUE.fullmatch(value) is not None + ) + + +def _selector(selector: object) -> str | None: + """A deployment selector as a kubectl label selector, or None when kubectl cannot express it exactly. + + `matchLabels` become `key=value`; `matchExpressions` become set-based requirements (`key in (a,b)`, + `key notin (a,b)`, `key`, `!key`). Anything else, including an empty selector, lists no pods. + """ + if not isinstance(selector, dict) or not set(selector) <= {"matchLabels", "matchExpressions"}: + return None + labels, expressions = selector.get("matchLabels") or {}, selector.get("matchExpressions") or [] + if not isinstance(labels, dict) or not isinstance(expressions, list): + return None + if not all(_label(key, value) for key, value in labels.items()): + return None + requirements = [f"{key}={value}" for key, value in sorted(labels.items())] + for expression in expressions: + if not isinstance(expression, dict) or not set(expression) <= {"key", "operator", "values"}: + return None + key, operator, values = expression.get("key"), expression.get("operator"), expression.get("values") or [] + if not _label(key) or not isinstance(values, list) or not all(_label(key, item) and item for item in values): + return None + if operator in {"In", "NotIn"} and values: + requirements.append(f"{key} {operator.lower()} ({','.join(sorted(values))})") + elif operator == "Exists" and not values: + requirements.append(f"{key}") + elif operator == "DoesNotExist" and not values: + requirements.append(f"!{key}") + else: + return None + return ",".join(requirements) or None def _quantity(value: Any) -> Decimal: @@ -57,8 +128,8 @@ def __init__(self, policy: DestinationPolicy, *, command: Callable[..., Awaitabl raise RunnerPolicyError() self.policy, self.command = policy, command - async def _json(self, *args: str, stdin: bytes | None = None) -> dict[str, Any]: - raw = await self.command( + async def _run(self, *args: str, stdin: bytes | None = None) -> bytes: + return await self.command( [ self.policy.executable, "--context", @@ -72,6 +143,9 @@ async def _json(self, *args: str, stdin: bytes | None = None) -> dict[str, Any]: timeout=30, limit=1024 * 1024, ) + + async def _json(self, *args: str, stdin: bytes | None = None) -> dict[str, Any]: + raw = await self._run(*args, stdin=stdin) try: value = json.loads(raw) if raw.strip() else {} if not isinstance(value, dict): @@ -108,7 +182,65 @@ def _container(self, value: dict[str, Any], binding: ComponentBinding) -> tuple[ raise RunnerPolicyError() return found[0] - def _fact(self, value: dict[str, Any], binding: ComponentBinding) -> ObservedResource: + async def _members(self, value: dict[str, Any], binding: ComponentBinding) -> list[ObservedMember] | None: + """The deployment's live pods as members, or None when they cannot all be listed. + + Members are placement only, so listing them never fails an observation or an apply: a selector kubectl + cannot express, a failed or unreadable listing, a malformed pod, or more than 100 live pods leaves them + out (unknown) rather than reporting part of them. + """ + selector = _selector(value.get("spec", {}).get("selector")) + if selector is None or binding.container is None: + return None + try: + raw = await self._run( + "get", "pods", "--selector=" + selector, "-o=jsonpath=" + _pod_fields(binding.container) + ) + text = raw.decode() + if text and not text.endswith("\n"): + raise ValueError("Incomplete pod listing") + members = [] + for line in text.split("\n")[:-1]: + member = self._member(line.split("\t"), value["metadata"]["name"]) + if member is not None: + members.append(member) + if len(members) > 100: + return None + return sorted(members, key=lambda member: member.host) + except Exception: + # Placement stays unknown; cancellation is not an Exception and still propagates. + return None + + def _member(self, fields: list[str], deployment: str) -> ObservedMember | None: + """A live pod of this deployment, None for any other pod, or an error for an unreadable one.""" + if len(fields) != len(_POD_FIELDS): + raise ValueError("Unexpected pod fields") + name, namespace, deleting, template, api, kind, owner, phase, ready, restarts, started, node = fields + # The deployment controller names each ReplicaSet `-`; exactly one + # controller owner is listed, so several controllers never match. + if namespace != self.policy.boundary or (api, kind) != ("apps/v1", "ReplicaSet") or not template: + return None + if owner != f"{deployment}-{template}": + return None + # Terminating, finished, failed and evicted pods run no worker that placement could describe. + if deleting or phase in {"Succeeded", "Failed"}: + return None + # A Ready status of Unknown (for example on a lost node) is not ready. + if ready not in {"", "True", "False", "Unknown"} or not re.fullmatch(r"[0-9]{0,10}", restarts): + raise ValueError("Unexpected pod status") + return ObservedMember( + host=name, + ordinal=None, + state=phase.lower() if phase else "unknown", + ready=ready == "True", + started_at=observed_time(started or None), + restarts=int(restarts or "0"), + node=node or None, + ) + + def _fact( + self, value: dict[str, Any], binding: ComponentBinding, members: list[ObservedMember] | None + ) -> ObservedResource: _, container = self._container(value, binding) metadata, spec, status = value["metadata"], value["spec"], value.get("status", {}) desired = spec.get("replicas", 1) @@ -138,6 +270,7 @@ def _fact(self, value: dict[str, Any], binding: ComponentBinding) -> ObservedRes if metadata.get("labels", {}).get(TARGET_LABEL) == str(self.policy.target_id) else "imported", state=("stopped" if desired == 0 else "ready") if settled else "progressing", + members=members or [], ) async def observe(self) -> ObservationReport: @@ -146,7 +279,9 @@ async def observe(self) -> ObservationReport: for binding in self.policy.components: value = await self._deployment(binding) if value: - resources.append(self._fact(value, binding)) + # Placement only concerns workers; other components need no pod reads. + members = await self._members(value, binding) if binding.kind == "worker" else None + resources.append(self._fact(value, binding, members)) return ObservationReport( resources=resources, complete=True, settled=all(item.state in {"ready", "stopped"} for item in resources) ) diff --git a/src/firefly_weave/deployment_runner/runtime.py b/src/firefly_weave/deployment_runner/runtime.py index d1c998cb..e623a716 100644 --- a/src/firefly_weave/deployment_runner/runtime.py +++ b/src/firefly_weave/deployment_runner/runtime.py @@ -19,6 +19,7 @@ from __future__ import annotations import asyncio +import re from collections.abc import Awaitable, Callable from datetime import UTC, datetime from typing import Protocol, cast @@ -41,6 +42,29 @@ def __init__(self) -> None: super().__init__("The operation does not match the runner's local authority or current observation.") +_OBSERVED_TIME = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})") + + +def observed_time(value: object) -> datetime | None: + """An RFC 3339 time a provider reported, to the microsecond; its zero time means never. + + Docker reports nanoseconds and `0001-01-01T00:00:00Z` for a container that never started. + Anything else that is not such a time refuses the observation instead of guessing. + """ + if value is None: + return None + match = _OBSERVED_TIME.fullmatch(value) if isinstance(value, str) else None + if match is None: + raise RunnerPolicyError() + if match[1].startswith("0001-01-01T"): + return None + fraction = "." + match[2][:6].ljust(6, "0") if match[2] else "" + try: + return datetime.fromisoformat(match[1] + fraction + ("+00:00" if match[3] == "Z" else match[3])) + except ValueError: + raise RunnerPolicyError() from None + + class InfrastructureAdapter(Protocol): async def observe(self) -> ObservationReport: ... async def apply(self, plan: DeploymentPlan, before_write: Callable[[], Awaitable[None]]) -> ObservationReport: ... diff --git a/src/firefly_weave/email/source.py b/src/firefly_weave/email/source.py index 9df2c3e6..c76aae2b 100644 --- a/src/firefly_weave/email/source.py +++ b/src/firefly_weave/email/source.py @@ -36,6 +36,7 @@ from firefly_weave.email.mime import parse_message from firefly_weave.email.service import EmailService, unavailable from firefly_weave.email.transport import IMAPTransport +from firefly_weave.operations.facts import RunStartFacts from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.repository import SCOPE, RuntimeRepository from firefly_weave.runtime.service import RuntimeService @@ -493,6 +494,7 @@ async def dispatch(self, actor: Principal, scope: Scope, receipt_id: UUID) -> di key, context=AuditContext(), tx=tx, + start_facts=RunStartFacts(origin="email"), ) run_id, signal_id = outcome_run.id, None await db.execute( diff --git a/src/firefly_weave/observability.py b/src/firefly_weave/observability.py index dee4a9d8..168bb843 100644 --- a/src/firefly_weave/observability.py +++ b/src/firefly_weave/observability.py @@ -21,6 +21,8 @@ """ import threading +import time +from collections.abc import Callable from opentelemetry.metrics import MeterProvider as MeterProviderAPI from opentelemetry.sdk.metrics import AlwaysOffExemplarFilter, MeterProvider @@ -47,11 +49,16 @@ class TelemetryDrops: - """A saturating diagnostic counter remains bounded even when no exporter is available.""" + """A saturating diagnostic counter remains bounded even when no exporter is available. - def __init__(self) -> None: + It also keeps the monotonic time of the last drop, so export health can be judged by recent drops. + """ + + def __init__(self, *, monotonic: Callable[[], float] = time.monotonic) -> None: self.count = 0 self._lock = threading.Lock() + self._monotonic = monotonic + self._last: float | None = None self.provider: MeterProviderAPI | None = None def add(self, count: int) -> None: @@ -59,12 +66,19 @@ def add(self, count: int) -> None: return with self._lock: self.count = min((1 << 63) - 1, self.count + count) + self._last = self._monotonic() provider = self.provider if provider is not None: provider.get_meter(SCOPE).create_counter("weave.telemetry_dropped").add( count, attributes("other", "rejected") ) + def dropped_within(self, seconds: float) -> bool: + """Whether anything was dropped in the last `seconds`.""" + with self._lock: + last = self._last + return last is not None and self._monotonic() - last <= seconds + @configuration class OwnedMeterConfiguration: diff --git a/src/firefly_weave/operations/exports.py b/src/firefly_weave/operations/exports.py index 5850810e..11875046 100644 --- a/src/firefly_weave/operations/exports.py +++ b/src/firefly_weave/operations/exports.py @@ -18,19 +18,22 @@ import json from collections.abc import Mapping -from typing import cast +from typing import Final, cast, get_args from firefly_weave.compiler.api import CompiledArtifact from firefly_weave.compiler.canonical import canonical_digest from firefly_weave.compiler.ir import ActionNode, HumanTaskNode, IRNode, SignalNode from firefly_weave.contracts.definitions import ActionDefinition, load_definition from firefly_weave.contracts.instance_keys import InvalidInstanceKey, split_instance +from firefly_weave.contracts.run_views import RunOrigin from firefly_weave.contracts.values import JsonObject, JsonValue from firefly_weave.contracts.workers import TaskError from firefly_weave.operations.redaction import Omission, project from firefly_weave.runtime.kernel import action_schemas, workflow from firefly_weave.runtime.models import RuntimeEvent, Transition +RUN_ORIGINS: Final = frozenset(get_args(RunOrigin.__value__)) + def event_node(nodes: Mapping[str, IRNode], key: object) -> IRNode | None: """The step named by an event's instance key; None unless the key is canonical and names a step.""" @@ -66,7 +69,7 @@ def safe_event(event: RuntimeEvent, artifact: CompiledArtifact | None, *, unavai data = dict(event.data) omissions: list[Omission] = [] fields = { - "started": {"admission_policy"}, + "started": {"admission_policy", "origin"}, "human_completed": {"node_id", "task_id", "actor_id", "output"}, "paused": {"actor_id", "reason", "control_revision"}, "resumed": {"actor_id", "reason", "control_revision"}, @@ -90,6 +93,13 @@ def safe_event(event: RuntimeEvent, artifact: CompiledArtifact | None, *, unavai "wait_elapsed": {"node_id", "deadline", "wait_id"}, "signal_received": {"node_id", "output", "accepted_at", "deadline"}, }[event.type] + if ( + event.type == "started" + and "origin" in data + and not (type(data["origin"]) is str and data["origin"] in RUN_ORIGINS) + ): + # Only the server-recorded closed set is disclosed; any other retained value stays withheld. + fields = fields - {"origin"} if any(key not in fields for key in data): data = {key: value for key, value in data.items() if key in fields} omissions.append(Omission(path="/data/*", reason="classification_unavailable")) diff --git a/src/firefly_weave/operations/fact_positions.py b/src/firefly_weave/operations/fact_positions.py new file mode 100644 index 00000000..c60abb73 --- /dev/null +++ b/src/firefly_weave/operations/fact_positions.py @@ -0,0 +1,73 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Typed positions for chronological run and step fact queries.""" + +from dataclasses import dataclass +from datetime import datetime +from uuid import UUID + +from pydantic import TypeAdapter + +from firefly_weave.contracts.definitions import ResourceName +from firefly_weave.contracts.instance_keys import InstanceKeyText, instance_view +from firefly_weave.contracts.run_views import utc_time +from firefly_weave.contracts.values import JsonValue + +type Position = tuple[JsonValue, str] | None + +_NODE: TypeAdapter[str] = TypeAdapter(ResourceName) +_INSTANCE: TypeAdapter[str] = TypeAdapter(InstanceKeyText) + + +@dataclass(frozen=True) +class RunPosition: + """A nullable recorded time and the UUID tie breaker.""" + + at: datetime | None + id: UUID + + +@dataclass(frozen=True) +class StepPosition: + """A nullable scheduled time and the canonical step instance identity.""" + + scheduled_at: datetime | None + node_id: str + instance_key: str + + +def run_position(position: Position) -> RunPosition | None: + """Validate a decoded run cursor before using its values in SQL.""" + if position is None: + return None + value, identifier = position + if type(identifier) is not str: + raise ValueError("Invalid run position identifier") + return RunPosition(utc_time(value), UUID(identifier)) + + +def step_position(position: Position) -> StepPosition | None: + """Validate a decoded step cursor: the scheduled time and the full author instance key it ends at. + + The key alone determines the step ID and the stored instance key ('' for the step itself). + """ + if position is None: + return None + at, identifier = position + identity = instance_view(_INSTANCE.validate_python(identifier, strict=True)) + _NODE.validate_python(identity.node_id, strict=True) + return StepPosition(utc_time(at), identity.node_id, identity.instance_key) diff --git a/src/firefly_weave/operations/fact_reads.py b/src/firefly_weave/operations/fact_reads.py new file mode 100644 index 00000000..7f580b0b --- /dev/null +++ b/src/firefly_weave/operations/fact_reads.py @@ -0,0 +1,726 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Scoped, bounded operational metadata reads; execution remains runtime-owned.""" + +import asyncio +import heapq +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Any, cast +from uuid import UUID + +from pydantic import TypeAdapter +from sqlalchemy import text + +from firefly_weave.compiler.ir import accepted_ir_versions +from firefly_weave.contracts import language_features +from firefly_weave.contracts.incident_queries import IncidentQuery +from firefly_weave.contracts.instance_keys import split_instance +from firefly_weave.contracts.operations import IncidentView +from firefly_weave.contracts.public import UnavailableResource +from firefly_weave.contracts.run_views import ( + RunListQuery, + RunStepQuery, + RunSummary, + RunSummaryQuery, + StepFact, + StepFactWithOutput, + StepKind, +) +from firefly_weave.definitions.models import CatalogError, ir_unsupported +from firefly_weave.operations.fact_positions import RunPosition, StepPosition +from firefly_weave.operations.facts import FactRepository, author_instance +from firefly_weave.operations.redaction import Omission, SafeProjection +from firefly_weave.persistence.paging import PAGE_BYTES +from firefly_weave.persistence.uow import Transaction +from firefly_weave.runtime import admission + +_RUN_SCOPE = "f.tenant_id=:tenant AND f.project_id=:project AND f.environment_id=:environment" +# Summary columns only: node kinds (up to 1 MiB) are read by the one timeline metadata query that needs them. +_RUN_COLUMNS = ( + "f.run_id,f.definition_version_id,f.workflow_name,f.workflow_version,f.activation_id,f.activation_name," + "f.activation_revision,f.status,f.paused,f.test,f.origin,f.caller_run_id,f.caller_node_id," + "f.caller_instance_key,f.retried_from_run_id,f.business_key,f.correlation_key,f.started_at,f.updated_at," + "f.ended_at,f.failed_node_id,f.failed_instance_key,f.failed_error_code,f.active_incidents,f.handled_errors," + "f.classification_state,f.classification_policy,f.pinned_ir_version,f.pinned_features" +) +_RUN_FROM = ( + ",coalesce(a.archived,false) AS archived,EXISTS(SELECT 1 FROM run_policy_blocks b " + "WHERE b.tenant_id=f.tenant_id AND b.project_id=f.project_id AND b.environment_id=f.environment_id " + "AND b.run_id=f.run_id) AS policy_blocked FROM run_facts f LEFT JOIN run_archives a ON " + "a.tenant_id=f.tenant_id AND a.project_id=f.project_id AND a.environment_id=f.environment_id " + "AND a.run_id=f.run_id WHERE " + _RUN_SCOPE +) +_RUN_SELECT = "SELECT " + _RUN_COLUMNS + _RUN_FROM +_METADATA_SELECT = "SELECT " + _RUN_COLUMNS + ",f.node_kinds" + _RUN_FROM +_MISSING = ( + "s.tenant_id=:tenant AND s.project_id=:project AND s.environment_id=:environment AND s.run_id=:run " + "AND left(s.node_id,1)<>'@' AND NOT EXISTS(SELECT 1 FROM step_facts f " + "WHERE f.tenant_id=s.tenant_id AND f.project_id=s.project_id AND f.environment_id=s.environment_id " + "AND f.run_id=s.run_id AND (f.instance_key=s.node_id OR (f.instance_key='' AND f.node_id=s.node_id)))" +) +_STEP_ORDER = 'scheduled_at ASC NULLS LAST,node_id COLLATE "C",instance_key COLLATE "C"' +_INCIDENT_SCOPE = "i.tenant_id=:tenant AND i.project_id=:project AND i.environment_id=:environment" +_INCIDENT_FACTS = ( + " LEFT JOIN incident_facts f ON f.tenant_id=i.tenant_id AND f.project_id=i.project_id " + "AND f.environment_id=i.environment_id AND f.incident_id=i.id" +) +# Ordering keys only: filters and sorting never read the incident's resolution or its run. +_INCIDENT_KEYS = "SELECT i.id,f.opened_at FROM incidents i" + _INCIDENT_FACTS + " WHERE " + _INCIDENT_SCOPE +# One statement for the selected incidents: their columns, measured together against the page budget before the +# resolutions are returned, their facts and their run's classification metadata. +_INCIDENT_DETAILS = ( + "WITH page AS (SELECT i.*,sum(octet_length(to_jsonb(i)::text)) OVER () AS page_bytes FROM incidents i " + "WHERE " + _INCIDENT_SCOPE + " AND i.id=ANY(:ids)) " + "SELECT i.id,i.run_id,i.incident_key,i.node_id,i.generation,i.origin_code,i.code,i.status,i.revision," + "i.actor_id,i.resolved_at,CASE WHEN i.page_bytes<=:budget THEN i.resolution END AS resolution," + "i.page_bytes>:budget AS oversized,f.incident_id IS NOT NULL AS projected,f.node_id AS static_node_id," + "f.instance_key,f.opened_at,f.updated_at,f.opened_at_estimated,f.workflow_name,f.workflow_version,f.test," + "r.status AS run_status,r.classification_state,r.classification_policy,r.pinned_ir_version,r.pinned_features," + "EXISTS(SELECT 1 FROM run_policy_blocks b WHERE b.tenant_id=i.tenant_id AND b.project_id=i.project_id " + "AND b.environment_id=i.environment_id AND b.run_id=i.run_id) AS policy_blocked FROM page i" + + _INCIDENT_FACTS + + " LEFT JOIN run_facts r ON r.tenant_id=i.tenant_id AND r.project_id=i.project_id " + "AND r.environment_id=i.environment_id AND r.run_id=i.run_id" +) +LEGACY_KEYS = 100_000 +LEGACY_SECONDS = 2 +_KIND: TypeAdapter[StepKind] = TypeAdapter(StepKind) + + +def unavailable(identifier: UUID) -> UnavailableResource: + return UnavailableResource(id=identifier, omissions=[Omission(path="", reason="classification_unavailable")]) + + +def _pinned_features(row: Mapping[str, Any]) -> list[str] | None: + features = row.get("pinned_features") + if ( + isinstance(features, list) + and len(features) <= 256 + and all(type(value) is str for value in features) + and len(json.dumps(features, ensure_ascii=False).encode()) <= 16384 + ): + return features + return None + + +def _supported(row: Mapping[str, Any], features: list[str]) -> bool: + advertised = language_features.ADVERTISED_FEATURES + return set(features) <= set(advertised) and row.get("pinned_ir_version") in accepted_ir_versions(advertised) + + +def available(row: Mapping[str, Any]) -> bool: + features = _pinned_features(row) + return ( + row.get("classification_state") == "available" + and row.get("classification_policy") == admission.POLICY + and row.get("policy_blocked") is False + and features is not None + and _supported(row, features) + ) + + +def withheld(row: Mapping[str, Any]) -> CatalogError: + """The answer for a run whose metadata is withheld, matching what ``runs.read`` answers for it. + + A run this platform does not run yet (cached as unsupported, or pinning an IR version or feature no longer + advertised) answers ``ir_unsupported``; it is transient. Legacy or blocked evidence stays unavailable. + """ + features = _pinned_features(row) + if ( + features is not None + and row.get("policy_blocked") is False + and row.get("classification_policy") == admission.POLICY + and ( + row.get("classification_state") == "unsupported" + or (row.get("classification_state") == "available" and not _supported(row, features)) + ) + ): + return ir_unsupported(tuple(sorted(set(features) - set(language_features.ADVERTISED_FEATURES)))) + return legacy_unavailable() + + +def milliseconds(start: datetime | None, end: datetime | None) -> int | None: + return int((end - start).total_seconds() * 1000) if start is not None and end is not None else None + + +def summary_of(row: Mapping[str, Any]) -> RunSummary | UnavailableResource: + identifier = row["run_id"] + if not available(row): + return unavailable(identifier) + try: + return RunSummary.model_validate( + dict( + id=identifier, + workflow=dict( + name=row["workflow_name"], + version=row["workflow_version"], + definition_version_id=row["definition_version_id"], + ), + activation_id=row["activation_id"], + activation=dict(name=row["activation_name"], revision=row["activation_revision"]), + status=row["status"], + paused=row["paused"], + test=row["test"], + origin=row["origin"], + caller=( + dict( + run_id=row["caller_run_id"], + node_id=row["caller_node_id"], + instance_key=row["caller_instance_key"], + ) + if row["caller_run_id"] is not None + else None + ), + retried_from_run_id=row["retried_from_run_id"], + started_at=row["started_at"], + updated_at=row["updated_at"], + ended_at=row["ended_at"], + duration_ms=milliseconds(row["started_at"], row["ended_at"]), + business_key=row["business_key"], + correlation_key=row["correlation_key"], + failed_step=( + dict( + node_id=row["failed_node_id"], + instance_key=row["failed_instance_key"], + error_code=row["failed_error_code"], + ) + if row["failed_node_id"] is not None + else None + ), + active_incidents=row["active_incidents"], + handled_errors=row["handled_errors"], + archived=row["archived"], + ) + ) + except (ValueError, TypeError, KeyError, OverflowError): + return unavailable(identifier) + + +def step_of(row: Mapping[str, Any], *, output: SafeProjection | None = None) -> StepFact | StepFactWithOutput: + try: + values: dict[str, Any] = { + key: row.get(key) + for key in ( + "node_id", + "instance_key", + "kind", + "status", + "scheduled_at", + "started_at", + "ended_at", + "attempts", + "worker_id", + "error_code", + "handled", + "child_run_id", + ) + } + values.update( + iteration=list(split_instance(row["instance_key"] or row["node_id"]).indexes), + duration_ms=milliseconds(row["started_at"], row["ended_at"]), + queue_wait_ms=milliseconds(row["scheduled_at"], row["started_at"]), + log_entries=0, + ) + if output is not None: + if output.available: + return StepFactWithOutput(**values, output=output.value) + values["omissions"] = [ + omission.model_copy(update={"path": "/output" + omission.path}) for omission in output.omissions + ] + return StepFact(**values) + except (ValueError, TypeError, KeyError, OverflowError): + raise legacy_unavailable() from None + + +def _json(value: object) -> str: + return value.isoformat() if isinstance(value, datetime) else str(value) + + +def _step_identity(source: str | None, static: str | None, instance: str | None) -> tuple[str | None, str] | None: + """The served ``node_id`` and ``instance_key``, or None when the incident and its facts disagree.""" + if source is None: + # A run-level incident has no step facts. + return (None, "") if static is None and not instance else None + if static is None: + # Facts never attribute a control step (a branch output or join) to an author step; it keeps its own ID. + return (source, "") if source.startswith("@") and not instance else None + # The facts normalized the incident's own key: its full instance key, or the static step ID for a plain step. + return (static, instance or "") if source == (instance or static) else None + + +def incident_of(row: Mapping[str, Any]) -> IncidentView | UnavailableResource: + """An incident from its own columns and its facts; withheld unless its run's metadata is available. + + The facts carry the static step ID and full instance key normalized when they were written; a control step keeps + the step ID stored on the incident. An incident whose facts are missing, or whose step identity disagrees with its + facts, is unavailable rather than guessed. + """ + identifier = row["id"] + identity = _step_identity(row["node_id"], row["static_node_id"], row["instance_key"]) + if not row["projected"] or not available(row) or identity is None: + return unavailable(identifier) + node_id, instance_key = identity + resolution = row["resolution"] + values = dict( + id=identifier, + run_id=row["run_id"], + incident_key=row["incident_key"], + node_id=node_id, + generation=row["generation"], + origin_code=row["origin_code"], + code=row["code"], + status=row["status"], + revision=row["revision"], + actor_id=row["actor_id"], + resolved_at=row["resolved_at"], + resolution=resolution, + external_effects_may_continue=isinstance(resolution, dict) and resolution.get("kind") == "terminate", + instance_key=instance_key, + opened_at=row["opened_at"], + # A time is only estimated when it is known; an unknown opening time stays unknown. + opened_at_estimated=row["opened_at_estimated"] is True and row["opened_at"] is not None, + updated_at=row["updated_at"], + workflow=dict(name=row["workflow_name"], version=row["workflow_version"]), + run_status=row["run_status"], + test=row["test"], + ) + try: + return IncidentView.model_validate_json(json.dumps(values, default=_json)) + except (ValueError, TypeError, KeyError, OverflowError): + return unavailable(identifier) + + +def legacy_unavailable() -> CatalogError: + return CatalogError(409, "WV-LEGACY-UNAVAILABLE", "Step classification is unavailable") + + +def _segmented(key: str) -> bool: + parsed = split_instance(key) + return bool(parsed.segments) or parsed.count is not None + + +def step_order(row: Mapping[str, Any]) -> tuple[bool, datetime, str, str]: + at = row["scheduled_at"] + return at is None, at or datetime.max.replace(tzinfo=UTC), row["node_id"], row["instance_key"] + + +@dataclass +class _Candidate: + order: tuple[bool, datetime, str, str] + row: dict[str, Any] = field(compare=False) + + def __lt__(self, other: "_Candidate") -> bool: + return self.order > other.order + + +class FactReadRepository(FactRepository): + def __init__(self, tx: Transaction) -> None: + super().__init__(tx) + # One read unit is one snapshot, so a timeline reads its run metadata once. + self._metadata: dict[UUID, dict[str, Any]] = {} + + async def summary_rows(self, query: RunSummaryQuery, position: RunPosition | None) -> list[dict[str, Any]]: + return await self._run_rows(query, position) + + async def legacy_rows(self, query: RunListQuery, position: RunPosition | UUID | None) -> list[dict[str, Any]]: + return await self._run_rows(query, position) + + async def _run_rows(self, query: RunSummaryQuery, position: RunPosition | UUID | None) -> list[dict[str, Any]]: + clauses = ["(:include_archived OR NOT coalesce(a.archived,false))"] + params: dict[str, Any] = dict(include_archived=query.include_archived, limit=query.limit + 1) + for name, column in ( + ("workflow", "workflow_name"), + ("version", "workflow_version"), + ("origin", "origin"), + ("caller_run_id", "caller_run_id"), + ("retried_from_run_id", "retried_from_run_id"), + ("business_key", "business_key"), + ("correlation_key", "correlation_key"), + ("activation_id", "activation_id"), + ): + value = getattr(query, name) + if value is not None: + clauses.append(f"f.{column}=:{name}") + params[name] = value + if query.status: + clauses.append("f.status=ANY(:statuses)") + params["statuses"] = query.status + if not query.include_test: + clauses.append("NOT f.test") + if query.top_level_only: + clauses.append("f.caller_run_id IS NULL") + for name, column in (("has_active_incident", "active_incidents"), ("has_handled_errors", "handled_errors")): + value = getattr(query, name) + if value is not None: + clauses.append(f"f.{column}>0" if value else f"f.{column}=0") + for name, operator in (("started_after", ">="), ("started_before", "<")): + value = getattr(query, name) + if value is not None: + clauses.append(f"f.started_at{operator}:{name}") + params[name] = value + column, direction, operator = { + "id": ("run_id", "ASC", ">"), + "started_desc": ("started_at", "DESC", "<"), + "started_asc": ("started_at", "ASC", ">"), + "updated_desc": ("updated_at", "DESC", "<"), + }[query.order] + if column == "run_id": + if position is not None: + if not isinstance(position, UUID): + raise ValueError("ID order requires an ID position") + clauses.append("f.run_id>:position_id") + params["position_id"] = position + return await self.rows( + _RUN_SELECT + " AND " + " AND ".join(clauses) + " ORDER BY f.run_id ASC LIMIT :limit", **params + ) + if position is not None and not isinstance(position, RunPosition): + raise ValueError("Chronological order requires a time position") + # Known times, then the unknown-time tail, as two keyset scans in one snapshot. The known-time scan excludes + # nulls, so the scoped DESC (nulls first) time indexes serve it in both directions with the row comparison + # as an index bound; the tail is read only when known times do not fill the page. + found: list[dict[str, Any]] = [] + if position is None or position.at is not None: + known = [*clauses, f"f.{column} IS NOT NULL"] + bounds: dict[str, Any] = {} + if position is not None: + known.append(f"(f.{column},f.run_id){operator}(:position_at,:position_id)") + bounds = dict(position_at=position.at, position_id=position.id) + found = await self.rows( + _RUN_SELECT + + " AND " + + " AND ".join(known) + + f" ORDER BY f.{column} {direction},f.run_id {direction} LIMIT :limit", + **params, + **bounds, + ) + if len(found) > query.limit: + return found + unknown = [*clauses, f"f.{column} IS NULL"] + bounds = {"limit": query.limit + 1 - len(found)} + if position is not None and position.at is None: + unknown.append(f"f.run_id{operator}:position_id") + bounds["position_id"] = position.id + return found + await self.rows( + _RUN_SELECT + " AND " + " AND ".join(unknown) + f" ORDER BY f.run_id {direction} LIMIT :limit", + **{**params, **bounds}, + ) + + async def incident_rows( + self, query: IncidentQuery, position: RunPosition | UUID | None, run_id: UUID | None + ) -> list[dict[str, Any]]: + """The ordering keys of one incident page and its lookahead: incident IDs and opening times.""" + clauses: list[str] = [] + params: dict[str, Any] = dict(limit=query.limit + 1) + for name, column, value in ( + ("run", "i.run_id", run_id), + ("code", "i.code", query.code), + ("workflow", "f.workflow_name", query.workflow), + ("node", "f.node_id", query.node_id), + ): + if value is not None: + clauses.append(f"{column}=:{name}") + params[name] = value + if query.status: + clauses.append("i.status=ANY(:statuses)") + params["statuses"] = query.status + if not query.include_test: + clauses.append("NOT coalesce(f.test,false)") + for name, operator in (("opened_after", ">="), ("opened_before", "<")): + value = getattr(query, name) + if value is not None: + clauses.append(f"f.opened_at{operator}:{name}") + params[name] = value + if query.order == "id": + if isinstance(position, RunPosition): + raise ValueError("ID order requires an ID position") + if position is not None: + clauses.append("i.id>:position_id") + params["position_id"] = position + return await self.rows( + _INCIDENT_KEYS + "".join(" AND " + clause for clause in clauses) + " ORDER BY i.id ASC LIMIT :limit", + **params, + ) + if isinstance(position, UUID): + raise ValueError("Chronological order requires a time position") + direction, operator = ("DESC", "<") if query.order == "opened_desc" else ("ASC", ">") + # Known opening times, then the unknown-time tail, as two keyset scans in one snapshot (as run pages are + # read). The known-time scan excludes nulls, so the scoped opening-time index serves it in both directions + # with the row comparison as an index bound; the tail is read only when known times do not fill the page. + found: list[dict[str, Any]] = [] + if position is None or position.at is not None: + known = [*clauses, "f.opened_at IS NOT NULL"] + bounds: dict[str, Any] = {} + if position is not None: + known.append(f"(f.opened_at,f.incident_id){operator}(:position_at,:position_id)") + bounds = dict(position_at=position.at, position_id=position.id) + found = await self.rows( + _INCIDENT_KEYS + + "".join(" AND " + clause for clause in known) + + f" ORDER BY f.opened_at {direction},f.incident_id {direction} LIMIT :limit", + **params, + **bounds, + ) + if len(found) > query.limit: + return found + unknown = [*clauses, "f.opened_at IS NULL"] + tail: dict[str, Any] = {"limit": query.limit + 1 - len(found)} + if position is not None and position.at is None: + unknown.append(f"i.id{operator}:position_id") + tail["position_id"] = position.id + return found + await self.rows( + _INCIDENT_KEYS + + "".join(" AND " + clause for clause in unknown) + + f" ORDER BY i.id {direction} LIMIT :limit", + **{**params, **tail}, + ) + + async def incident_details(self, ids: list[UUID]) -> list[dict[str, Any]]: + """The selected incidents with their facts; a page above the logical byte limit is refused unread.""" + if not ids: + return [] + rows = await self.rows(_INCIDENT_DETAILS, ids=ids, budget=PAGE_BYTES) + if any(row["oversized"] for row in rows): + raise CatalogError(429, "WV-PAGE-LIMIT", "Selected page exceeds the logical byte limit") + return rows + + async def run_exists(self, run_id: UUID) -> bool: + return bool( + await self.tx.session.scalar( + text( + "SELECT EXISTS(SELECT 1 FROM runs r WHERE r.tenant_id=:tenant AND r.project_id=:project " + "AND r.environment_id=:environment AND r.id=:run)" + ), + {**self.params, "run": run_id}, + ) + ) + + async def page_bytes(self, run_ids: list[UUID]) -> int: + """Logical bytes of the retained runs on one page, measured before any of them is fetched.""" + if not run_ids: + return 0 + rows = await self.rows( + "SELECT coalesce(sum(public.weave_runs_bytes(r)),0) AS bytes FROM runs r " + "WHERE r.tenant_id=:tenant AND r.project_id=:project AND r.environment_id=:environment " + "AND r.id=ANY(:ids)", + ids=run_ids, + ) + return int(rows[0]["bytes"]) + + async def metadata(self, run_id: UUID) -> dict[str, Any]: + if run_id in self._metadata: + return self._metadata[run_id] + rows = await self.rows(_METADATA_SELECT + " AND f.run_id=:run", run=run_id) + if not rows: + raise CatalogError(404, "WV-NOT-FOUND", "Run not found") + row = rows[0] + if isinstance(summary_of(row), UnavailableResource): + raise withheld(row) + kinds = row.get("node_kinds") + if not isinstance(kinds, dict) or len(kinds) > 10000 or len(json.dumps(kinds).encode()) > 1048576: + raise legacy_unavailable() + self._metadata[run_id] = row + return row + + async def step_rows( + self, run_id: UUID, query: RunStepQuery, position: StepPosition | None + ) -> tuple[list[dict[str, Any]], bool]: + metadata = await self.metadata(run_id) + clauses = [_RUN_SCOPE, "f.run_id=:run", "left(f.node_id,1)<>'@'", "left(f.instance_key,1)<>'@'"] + params: dict[str, Any] = dict(run=run_id, limit=query.limit + 1) + if query.step is not None: + clauses.append("f.node_id=:step") + params["step"] = query.step + if position is not None: + params.update(node=position.node_id, instance=position.instance_key) + identity_predicate = ( + '(f.node_id COLLATE "C",f.instance_key COLLATE "C")>' + '(cast(:node AS text) COLLATE "C",cast(:instance AS text) COLLATE "C")' + ) + if position.scheduled_at is None: + clauses.append("(f.scheduled_at IS NULL AND " + identity_predicate + ")") + else: + params["scheduled"] = position.scheduled_at + clauses.append( + "(f.scheduled_at>:scheduled OR f.scheduled_at IS NULL OR (f.scheduled_at=:scheduled AND " + + identity_predicate + + "))" + ) + rows = await self.rows( + "SELECT f.* FROM step_facts f WHERE " + + " AND ".join(clauses) + + " ORDER BY " + + _STEP_ORDER + + " LIMIT :limit", + **params, + ) + for row in rows: + try: + identity = author_instance(row["instance_key"] or row["node_id"]) + except ValueError: + raise legacy_unavailable() from None + if ( + identity is None + or identity.node_id != row["node_id"] + or metadata["node_kinds"].get(row["node_id"]) != row["kind"] + ): + raise legacy_unavailable() + missing = await self.tx.session.scalar( + text("SELECT EXISTS(SELECT 1 FROM step_instances s WHERE " + _MISSING + ")"), {**self.params, "run": run_id} + ) + if not missing: + return rows, True + historical, complete = await self.historical_steps(run_id, query.step, position, query.limit) + return sorted([*rows, *historical], key=step_order)[: query.limit + 1], complete + + async def historical_steps( + self, run_id: UUID, step: str | None, position: StepPosition | None, limit: int + ) -> tuple[list[dict[str, Any]], bool]: + metadata = await self.metadata(run_id) + boundary = step_order(vars(position)) if position is not None else None + heap: list[_Candidate] = [] + stream = None + scanned = 0 + try: + async with asyncio.timeout(LEGACY_SECONDS): + # Unsorted on purpose: the bounded heap orders candidates, so the key cap also bounds the database. + stream = await self.tx.session.stream( + text("SELECT s.node_id,s.status FROM step_instances s WHERE " + _MISSING), + {**self.params, "run": run_id}, + execution_options={"yield_per": 500}, + ) + async for batch in stream.mappings().partitions(500): + for source in batch: + scanned += 1 + if scanned > LEGACY_KEYS: + raise CatalogError(429, "WV-RUNTIME-LIMIT", "Legacy step scan exceeds its bounded budget") + identity = author_instance(source["node_id"]) + if identity is None: + continue + kind: StepKind = _KIND.validate_python( + metadata["node_kinds"].get(identity.node_id), strict=True + ) + if source["status"] not in {"completed", "waiting"}: + raise legacy_unavailable() + row = dict( + node_id=identity.node_id, + instance_key=identity.instance_key, + kind=kind, + status="succeeded" if source["status"] == "completed" else "waiting", + scheduled_at=None, + started_at=None, + ended_at=None, + attempts=0, + worker_id=None, + error_code=None, + handled=None, + child_run_id=None, + ) + order = step_order(row) + if (step is not None and identity.node_id != step) or ( + boundary is not None and order <= boundary + ): + continue + candidate = _Candidate(order, row) + if len(heap) < limit + 1: + heapq.heappush(heap, candidate) + elif order < heap[0].order: + heapq.heapreplace(heap, candidate) + await asyncio.sleep(0) + except TimeoutError: + raise CatalogError(429, "WV-RUNTIME-LIMIT", "Legacy step scan exceeds its bounded budget") from None + except (ValueError, TypeError, KeyError): + raise legacy_unavailable() from None + finally: + if stream is not None: + await stream.close() + return [candidate.row for candidate in sorted(heap, key=lambda value: value.order)], scanned == 0 + + async def outputs(self, run_id: UUID, rows: list[dict[str, Any]]) -> dict[str, SafeProjection]: + from firefly_weave.compiler.api import import_artifact + from firefly_weave.compiler.ir import ActionNode, HumanTaskNode, SignalNode, WaitNode + from firefly_weave.contracts.definitions import ActionDefinition, load_definition + from firefly_weave.contracts.values import JsonObject + from firefly_weave.operations.redaction import project + from firefly_weave.runtime.kernel import action_schemas, workflow + from firefly_weave.runtime.repository import RuntimeRepository + + await self.metadata(run_id) + retained = await RuntimeRepository(self.tx).run(run_id) + admission.require_available(retained) + ir = workflow(import_artifact(retained["artifact"])) + bundle = { + dependency.reference: dependency.document for dependency in ir.dependencies if dependency.kind == "Schema" + } + nodes = {node.id: node for node in ir.graph.nodes} + keys = [row["instance_key"] or row["node_id"] for row in rows] + unclassified = SafeProjection( + available=False, omissions=[Omission(path="", reason="classification_unavailable")] + ) + result = dict.fromkeys(keys, unclassified) + # An agent turn, tool call or loop yield has no schema of its own yet; the step's schema would classify + # another value, so those outputs stay withheld and are never fetched. + keys = [key for key in keys if not _segmented(key)] + if not keys: + return result + predicate = ( + "tenant_id=:tenant AND project_id=:project AND environment_id=:environment AND run_id=:run " + "AND left(node_id,1)<>'@' AND node_id=ANY(:keys)" + ) + outputs = await self.rows( + "WITH sized AS (SELECT node_id,output,sum(octet_length(output::text)) OVER () AS total_bytes " + "FROM step_instances WHERE " + predicate + ") " + "SELECT node_id,CASE WHEN total_bytes<=:budget THEN output ELSE NULL END AS output," + "total_bytes>:budget AS oversized FROM sized", + run=run_id, + keys=keys, + budget=PAGE_BYTES, + ) + if any(output["oversized"] for output in outputs): + raise CatalogError(429, "WV-PAGE-LIMIT", "Selected page exceeds the logical byte limit") + for output in outputs: + identity = author_instance(output["node_id"]) + if identity is None: + continue + node = nodes.get(identity.node_id) + schemas: list[JsonObject] = [] + if isinstance(node, ActionNode): + definition = load_definition( + next(dependency.document for dependency in ir.dependencies if dependency.digest == node.dependency) + ) + schemas = action_schemas(ir, cast(ActionDefinition, definition), "output", node=node) + elif isinstance(node, SignalNode): + schemas = [ir.schemas[node.schema_ref]] + elif isinstance(node, HumanTaskNode): + schemas = [{"type": "object", "properties": {"data": node.form_schema, "decision": {"type": "string"}}}] + elif isinstance(node, WaitNode): + schemas = [{"type": "null"}] + elif node is not None: + schemas = [ + ir.schemas[guard.schema_ref] + for guard in ir.guards + if guard.path in {node.path, node.path + "/value"} + and guard.purpose in {"transform_output", "decision_output", "branch_output"} + ] + if schemas: + projections = [project(output["output"], schema, bundle) for schema in schemas] + result[output["node_id"]] = next( + (projection for projection in projections if not projection.available), projections[0] + ) + return result diff --git a/src/firefly_weave/operations/facts.py b/src/firefly_weave/operations/facts.py new file mode 100644 index 00000000..ea195c5f --- /dev/null +++ b/src/firefly_weave/operations/facts.py @@ -0,0 +1,569 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Transaction-bound SQL for operational projections.""" + +import json +import re +from dataclasses import dataclass +from datetime import datetime +from functools import partial +from typing import Any, Literal +from uuid import UUID + +from pydantic import TypeAdapter +from sqlalchemy import text + +from firefly_weave.contracts.catalog import Activation +from firefly_weave.contracts.definitions import ResourceName +from firefly_weave.contracts.instance_keys import InstanceKeyText, InstanceView, instance_view +from firefly_weave.contracts.run_views import RunOrigin, RunSummaryCaller, StepKind +from firefly_weave.contracts.runtime import RunView, StartRunRequest +from firefly_weave.persistence.uow import Transaction +from firefly_weave.runtime.models import ( + ControlCommand, + Deadline, + HumanTaskIntent, + RunState, + RuntimeEvent, + TaskIntent, + TerminalControl, + Transition, +) + +FACT_TABLES: tuple[str, str, str, str, str] = ( + "run_facts", + "step_facts", + "task_facts", + "incident_facts", + "worker_facts", +) + + +SCOPE = "tenant_id=:tenant AND project_id=:project AND environment_id=:environment" +TERMINAL = {"succeeded", "failed", "cancelled", "timed_out"} +_AUTHOR_KEY: TypeAdapter[str] = TypeAdapter(InstanceKeyText) +_AUTHOR_NODE: TypeAdapter[str] = TypeAdapter(ResourceName) +_STEP_KIND: TypeAdapter[StepKind] = TypeAdapter(StepKind) + + +def author_instance(key: str) -> InstanceView | None: + if type(key) is not str: + raise ValueError("Instance key must be text") + if key.startswith("@"): + return None + _AUTHOR_KEY.validate_python(key, strict=True) + value = instance_view(key) + _AUTHOR_NODE.validate_python(value.node_id, strict=True) + return value + + +def listing_metadata(row: dict[str, Any], *, imported: Any = None, ir: Any = None) -> dict[str, Any]: + """Bounded cache metadata; execution admission remains the runtime's authority. + + A caller that has already imported the row's artifact and compiled its workflow (a run start) passes them as + ``imported`` and ``ir``, so the artifact is not imported again while the caller holds its transaction. + """ + from firefly_weave.compiler.api import import_artifact + from firefly_weave.runtime import admission as retained + from firefly_weave.runtime.kernel import workflow + + result: dict[str, Any] = dict( + classification_state="unavailable", + classification_policy=retained.POLICY, + pinned_ir_version=None, + pinned_features=[], + node_kinds={}, + ) + try: + RunState.model_validate_json(json.dumps(row["state"])) + Activation.model_validate_json(json.dumps(row["activation"])) + StartRunRequest.model_validate_json(json.dumps(row["request"])) + decision = retained.admission(row, imported=imported) + executable = row["artifact"].get("executable", {}) + version, features = executable.get("irVersion"), executable.get("features", []) + if not isinstance(version, str) or len(version) > 256 or not isinstance(features, list): + return result + if len(features) > 256 or any(type(value) is not str for value in features): + return result + if len(json.dumps(features, ensure_ascii=False).encode()) > 16384: + return result + result.update(classification_state=decision, pinned_ir_version=version, pinned_features=features) + if decision == "available": + kinds: dict[str, StepKind] = {} + graph = ir if ir is not None else workflow(imported or import_artifact(row["artifact"])) + for node in graph.graph.nodes: + identity = author_instance(node.id) + if identity is not None: + kinds[identity.node_id] = _STEP_KIND.validate_python(node.kind, strict=True) + if len(kinds) > 10000 or len(json.dumps(kinds, ensure_ascii=False).encode()) > 1048576: + raise ValueError("Node-kind projection exceeds its bound") + result["node_kinds"] = kinds + except (ValueError, TypeError, KeyError, AttributeError, RecursionError): + result.update(classification_state="unavailable", node_kinds={}) + state = row.get("state") + status = state.get("status") if isinstance(state, dict) else None + if type(status) is not str or status not in {"queued", "running", "waiting", "suspended", *TERMINAL}: + result.update(classification_state="unavailable", node_kinds={}) + return result + + +REVALIDATION_BATCH = 100 + + +def revalidation_due(policy: str) -> str: + """SQL condition: the cached decision of fact row ``f`` is due for a refresh from its retained source. + + Due when an available or unsupported decision was recorded under another classification policy, or when it is a + transient unsupported decision whose pinned IR version and features this platform runs again. A policy change + never re-checks an unavailable decision: it can rest on evidence the source row does not show, such as an + incoherent event history found by the upgrade, so it stays unavailable. Bind + ``runtime.admission.runnable_parameters()``. The policy is written as a literal so the planner can prove that the + partial index ``run_facts_revalidation``, which holds the rows not classified under the current policy and the + unsupported ones, applies. + """ + if re.fullmatch(r"[a-z0-9][a-z0-9.-]{0,63}", policy) is None: + raise ValueError("Classification policy must be a plain identifier") + return ( + f"((f.classification_policy<>'{policy}' AND f.classification_state<>'unavailable') " + "OR (f.classification_state='unsupported' " + "AND f.pinned_ir_version=ANY(cast(:ir_versions AS text[])) " + "AND f.pinned_features <@ cast(:ir_features AS jsonb)))" + ) + + +def task_fact_update(source: str) -> str: + """Enlist task metadata in the existing authoritative source statement.""" + return ( + "WITH changed AS (" + source + " AND status<>'handled' RETURNING id,status,next_attempt_at) " + "UPDATE task_facts f SET status=c.status,ready_since=CASE WHEN c.status='ready' " + "THEN greatest(:at,coalesce(c.next_attempt_at,:at)) ELSE NULL END," + "first_claimed_at=CASE WHEN c.status='leased' THEN coalesce(f.first_claimed_at,:at) " + "ELSE f.first_claimed_at END FROM changed c WHERE f.task_id=c.id " + "AND f.tenant_id=:tenant AND f.project_id=:project AND f.environment_id=:environment" + ) + + +@dataclass(frozen=True) +class RunStartFacts: + origin: RunOrigin = "manual" + test: bool = False + caller: RunSummaryCaller | None = None + retried_from_run_id: UUID | None = None + + def __post_init__(self) -> None: + TypeAdapter(RunOrigin).validate_python(self.origin, strict=True) + if ( + type(self.test) is not bool + or (self.origin == "test" and not self.test) + or (self.test and self.origin not in {"test", "call"}) + ): + raise ValueError("Test flag requires a direct test or inherited call origin") + if (self.origin == "call") != (self.caller is not None): + raise ValueError("Call origin requires a caller") + if self.caller is not None: + if not isinstance(self.caller, RunSummaryCaller): + raise ValueError("Call origin requires typed caller metadata") + RunSummaryCaller.model_validate_json(self.caller.model_dump_json()) + if (self.origin == "retry") != (self.retried_from_run_id is not None): + raise ValueError("Retry origin requires a source run") + if self.retried_from_run_id is not None and not isinstance(self.retried_from_run_id, UUID): + raise ValueError("Retry source must be a UUID") + + +class FactRepository: + def __init__(self, tx: Transaction) -> None: + self.tx = tx + self.params = { + "tenant": tx.scope.tenant_id, + "project": tx.scope.project_id, + "environment": tx.scope.environment_id, + } + + async def execute(self, sql: str, **values: Any) -> None: + await self.tx.session.execute(text(sql), {**values, **self.params}) + + async def rows(self, sql: str, **values: Any) -> list[dict[str, Any]]: + result = await self.tx.session.execute(text(sql), {**values, **self.params}) + return [dict(row) for row in result.mappings()] + + async def revalidation_cursor(self) -> UUID | None: + """The environment's keyset boundary: the last run whose refresh was started.""" + value: UUID | None = await self.tx.session.scalar( + text(f"SELECT after_run_id FROM operations_fact_cursors WHERE {SCOPE}"), self.params + ) + return value + + async def advance_revalidation_cursor(self, after: UUID | None) -> None: + """Move the boundary; bookkeeping only, with no fact or source change and no storage charge.""" + await self.execute( + "INSERT INTO operations_fact_cursors(tenant_id,project_id,environment_id,after_run_id) " + "VALUES(:tenant,:project,:environment,:after) ON CONFLICT(tenant_id,project_id,environment_id) " + "DO UPDATE SET after_run_id=excluded.after_run_id", + after=after, + ) + + async def revalidation_candidates(self, after: UUID | None) -> list[UUID]: + """Up to ``REVALIDATION_BATCH`` due runs after ``after`` in UUID order, from fact metadata only.""" + from firefly_weave.runtime import admission + + rows = await self.rows( + "SELECT f.run_id FROM run_facts f WHERE f.tenant_id=:tenant AND f.project_id=:project " + f"AND f.environment_id=:environment AND {revalidation_due(admission.POLICY)} " + f"AND (cast(:after AS uuid) IS NULL OR f.run_id>:after) ORDER BY f.run_id LIMIT {REVALIDATION_BATCH}", + after=after, + **admission.runnable_parameters(), + ) + return [row["run_id"] for row in rows] + + async def refresh_listing_metadata(self, run_id: UUID) -> None: + """Recompute one run's cached listing decision from its retained source. + + Only the five classification columns change: never the run's state, timing or status, and never a runtime + transition, capacity block or policy block. Runtime admission stays the authority for execution. A source + larger than the bounded fetch is recorded unavailable without being read. A run whose fact changed since it + was selected, so that it is no longer due, keeps that newer decision. + """ + from firefly_weave.operations.execution import execute_pure + from firefly_weave.runtime import admission + from firefly_weave.runtime.repository import RUN_FETCH_BYTES, RuntimeRepository + + values = {**self.params, "run": run_id} + # Lock the source first, in the order runtime writers lock a run and then its facts. + locked = await self.tx.session.scalar(text(f"SELECT id FROM runs WHERE {SCOPE} AND id=:run FOR UPDATE"), values) + if locked is None: + return + # Rechecked under the lock: an authoritative update since the selection keeps its newer decision. + due = await self.tx.session.scalar( + text( + "SELECT f.run_id FROM run_facts f WHERE f.tenant_id=:tenant AND f.project_id=:project " + f"AND f.environment_id=:environment AND f.run_id=:run AND {revalidation_due(admission.POLICY)} " + "FOR UPDATE" + ), + {**values, **admission.runnable_parameters()}, + ) + if due is None: + return + # Measured under the lock, so the size cannot change before the fetch. + size = await self.tx.session.scalar( + text(f"SELECT public.weave_runs_bytes(runs) FROM runs WHERE {SCOPE} AND id=:run"), values + ) + if size is None or size > RUN_FETCH_BYTES: + await self.execute( + "UPDATE run_facts SET classification_state='unavailable',classification_policy=:policy " + f"WHERE {SCOPE} AND run_id=:run", + policy=admission.POLICY, + run=run_id, + ) + return + row = await RuntimeRepository(self.tx).run(run_id, lock=True) + # A refresh is never a runtime candidate: a failed write must not leave a capacity block on the run. + self.tx.session.info.pop("weave_runtime_candidate", None) + metadata = await execute_pure(partial(listing_metadata, row)) + await self.execute( + "UPDATE run_facts SET classification_state=:classification_state," + "classification_policy=:classification_policy,pinned_ir_version=:pinned_ir_version," + "pinned_features=cast(:pinned_features AS jsonb),node_kinds=cast(:node_kinds AS jsonb) " + f"WHERE {SCOPE} AND run_id=:run", + run=run_id, + classification_state=metadata["classification_state"], + classification_policy=metadata["classification_policy"], + pinned_ir_version=metadata["pinned_ir_version"], + pinned_features=json.dumps(metadata["pinned_features"], ensure_ascii=False), + node_kinds=json.dumps(metadata["node_kinds"], ensure_ascii=False), + ) + + async def insert_run( + self, + view: RunView, + artifact: dict[str, Any], + request: StartRunRequest, + facts: RunStartFacts, + at: datetime, + *, + imported: Any = None, + ir: Any = None, + ) -> None: + metadata = listing_metadata( + dict( + state=view.state.model_dump(mode="json"), + artifact=artifact, + activation=view.activation.model_dump(mode="json"), + request=request.model_dump(mode="json"), + ), + imported=imported, + ir=ir, + ) + await self.execute( + "INSERT INTO run_facts(tenant_id,project_id,environment_id,run_id,definition_version_id," + "workflow_name,workflow_version,activation_id,activation_name,activation_revision,status,paused," + "test,origin,caller_run_id,caller_node_id,caller_instance_key,retried_from_run_id,business_key," + "correlation_key,started_at,updated_at,classification_state,classification_policy,pinned_ir_version," + "pinned_features,node_kinds,facts_started_at) SELECT :tenant,:project,:environment,:run,:version," + "name,version,:activation,:name,:revision,:status,:paused,:test,:origin,:caller_run,:caller_node," + ":caller_instance,:retried,:business,:correlation,:at,:at,:classification_state,:classification_policy," + ":pinned_ir_version,cast(:pinned_features AS jsonb),cast(:node_kinds AS jsonb),:at " + "FROM definition_versions WHERE tenant_id=:tenant AND project_id=:project AND id=:version", + run=view.id, + version=view.activation.request.version_id, + activation=view.activation.id, + name=view.activation.name, + revision=view.activation.revision, + status=view.state.status, + paused=view.state.manual_paused, + test=facts.test, + origin=facts.origin, + caller_run=facts.caller.run_id if facts.caller else None, + caller_node=facts.caller.node_id if facts.caller else None, + caller_instance=facts.caller.instance_key if facts.caller else None, + retried=facts.retried_from_run_id, + business=request.business_key, + correlation=request.correlation_key, + at=at, + **{ + **metadata, + "pinned_features": json.dumps(metadata["pinned_features"], ensure_ascii=False), + "node_kinds": json.dumps(metadata["node_kinds"], ensure_ascii=False), + }, + ) + + def task_values( + self, task_id: UUID, run_id: UUID, command: TaskIntent, task_type: str, at: datetime + ) -> dict[str, Any]: + identity = author_instance(command.node_id) + if identity is None: + raise ValueError("Task requires an author step") + return dict( + id=task_id, + run=run_id, + fact_node=identity.node_id, + instance=identity.instance_key, + task_type=task_type, + at=at, + ) + + async def project_run(self, view: RunView | TerminalControl, event: RuntimeEvent, result: Transition) -> None: + failed = None + if result.state.status == "failed": + key = next( + (c.node_id for c in result.commands if isinstance(c, ControlCommand) and c.kind == "terminate"), + event.data.get("node_id"), + ) + failed = author_instance(key) if isinstance(key, str) else None + rows = await self.rows( + f"UPDATE run_facts SET status=:status,paused=:paused,updated_at=greatest(updated_at,:at)," + "ended_at=CASE WHEN :terminal THEN :at ELSE ended_at END,failed_node_id=:failed_node," + "failed_instance_key=:failed_instance,failed_error_code=:code,last_event_sequence=:sequence," + "classification_state=CASE WHEN :unavailable THEN 'unavailable' ELSE classification_state END," + f"active_incidents=(SELECT count(*) FROM incidents WHERE {SCOPE} AND run_id=:run AND status='active') " + f"WHERE {SCOPE} AND run_id=:run AND last_event_sequence<:sequence RETURNING node_kinds", + run=view.id, + status=result.state.status, + paused=result.state.manual_paused, + at=event.timestamp, + terminal=result.state.status in TERMINAL, + failed_node=failed.node_id if failed else None, + failed_instance=failed.instance_key if failed else None, + code=result.state.incident if result.state.status == "failed" else None, + sequence=event.sequence, + unavailable=result.state.unavailable, + ) + if not rows: + return + kinds = rows[0]["node_kinds"] + steps: dict[tuple[str, str], dict[str, Any]] = {} + + def kind_of(node: str) -> StepKind | None: + # Retained runs can be admitted even when their listing cache is incomplete. + try: + return _STEP_KIND.validate_python(kinds.get(node), strict=True) + except ValueError: + return None + + def add(key: str, status: str, *, scheduled: bool = False, error: str | None = None) -> None: + # A loop's yield (each~1) is a wake-up of the loop, not a run of a step. + identity = author_instance(key) if "~" not in key else None + if identity is None: + return + kind = kind_of(identity.node_id) + observed = scheduled or kind in {"transform", "decisionTable", "fail"} + previous = steps.get((identity.node_id, identity.instance_key), {}) + at = event.timestamp.isoformat() + steps[identity.node_id, identity.instance_key] = dict( + node_id=identity.node_id, + instance_key=identity.instance_key, + kind=kind, + status=status, + scheduled_at=previous.get("scheduled_at") or (at if observed else None), + started_at=previous.get("started_at") + or (at if observed and status != "scheduled" and kind not in {"action", "llm", "agent"} else None), + ended_at=at if status in TERMINAL else None, + error_code=error, + ) + + if not isinstance(view, TerminalControl): + tasks = {command.node_id for command in result.commands if isinstance(command, TaskIntent)} + for command in result.commands: + if isinstance(command, (TaskIntent, Deadline, HumanTaskIntent)): + add(command.node_id, "scheduled" if isinstance(command, TaskIntent) else "waiting", scheduled=True) + elif command.kind == "spawn_branch": + join = result.state.joins.get(command.node_id) + first = result.state.branches.get(join.branches[0]) if join and join.branches else None + # Later branch dispatches do not witness the join's original activation. + add(command.node_id, "running", scheduled=first is not None and command.branch == first.name) + for change in result.steps: + identity = author_instance(change.node_id) + if identity is None: + continue + kind = kind_of(identity.node_id) + status = ( + "succeeded" + if change.status == "completed" + else ("scheduled" if kind in {"action", "llm", "agent"} or change.node_id in tasks else "waiting") + ) + add(change.node_id, status, scheduled=change.status == "waiting") + for owner, join in result.state.joins.items(): + if join.status == "waiting" and (owner, "") not in steps: + add(owner, "running") + node = event.data.get("node_id") + if isinstance(node, str) and any(value.id == event.id for value in result.state.deferred_results): + add(node, "succeeded") + if isinstance(node, str) and event.type in {"task_failed", "incident_opened", "recovery_scheduled"}: + output = event.data.get("output") + code = output.get("code") if isinstance(output, dict) else event.data.get("code") + add( + node, + "scheduled" if event.type == "recovery_scheduled" else "failed", + error=code if isinstance(code, str) else None, + ) + if ( + isinstance(node, str) + and event.type == "incident_resolved" + and event.data.get("kind") == "retry_safe" + and result.state.status not in TERMINAL + ): + add(node, "scheduled") + if failed: + add(failed.instance_key or failed.node_id, "failed", error=result.state.incident) + if not steps and result.state.status not in TERMINAL: + return + closed = result.state.status if result.state.status in {"cancelled", "timed_out"} else "cancelled" + if result.state.status in TERMINAL: + for step in steps.values(): + if step["status"] in {"scheduled", "running", "waiting"}: + step.update(status=closed, ended_at=event.timestamp.isoformat()) + await self.execute( + "WITH projected AS (INSERT INTO step_facts(tenant_id,project_id,environment_id,run_id,node_id," + "instance_key,kind,status,scheduled_at,started_at,ended_at,error_code) " + "SELECT :tenant,:project,:environment,:run,s.node_id,s.instance_key,coalesce(s.kind,known.kind)," + "s.status,s.scheduled_at," + "s.started_at,s.ended_at,s.error_code FROM jsonb_to_recordset(cast(:steps AS jsonb)) " + "AS s(node_id text,instance_key text,kind text,status text,scheduled_at timestamptz,started_at timestamptz," + "ended_at timestamptz,error_code text) LEFT JOIN step_facts known ON known.run_id=:run " + "AND known.tenant_id=:tenant AND known.project_id=:project AND known.environment_id=:environment " + "AND known.node_id=s.node_id AND known.instance_key=s.instance_key " + "WHERE coalesce(s.kind,known.kind) IS NOT NULL ON CONFLICT(run_id,node_id,instance_key) DO UPDATE SET " + "status=excluded.status,scheduled_at=coalesce(step_facts.scheduled_at,excluded.scheduled_at)," + "started_at=coalesce(step_facts.started_at,excluded.started_at)," + "ended_at=CASE WHEN step_facts.status=excluded.status THEN coalesce(step_facts.ended_at,excluded.ended_at) " + "ELSE excluded.ended_at END,error_code=excluded.error_code WHERE step_facts.handled IS NULL " + "RETURNING node_id,instance_key) UPDATE step_facts SET status=:closed,ended_at=:at " + f"WHERE {SCOPE} AND run_id=:run AND :terminal AND status IN ('scheduled','running','waiting') " + "AND NOT EXISTS(SELECT 1 FROM projected p WHERE p.node_id=step_facts.node_id " + "AND p.instance_key=step_facts.instance_key)", + run=view.id, + at=event.timestamp, + steps=json.dumps(list(steps.values())), + terminal=result.state.status in TERMINAL, + closed=closed, + ) + + async def step_claimed(self, task_id: UUID, generation: int, worker_id: UUID, at: datetime) -> None: + await self.execute( + "UPDATE step_facts s SET started_at=coalesce(s.started_at,:at),status='running'," + "ended_at=NULL,error_code=NULL,attempts=greatest(s.attempts,:generation),worker_id=:worker " + "FROM task_facts f WHERE f.task_id=:task AND f.status='leased' " + "AND f.tenant_id=:tenant AND f.project_id=:project AND f.environment_id=:environment " + "AND s.tenant_id=f.tenant_id AND s.project_id=f.project_id AND s.environment_id=f.environment_id " + "AND s.run_id=f.run_id AND s.node_id=f.node_id AND s.instance_key=f.instance_key AND s.handled IS NULL", + task=task_id, + generation=generation, + worker=worker_id, + at=at, + ) + + async def project_terminal( + self, run_id: UUID, status: Literal["cancelled", "timed_out"], at: datetime, sequence: int + ) -> None: + if status not in {"cancelled", "timed_out"}: + raise ValueError("Terminal control status required") + await self.execute( + "WITH changed AS (UPDATE run_facts SET status=:status,updated_at=greatest(updated_at,:at),ended_at=:at," + "failed_node_id=NULL,failed_instance_key=NULL,failed_error_code=NULL,active_incidents=0," + f"last_event_sequence=:sequence WHERE {SCOPE} AND run_id=:run AND last_event_sequence<:sequence " + "RETURNING run_id), tasks AS (UPDATE task_facts f SET status=t.status,ready_since=NULL FROM " + "task_intents t WHERE f.task_id=t.id AND f.run_id IN (SELECT run_id FROM changed) " + "AND f.tenant_id=:tenant AND f.project_id=:project AND f.environment_id=:environment " + "AND t.tenant_id=f.tenant_id AND t.project_id=f.project_id AND t.environment_id=f.environment_id " + f"RETURNING f.task_id) UPDATE step_facts SET status=:status,ended_at=:at WHERE {SCOPE} " + "AND run_id IN (SELECT run_id FROM changed) AND status IN ('scheduled','running','waiting')", + run=run_id, + status=status, + at=at, + sequence=sequence, + ) + + async def handled_failure( + self, + run_id: UUID, + key: str, + kind: StepKind, + disposition: Literal["continue", "errorOutput"], + error_code: str, + at: datetime, + *, + task_id: UUID | None = None, + ) -> None: + identity = author_instance(key) + if identity is None or disposition not in {"continue", "errorOutput"}: + raise ValueError("Handled failure requires an author step and disposition") + _STEP_KIND.validate_python(kind, strict=True) + if task_id is not None: + tasks = await self.rows( + f"SELECT run_id,node_id,status FROM task_intents WHERE {SCOPE} AND id=:task", task=task_id + ) + if len(tasks) != 1 or tasks[0] != dict(run_id=run_id, node_id=key, status="handled"): + raise ValueError("Handled task authority does not match the projection") + await self.execute( + "WITH changed AS (INSERT INTO step_facts(tenant_id,project_id,environment_id,run_id,node_id," + "instance_key,kind,status,ended_at,error_code,handled) VALUES(:tenant,:project,:environment,:run," + ":node,:instance,:kind,'failed',:at,:code,:handled) ON CONFLICT(run_id,node_id,instance_key) " + "DO UPDATE SET status='failed',ended_at=excluded.ended_at,error_code=excluded.error_code," + "handled=excluded.handled WHERE step_facts.handled IS NULL RETURNING run_id) " + f"UPDATE run_facts SET handled_errors=handled_errors+1 WHERE {SCOPE} " + "AND run_id IN (SELECT run_id FROM changed)", + run=run_id, + node=identity.node_id, + instance=identity.instance_key, + kind=kind, + at=at, + code=error_code, + handled=disposition, + ) + if task_id is not None: + await self.execute( + f"UPDATE task_facts SET status='handled',ready_since=NULL WHERE {SCOPE} AND task_id=:task", task=task_id + ) diff --git a/src/firefly_weave/operations/incidents.py b/src/firefly_weave/operations/incidents.py index d0235886..071374d1 100644 --- a/src/firefly_weave/operations/incidents.py +++ b/src/firefly_weave/operations/incidents.py @@ -26,15 +26,21 @@ from firefly_weave.access.audit import AuditContext from firefly_weave.access.models import Principal from firefly_weave.access.service import AccessService, audit +from firefly_weave.api.transport import encode_cursor, encode_cursor_v2 from firefly_weave.compiler.api import import_artifact from firefly_weave.compiler.canonical import canonical_digest from firefly_weave.compiler.expressions import measure_value from firefly_weave.compiler.ir import ActionNode from firefly_weave.contracts.access import Scope from firefly_weave.contracts.definitions import ActionDefinition, load_definition +from firefly_weave.contracts.incident_queries import IncidentQuery from firefly_weave.contracts.instance_keys import node_of from firefly_weave.contracts.operations import IncidentResolution, IncidentView +from firefly_weave.contracts.public import UnavailableResource from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.fact_positions import RunPosition +from firefly_weave.operations.fact_reads import FactReadRepository, incident_of +from firefly_weave.operations.facts import task_fact_update from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.kernel import transition_async as transition from firefly_weave.runtime.kernel import validate_action, workflow @@ -212,16 +218,20 @@ async def resolve( await repository.persist(view, event, result, event_hash(event)) if task is not None and result.state.status not in TERMINAL: await repository.execute( - f"UPDATE task_intents SET status=:status,next_attempt_at=NULL WHERE {SCOPE} AND id=:id", + task_fact_update( + f"UPDATE task_intents SET status=:status,next_attempt_at=NULL WHERE {SCOPE} AND id=:id" + ), + at=event.timestamp, id=task["id"], status="ready" if request.kind == "retry_safe" else "completed", ) elif task is not None and request.kind == "accept_reconciled_result": - await repository.task_status(task["id"], "completed") + await repository.task_status(task["id"], "completed", at=event.timestamp) await repository.execute( - "UPDATE incidents SET status='resolved',actor_id=:actor,resolved_at=:now," + "WITH changed AS (UPDATE incidents SET status='resolved',actor_id=:actor,resolved_at=:now," "resolution=cast(:resolution AS jsonb) " - f"WHERE {SCOPE} AND id=:id", + f"WHERE {SCOPE} AND id=:id RETURNING id) UPDATE incident_facts SET updated_at=:now " + f"WHERE {SCOPE} AND incident_id IN (SELECT id FROM changed)", id=incident_id, actor=actor.id, now=event.timestamp, @@ -257,6 +267,18 @@ async def resolve( await settle(repository, await repository.run(previous.id)) return resolved + async def read( + self, actor: Principal, scope: Scope, identifier: UUID, *, context: AuditContext + ) -> IncidentView | UnavailableResource: + """One incident from its facts; the run's classification metadata decides whether it is withheld.""" + self.runtime.require(actor, scope, "incident.read", context) + async with self.runtime.definitions.transaction(scope, None, mutation=False) as tx: + await self._require_current(actor, scope, tx, context) + rows = await FactReadRepository(tx).incident_details([identifier]) + if not rows: + raise CatalogError(404, "WV-NOT-FOUND", "Incident not found") + return incident_of(rows[0]) + async def list_all( self, actor: Principal, @@ -265,32 +287,47 @@ async def list_all( limit: int = 50, cursor: UUID | None = None, run_id: UUID | None = None, + query: IncidentQuery | None = None, + position: RunPosition | UUID | None = None, context: AuditContext, ) -> dict[str, Any]: - from firefly_weave.persistence.paging import page_ids + """A page of incidents read from facts, never from run payloads. + The keyword arguments are the original contract: every incident in ID order, with raw ID cursors in both + directions. A typed query selects filters and order, and its decoded position continues a page. + """ + keywords = query is None + if query is None: + if type(limit) is not int or not 1 <= limit <= 100: + raise CatalogError(422, "WV-PAGE", "A bounded page is required") + query, position = IncidentQuery(order="id", limit=limit), cursor + elif cursor is not None or limit != 50: + raise ValueError("Use either a query or legacy list arguments") + if query.order == "id" and isinstance(position, RunPosition): + raise ValueError("ID order requires an ID position") + if query.order != "id" and isinstance(position, UUID): + raise ValueError("Chronological order requires a time position") + self.runtime.require(actor, scope, "incident.read", context) async with self.runtime.definitions.transaction(scope, None, mutation=False) as tx: - actor = await self.access.load_principal(actor.id, tx=tx) - self.runtime.require(actor, scope, "incident.read", context) - if run_id is not None: - await RuntimeRepository(tx, self.runtime.definitions.outbox).run(run_id) - ids = await page_ids(tx, "incidents", limit, cursor, run_id=run_id) - items = [] - repository = RuntimeRepository(tx, self.runtime.definitions.outbox) - for identifier in ids[:limit]: - row = (await repository.rows(f"SELECT * FROM incidents WHERE {SCOPE} AND id=:id", id=identifier))[0] - try: - view_of(await repository.run(row["run_id"])) - items.append(incident_view(row).model_dump(mode="json")) - except CatalogError as error: - # A run waiting for an upgrade (ir_unsupported) is listed like legacy evidence. - if error.code not in {"WV-LEGACY-UNAVAILABLE", "WV-IR-UNSUPPORTED"}: - raise - items.append( - { - "id": str(identifier), - "unavailable": True, - "omissions": [{"path": "", "reason": "classification_unavailable"}], - } - ) - return {"items": items, "next_cursor": str(ids[limit - 1]) if len(ids) > limit else None} + await self._require_current(actor, scope, tx, context) + repository = FactReadRepository(tx) + if run_id is not None and not await repository.run_exists(run_id): + raise CatalogError(404, "WV-NOT-FOUND", "Run not found") + rows = await repository.incident_rows(query, position, run_id) + visible = rows[: query.limit] + details = {row["id"]: row for row in await repository.incident_details([row["id"] for row in visible])} + items = [incident_of(details[row["id"]]).model_dump(mode="json") for row in visible] + next_cursor = None + if len(rows) > query.limit: + last = visible[-1] + collection = query.cursor_collection(run_id) + if query.order == "id": + next_cursor = str(last["id"]) if keywords else encode_cursor(scope, collection, last["id"]) + else: + at = last["opened_at"] + next_cursor = encode_cursor_v2(scope, collection, at.isoformat() if at else None, str(last["id"])) + return {"items": items, "next_cursor": next_cursor} + + async def _require_current(self, actor: Principal, scope: Scope, tx: Transaction, context: AuditContext) -> None: + """`incident.read` on the principal stored now, in the read transaction; the token was checked first.""" + self.runtime.require(await self.access.load_principal(actor.id, tx=tx), scope, "incident.read", context) diff --git a/src/firefly_weave/operations/loop.py b/src/firefly_weave/operations/loop.py new file mode 100644 index 00000000..590d772f --- /dev/null +++ b/src/firefly_weave/operations/loop.py @@ -0,0 +1,229 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""The lifespan-owned operations loop: bounded per-environment quanta under execute-only traversal authority. + +Every minute it pages tenants with its own cursors and gives one environment per tenant a budget of one second of +database work. Today that work refreshes cached run listing decisions; it never grants execution authority. +""" + +import asyncio +import logging +import time +from collections.abc import AsyncIterator, Callable +from contextlib import asynccontextmanager, suppress +from contextvars import Context +from datetime import datetime + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine + +from firefly_weave.access.authorization import AccessDenied +from firefly_weave.access.scheduler import _SchedulerScope, next_scope, tenant_page +from firefly_weave.operations.execution import ReservedSlots +from firefly_weave.operations.facts import FactRepository +from firefly_weave.operations.loop_health import OPERATIONS_SECONDS, LoopHealth, track, utc_now +from firefly_weave.persistence.migrations import check_schema +from firefly_weave.persistence.uow import Transaction, UnitOfWork +from firefly_weave.runtime.scheduler import failure_cause +from firefly_weave.settings import Settings + +# Database work one environment may use per cycle. A quantum that runs out resumes on a later cycle. +ENVIRONMENT_SECONDS = 1.0 +# The wall-clock guard waits this much longer, so the database's own timeout normally ends a slow statement. +CANCEL_GRACE_SECONDS = 0.25 +# A cycle runs one pure classification at a time; the spare slot absorbs one that outlived its cancelled quantum. +OPERATIONS_EXECUTION = ReservedSlots(2) + + +class QuantumIncomplete(Exception): + """An environment quantum that failed a row or ran out of budget; its committed progress stays.""" + + +class OperationsLoop: + def __init__( + self, + settings: Settings, + uow: UnitOfWork, + health: LoopHealth, + *, + clock: Callable[[], datetime] = utc_now, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self.settings, self.uow, self.health = settings, uow, health + self.clock, self.monotonic = clock, monotonic + self.engine: AsyncEngine | None = None + self.task: asyncio.Task[None] | None = None + + async def open(self) -> None: + if self.task is not None or not self.settings.scheduler_enabled: + return + if self.settings.scheduler_database_url is None: + raise RuntimeError( + "The operations loop requires WEAVE_SCHEDULER_DATABASE_URL; API-only replicas explicitly disable " + "the scheduler" + ) + self.engine = create_async_engine(self.settings.scheduler_database_url.get_secret_value(), hide_parameters=True) + try: + async with asyncio.timeout(self.settings.database_timeout_seconds), self.engine.connect() as connection: + await check_schema(connection) + valid = await connection.scalar( + text( + "SELECT NOT rolsuper AND NOT rolbypassrls AND " + "pg_has_role(current_user,'weave_scheduler','MEMBER') AND NOT " + "pg_has_role(current_user,'weave_app','MEMBER') AND NOT " + "pg_has_role(current_user,'weave_catalog_reader','MEMBER') AND NOT " + "has_table_privilege(current_user,'run_facts','SELECT') AND NOT " + "has_table_privilege(current_user,'tenants','SELECT') AND " + "has_function_privilege(current_user,'weave_operations_tenants(integer)','EXECUTE') " + "FROM pg_roles WHERE rolname=current_user" + ) + ) + if not valid: + raise RuntimeError("Operations traversal requires execute-only scheduler authority") + # A request can open this loop. An empty context keeps the request's execution + # lease and identity, which end with its response, out of the long-lived task. + self.task = asyncio.create_task(self.poll(), name="weave-operations", context=Context()) + track(self.health, self.task, "operations_loop") + except BaseException: + await self.close() + raise + + async def poll(self) -> None: + while True: + self.health.started("operations_loop") + try: + healthy = await self.cycle() + except asyncio.CancelledError: + raise + except Exception as error: + logging.getLogger(__name__).error( + "Operations traversal failed (%s); retrying on the next cycle", failure_cause(error) + ) + healthy = False + # Coverage times are reported only once statistics exist, so a success carries none. + if healthy: + self.health.succeeded("operations_loop") + else: + self.health.failed("operations_loop") + await asyncio.sleep(OPERATIONS_SECONDS) + + async def cycle(self) -> bool: + """One bounded traversal; True only when every visited environment's quantum completed.""" + assert self.engine is not None + healthy = True + async with OPERATIONS_EXECUTION.execution(): + budget = min(self.settings.database_timeout_seconds, 5.0) + async with asyncio.timeout(budget): + tenants = await tenant_page(self.engine, operations=True) + for tenant in tenants: + try: + async with asyncio.timeout(budget): + authority = await next_scope(self.uow, tenant, operations=True) + if authority is None: + continue + await self._environment(authority) + except Exception as error: + logging.getLogger(__name__).error( + "Operations environment quantum failed (%s); traversal will continue", failure_cause(error) + ) + healthy = False + return healthy + + async def _environment(self, authority: _SchedulerScope) -> None: + deadline = self.monotonic() + ENVIRONMENT_SECONDS + async with asyncio.timeout(ENVIRONMENT_SECONDS + CANCEL_GRACE_SECONDS): + if not await self.revalidate(authority, deadline): + raise QuantumIncomplete() + + @asynccontextmanager + async def _unit( + self, authority: _SchedulerScope, deadline_monotonic: float, *, mutation: bool + ) -> AsyncIterator[Transaction]: + """An ordinary scoped transaction, verified as scheduler work and limited to the remaining budget.""" + if self.monotonic() >= deadline_monotonic: + raise QuantumIncomplete() + async with self.uow.open(authority.scope, mutation=mutation) as tx: + await authority.verify(tx) + remaining = deadline_monotonic - self.monotonic() + if remaining <= 0: + raise QuantumIncomplete() + limit = f"{max(1, min(1000, int(remaining * 1000)))}ms" + await tx.session.execute( + text("SELECT set_config('statement_timeout',:limit,true),set_config('lock_timeout',:limit,true)"), + {"limit": limit}, + ) + yield tx + + async def revalidate(self, authority: _SchedulerScope, deadline_monotonic: float) -> bool: + """Refresh due cached listing decisions in one environment; True when this quantum completed without failure. + + The keyset boundary is committed before each run's refresh, in its own transaction, so a run that fails or + outlives the budget never pins the environment: later runs advance, and the failed one is retried after + the boundary wraps. Each run's refresh commits on its own; a failure rolls back only that run. + """ + if not isinstance(authority, _SchedulerScope): + raise AccessDenied() + log = logging.getLogger(__name__) + try: + async with self._unit(authority, deadline_monotonic, mutation=False) as tx: + facts = FactRepository(tx) + after = await facts.revalidation_cursor() + candidates = await facts.revalidation_candidates(after) + except Exception as error: + log.error("Operations revalidation selection failed (%s)", failure_cause(error)) + return False + if not candidates: + if after is None: + return True + # Wrap once: the next cycle starts again from the first due run. + try: + async with self._unit(authority, deadline_monotonic, mutation=True) as tx: + await FactRepository(tx).advance_revalidation_cursor(None) + except Exception as error: + log.error("Operations revalidation cursor update failed (%s)", failure_cause(error)) + return False + return True + completed = True + for run_id in candidates: + if self.monotonic() >= deadline_monotonic: + return False + try: + async with self._unit(authority, deadline_monotonic, mutation=True) as tx: + await FactRepository(tx).advance_revalidation_cursor(run_id) + except Exception as error: + # Without a committed boundary the run is not attempted, so no progress is claimed. + log.error("Operations revalidation cursor update failed (%s)", failure_cause(error)) + return False + try: + async with self._unit(authority, deadline_monotonic, mutation=True) as tx: + await FactRepository(tx).refresh_listing_metadata(run_id) + except Exception as error: + log.error("Operations revalidation of a run failed (%s); later runs continue", failure_cause(error)) + completed = False + return completed + + async def close(self) -> None: + if self.task is not None: + self.task.cancel() + with suppress(asyncio.CancelledError): + await self.task + self.task = None + self.health.running("operations_loop", False) + # Cancellation completes before the engine it uses is disposed. + if self.engine is not None: + await self.engine.dispose() + self.engine = None diff --git a/src/firefly_weave/operations/loop_health.py b/src/firefly_weave/operations/loop_health.py new file mode 100644 index 00000000..07aa1a41 --- /dev/null +++ b/src/firefly_weave/operations/loop_health.py @@ -0,0 +1,259 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""In-process health of the background loops; a loop reads as healthy only after it proved a recent cycle. + +The API process that runs the scheduler is a singleton, so this per-process registry is authoritative for it. +Each loop reports its own cycles. Reporting never raises into the loop, and nothing here is per tenant. +""" + +import asyncio +import logging +import math +import time +from collections.abc import Callable +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Literal + +from firefly_weave.settings import Settings + +type LoopState = Literal["ok", "degraded", "down", "disabled"] + +# The fixed set of components: health is per process and per loop, never per tenant or environment. +COMPONENTS = ( + "recovery_loop", + "schedules", + "outbox_loop", + "provider_loop", + "kafka_loop", + "operations_loop", + "alert_evaluator", +) +FAILURES_DEGRADED = 3 +INTERVALS_STALE = 3 +# Failures are also counted over this many recent cycles, so failures that alternate with successes, such as one +# permanently failing run or tenant in a rotating traversal, still degrade the loop. +RECENT_CYCLES = 20 +# The outbox, provider and broker traversals poll every second; the operations loop every minute. +TRAVERSAL_SECONDS = 1.0 +OPERATIONS_SECONDS = 60.0 +# How long one healthy cycle may take. A cycle still running within its budget is on time, however many poll +# intervals that is. Outbox and broker traversals time out after 5 s, a recovery cycle scans up to 16 tenants, and a +# provider cycle can wait on one mailbox poll. +TRAVERSAL_BUDGET_SECONDS = 5.0 +RECOVERY_BUDGET_SECONDS = 30.0 +PROVIDER_BUDGET_SECONDS = 60.0 +_FAILURE_CEILING = 1_000_000_000 +_RECENT_MASK = (1 << RECENT_CYCLES) - 1 + + +def utc_now() -> datetime: + return datetime.now(UTC) + + +@dataclass(frozen=True) +class LoopSnapshot: + component: str + state: LoopState + last_started_at: datetime | None + last_ok_at: datetime | None + consecutive_failures: int + complete_until: datetime | None + # Failed cycles among the last RECENT_CYCLES cycles. + recent_failures: int = 0 + + +@dataclass +class _Record: + interval: float + budget: float + enabled: bool + running: bool = False + # Wall-clock times, for display only. + last_started_at: datetime | None = None + last_ok_at: datetime | None = None + complete_until: datetime | None = None + # Monotonic times, for aging: the start of the cycle in flight and the last success. + started: float | None = None + ok: float | None = None + consecutive_failures: int = 0 + # The outcomes of the most recent cycles, newest in the lowest bit; a set bit is a failed cycle. + outcomes: int = 0 + + +def _utc(value: datetime) -> datetime: + if not isinstance(value, datetime) or value.tzinfo is None or value.utcoffset() is None: + raise ValueError("Loop health times must be timezone-aware") + return value.astimezone(UTC) + + +def _seconds(value: object, *, positive: bool) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < 0 + or (positive and value == 0) + ): + raise ValueError("Loop intervals and budgets must be finite numbers of seconds") + return float(value) + + +class LoopHealth: + """Per-loop start, success and failure records, judged against the loop's poll interval. + + ``disabled`` when the loop is turned off, ``down`` while its task is not running, ``degraded`` before its first + successful cycle, after ``FAILURES_DEGRADED`` consecutive failed cycles or as many among its last + ``RECENT_CYCLES``, or when its last success is older than ``INTERVALS_STALE`` intervals, and ``ok`` otherwise. A + cycle still running within the loop's cycle budget does not age the loop: it is judged by when that cycle + started. Aging uses ``monotonic``, so a wall-clock step neither hides a stalled loop nor degrades a healthy one; + ``clock`` gives the aware UTC times that are shown. + """ + + def __init__(self, clock: Callable[[], datetime] = utc_now, *, monotonic: Callable[[], float] = time.monotonic): + self._clock = clock + self._monotonic = monotonic + self._records: dict[str, _Record] = {} + + def register(self, name: str, interval_seconds: float, *, enabled: bool, budget_seconds: float = 0.0) -> None: + """Declare a loop at startup; registering it again keeps what it already reported.""" + if name not in COMPONENTS: + raise ValueError("Unknown background loop") + interval = _seconds(interval_seconds, positive=True) + budget = _seconds(budget_seconds, positive=False) + if type(enabled) is not bool: + raise ValueError("Loop enablement must be a boolean") + record = self._records.get(name) + if record is None: + self._records[name] = _Record(interval=interval, budget=budget, enabled=enabled) + else: + record.interval, record.budget, record.enabled = interval, budget, enabled + + def running(self, name: str, value: bool) -> None: + self._update(name, lambda record: setattr(record, "running", value is True)) + + def started(self, name: str) -> None: + def update(record: _Record) -> None: + shown, at = self._now(), self._monotonic() + record.last_started_at, record.started = shown, at + + self._update(name, update) + + def succeeded(self, name: str, *, complete_until: datetime | None = None) -> None: + def update(record: _Record) -> None: + shown, at = self._now(), self._monotonic() + until = None if complete_until is None else _utc(complete_until) + record.last_ok_at, record.ok, record.complete_until = shown, at, until + record.started, record.consecutive_failures = None, 0 + record.outcomes = (record.outcomes << 1) & _RECENT_MASK + + self._update(name, update) + + def failed(self, name: str) -> None: + def update(record: _Record) -> None: + record.consecutive_failures = min(record.consecutive_failures + 1, _FAILURE_CEILING) + record.started = None + record.outcomes = ((record.outcomes << 1) | 1) & _RECENT_MASK + + self._update(name, update) + + def skipped(self, name: str) -> None: + """A cycle that did no work, such as one waiting for readiness: neither a success nor a failure.""" + self._update(name, lambda record: setattr(record, "started", None)) + + def snapshot(self, name: str) -> LoopSnapshot: + record = self._records[name] + recent = record.outcomes.bit_count() + if not record.enabled: + state: LoopState = "disabled" + elif not record.running: + state = "down" + elif ( + record.ok is None + or record.consecutive_failures >= FAILURES_DEGRADED + or recent >= FAILURES_DEGRADED + or self._behind(record, record.ok) + ): + state = "degraded" + else: + state = "ok" + return LoopSnapshot( + component=name, + state=state, + last_started_at=record.last_started_at, + last_ok_at=record.last_ok_at, + consecutive_failures=record.consecutive_failures, + complete_until=record.complete_until, + recent_failures=recent, + ) + + def snapshots(self) -> tuple[LoopSnapshot, ...]: + return tuple(self.snapshot(name) for name in COMPONENTS if name in self._records) + + def _behind(self, record: _Record, ok: float) -> bool: + now, limit = self._monotonic(), record.interval * INTERVALS_STALE + if record.started is not None and now - record.started <= record.budget: + # The cycle in flight is within its budget: the loop is on time if that cycle started on time. + return record.started - ok > limit + return now - ok > limit + + def _now(self) -> datetime: + return _utc(self._clock()) + + def _update(self, name: str, change: Callable[[_Record], None]) -> None: + # Unregistered names are ignored: an update never invents a component or fails the loop reporting it. + record = self._records.get(name) + if record is None: + return + try: + change(record) + except Exception as error: + logging.getLogger(__name__).warning("Loop health update skipped (%s)", type(error).__name__) + + +def register_background_loops(health: LoopHealth, settings: Settings) -> None: + """Every component, registered at startup so a loop that has not opened yet reads as down, not missing.""" + scheduled = settings.scheduler_enabled + for name in ("recovery_loop", "schedules"): + health.register( + name, settings.scheduler_poll_seconds, enabled=scheduled, budget_seconds=RECOVERY_BUDGET_SECONDS + ) + health.register("outbox_loop", TRAVERSAL_SECONDS, enabled=scheduled, budget_seconds=TRAVERSAL_BUDGET_SECONDS) + health.register("provider_loop", TRAVERSAL_SECONDS, enabled=scheduled, budget_seconds=PROVIDER_BUDGET_SECONDS) + # The broker consumer has its own switch; the scheduler setting does not stop it. + health.register( + "kafka_loop", + TRAVERSAL_SECONDS, + enabled=settings.broker.enabled and settings.kafka_consumer_enabled, + budget_seconds=TRAVERSAL_BUDGET_SECONDS, + ) + # Its cycles finish well within three of its one-minute intervals. + health.register("operations_loop", OPERATIONS_SECONDS, enabled=scheduled) + # Alert evaluation does not exist yet. + health.register("alert_evaluator", OPERATIONS_SECONDS, enabled=False) + + +def track(health: LoopHealth, task: "asyncio.Task[None]", *names: str) -> None: + """Report ``names`` running while ``task`` runs, and down as soon as it ends for any reason.""" + for name in names: + health.running(name, True) + + def ended(_: "asyncio.Task[None]") -> None: + for name in names: + health.running(name, False) + + task.add_done_callback(ended) diff --git a/src/firefly_weave/operations/outbox_loop.py b/src/firefly_weave/operations/outbox_loop.py index 7d1a9501..fb312c25 100644 --- a/src/firefly_weave/operations/outbox_loop.py +++ b/src/firefly_weave/operations/outbox_loop.py @@ -25,17 +25,23 @@ from firefly_weave.access.scheduler import next_scope from firefly_weave.operations.event_delivery import OutboxDispatcher, release, reserve +from firefly_weave.operations.loop_health import TRAVERSAL_SECONDS, LoopHealth, track from firefly_weave.persistence.migrations import check_schema from firefly_weave.persistence.uow import UnitOfWork from firefly_weave.settings import Settings class OutboxLoop: - def __init__(self, settings: Settings, uow: UnitOfWork, dispatcher: OutboxDispatcher) -> None: + def __init__( + self, settings: Settings, uow: UnitOfWork, dispatcher: OutboxDispatcher, *, health: LoopHealth | None = None + ) -> None: self.settings, self.uow, self.dispatcher = settings, uow, dispatcher + self.health = health if health is not None else LoopHealth() self.engine: AsyncEngine | None = None self.task: asyncio.Task[None] | None = None self.turns: set[asyncio.Task[None]] = set() + # A delivery turn runs in its own task; its failure fails the next traversal cycle. + self.turn_failed = False async def open(self) -> None: if not self.settings.scheduler_enabled: @@ -63,17 +69,26 @@ async def open(self) -> None: # A request can open this loop. An empty context keeps the request's execution # lease and identity, which end with its response, out of the long-lived task. self.task = asyncio.create_task(self.poll(), name="weave-outbox-traversal", context=Context()) + track(self.health, self.task, "outbox_loop") except BaseException: await self.close() raise - async def cycle(self) -> None: + async def cycle(self) -> bool | None: + """One bounded traversal turn. + + False when a delivery turn failed since the previous cycle. None when no turn could start, because the + platform is not ready or no delivery slot is free: the cycle is neither a success nor a failure, so the + outbox reads degraded once that lasts longer than three intervals. + """ + healthy, self.turn_failed = not self.turn_failed, False # A turn started now would be refused, so wait for readiness without moving a cursor. if not self.dispatcher.operational(): - return + return None if healthy else False count = reserve(1) if not count: - return + # Every slot is held by a delivery in flight, or was never given back. + return None if healthy else False owned = True try: assert self.engine is not None @@ -83,10 +98,10 @@ async def cycle(self) -> None: await connection.execute(text("SET LOCAL lock_timeout='1000ms'")) tenant = await connection.scalar(text("SELECT weave_outbox_tenants(1)")) if tenant is None: - return + return healthy authority = await next_scope(self.uow, tenant, outbox=True) if authority is None: - return + return healthy started = False @@ -97,6 +112,7 @@ async def turn() -> None: try: await self.dispatcher._dispatch(authority.scope, 1, None, AuditContext(), authority, reserved=count) except Exception: + self.turn_failed = True logging.getLogger(__name__).error("Outbox turn failed; durable lease recovery remains available") def finished(task: asyncio.Task[None]) -> None: @@ -114,14 +130,24 @@ def finished(task: asyncio.Task[None]) -> None: finally: if owned: release(count) + return healthy async def poll(self) -> None: while True: + self.health.started("outbox_loop") try: - await self.cycle() + healthy = await self.cycle() except Exception: logging.getLogger(__name__).error("Outbox traversal failed; retrying bounded turn") - await asyncio.sleep(1) + healthy = False + if healthy is None: + # No turn could start, so the cycle proves nothing and the last success keeps aging. + self.health.skipped("outbox_loop") + elif healthy: + self.health.succeeded("outbox_loop") + else: + self.health.failed("outbox_loop") + await asyncio.sleep(TRAVERSAL_SECONDS) async def close(self) -> None: if self.task is not None: @@ -129,6 +155,7 @@ async def close(self) -> None: with suppress(asyncio.CancelledError): await self.task self.task = None + self.health.running("outbox_loop", False) for task in tuple(self.turns): task.cancel() if self.turns: diff --git a/src/firefly_weave/operations/permissions.py b/src/firefly_weave/operations/permissions.py new file mode 100644 index 00000000..06f8ab1b --- /dev/null +++ b/src/firefly_weave/operations/permissions.py @@ -0,0 +1,66 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Deployment targets a reader may see, from both the token and the principal stored now.""" + +from uuid import UUID + +from firefly_weave.access.audit import AuditContext +from firefly_weave.access.authorization import AuthorizationService, contains +from firefly_weave.access.models import Principal +from firefly_weave.access.roles import ROLE_CAPABILITIES +from firefly_weave.contracts.access import Scope + + +def target_permissions( + authorization: AuthorizationService, + actor: Principal, + current: Principal, + scope: Scope, + capability: str, + context: AuditContext, +) -> tuple[UUID, ...] | None: + """`None` when both principals hold `capability` for every target; otherwise the sorted finite intersection. + + The result is empty when either principal holds it for no target. Each granted target is checked through the + authorization service, so callers never collect every target and filter afterwards. + """ + + def allowed(principal: Principal) -> set[UUID] | None: + if not principal.active or principal.kind == "worker": + return set() + candidates: set[UUID] = set() + for grant in principal.grants: + if capability not in ROLE_CAPABILITIES[grant.role] or not contains(grant.scope, scope): + continue + if not grant.resources: + authorization.require(principal, scope, capability, context=context) + return None + for resource in grant.resources: + try: + identifier = UUID(resource) + except ValueError: + continue + # Target grants name the canonical ID, as single target reads check it. + if str(identifier) != resource: + continue + authorization.require(principal, scope, capability, resource=str(identifier), context=context) + candidates.add(identifier) + return candidates + + left, right = allowed(actor), allowed(current) + result = right if left is None else left if right is None else left & right + return None if result is None else tuple(sorted(result)) diff --git a/src/firefly_weave/operations/run_views.py b/src/firefly_weave/operations/run_views.py index 45294aee..b5cb68e6 100644 --- a/src/firefly_weave/operations/run_views.py +++ b/src/firefly_weave/operations/run_views.py @@ -16,8 +16,7 @@ """Run summaries, run steps and run logs behind current `run.read` authority. -The wire contract is frozen in `contracts/run_views.py`. Until run facts exist, every authorized read answers -501 WV-UNAVAILABLE after its query and cursor were validated. +Summaries and timelines read bounded facts in the authorization transaction; logs remain unavailable. """ from uuid import UUID @@ -28,6 +27,7 @@ from firefly_weave.access.authorization import AuthorizationService from firefly_weave.access.models import Principal from firefly_weave.access.repository import load_principal +from firefly_weave.api.transport import encode_cursor_v2 from firefly_weave.contracts.access import Scope from firefly_weave.contracts.run_views import ( RunLogPage, @@ -39,7 +39,9 @@ ) from firefly_weave.contracts.values import JsonValue from firefly_weave.definitions.models import CatalogError -from firefly_weave.persistence.uow import UnitOfWork +from firefly_weave.operations.fact_positions import run_position, step_position +from firefly_weave.operations.fact_reads import FactReadRepository, step_of, summary_of +from firefly_weave.persistence.uow import Transaction, UnitOfWork type Position = tuple[JsonValue, str] | None @@ -53,21 +55,31 @@ class RunViewService: def __init__(self, uow: UnitOfWork, authorization: AuthorizationService) -> None: self.uow, self.authorization = uow, authorization - async def authorize(self, actor: Principal, scope: Scope, *, context: AuditContext) -> Principal: - """`run.read` on the token principal and again on the principal stored now.""" - if scope.environment_id is None: - raise CatalogError(422, "WV-SCOPE", "Runtime requires an environment") - self.authorization.require(actor, scope, "run.read", context=context) - async with self.uow.open(scope, mutation=False) as tx: - current = await load_principal(tx.session, actor.id) + async def _authorize_current( + self, actor: Principal, scope: Scope, tx: Transaction, *, context: AuditContext + ) -> Principal: + """`run.read` on the principal stored now, inside the read unit; callers run `precheck` first.""" + current = await load_principal(tx.session, actor.id) self.authorization.require(current, scope, "run.read", context=context) return current async def summaries( self, actor: Principal, scope: Scope, query: RunSummaryQuery, position: Position, *, context: AuditContext ) -> RunSummaryPage: - await self.authorize(actor, scope, context=context) - raise not_served("run_summaries.list") + typed = run_position(position) + self.precheck(actor, scope, context) + async with self.uow.open(scope, mutation=False) as tx: + await self._authorize_current(actor, scope, tx, context=context) + rows = await FactReadRepository(tx).summary_rows(query, typed) + visible = rows[: query.limit] + cursor = None + if len(rows) > query.limit: + last = visible[-1] + at = last["updated_at"] if query.order == "updated_desc" else last["started_at"] + cursor = encode_cursor_v2( + scope, query.cursor_collection(), at.isoformat() if at else None, str(last["run_id"]) + ) + return RunSummaryPage(items=[summary_of(row) for row in visible], next_cursor=cursor) async def steps( self, @@ -79,8 +91,30 @@ async def steps( *, context: AuditContext, ) -> StepFactPage: - await self.authorize(actor, scope, context=context) - raise not_served("runs.steps") + typed = step_position(position) + self.precheck(actor, scope, context) + async with self.uow.open(scope, mutation=False) as tx: + await self._authorize_current(actor, scope, tx, context=context) + repository = FactReadRepository(tx) + rows, complete = await repository.step_rows(run_id, query, typed) + visible = rows[: query.limit] + outputs = await repository.outputs(run_id, visible) if query.include == "output" else {} + cursor = None + if len(rows) > query.limit: + last = visible[-1] + at = last["scheduled_at"] + # The full instance key determines the step ID, so the longest valid key still fits the cursor. + cursor = encode_cursor_v2( + scope, + query.cursor_collection(run_id), + at.isoformat() if at else None, + last["instance_key"] or last["node_id"], + ) + return StepFactPage( + items=[step_of(row, output=outputs.get(row["instance_key"] or row["node_id"])) for row in visible], + next_cursor=cursor, + complete=complete, + ) async def logs( self, @@ -92,5 +126,13 @@ async def logs( *, context: AuditContext, ) -> RunLogPage: - await self.authorize(actor, scope, context=context) - raise not_served("runs.logs") + self.precheck(actor, scope, context) + async with self.uow.open(scope, mutation=False) as tx: + await self._authorize_current(actor, scope, tx, context=context) + raise not_served("runs.logs") + + def precheck(self, actor: Principal, scope: Scope, context: AuditContext) -> None: + """`run.read` on the token principal, before any read unit opens.""" + if scope.environment_id is None: + raise CatalogError(422, "WV-SCOPE", "Runtime requires an environment") + self.authorization.require(actor, scope, "run.read", context=context) diff --git a/src/firefly_weave/operations/summary.py b/src/firefly_weave/operations/summary.py new file mode 100644 index 00000000..90f77911 --- /dev/null +++ b/src/firefly_weave/operations/summary.py @@ -0,0 +1,672 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""The environment summary and the telemetry settings an operator may see. + +The summary is read in one transaction that also reloads the caller's grants. Beyond `status.read`, each section +and attention item needs its own capability on both the token and the principal stored now; a section the caller +may not read is forbidden before any of its statements run, and finding that out records nothing. Each section runs +in a savepoint with two seconds of database time in all: every statement runs under what is left of it, and a +section that runs out is unavailable while the others are still served. Run figures leave out test runs, and times +the platform only estimated are never counted as known; queue and worker figures measure shared capacity and count +every task. +""" + +import time +from collections.abc import Awaitable, Callable, Iterable +from dataclasses import dataclass +from datetime import UTC, datetime +from decimal import Decimal +from typing import Any, Literal, Protocol, cast +from urllib.parse import urlsplit +from uuid import UUID + +from sqlalchemy import Result, text +from sqlalchemy.exc import DBAPIError +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.sql import Executable + +from firefly_weave import __version__ +from firefly_weave.access.audit import AuditContext +from firefly_weave.access.authorization import AccessDenied, AuthorizationService +from firefly_weave.access.models import Principal +from firefly_weave.access.repository import load_principal +from firefly_weave.contracts.access import Scope +from firefly_weave.contracts.operations_summary import ( + ATTENTION_LIMIT, + COMPATIBILITY_ITEM, + ApiSection, + AttentionItem, + AttentionKind, + CompatibilitySummary, + ForbiddenSection, + HealthComponent, + HealthComponentName, + HealthState, + IncidentsSection, + LoadSection, + OperationsSummary, + OperationsTelemetry, + TargetsSection, + TelemetryEndpoint, + TelemetryMetricEndpoint, + UnavailableSection, + WorkersSection, +) +from firefly_weave.contracts.telemetry import TelemetryOptions +from firefly_weave.observability import TelemetryDrops +from firefly_weave.operations.compatibility import CompatibilityService +from firefly_weave.operations.fact_reads import available +from firefly_weave.operations.loop_health import LoopHealth, LoopSnapshot +from firefly_weave.operations.permissions import target_permissions +from firefly_weave.persistence.uow import Transaction, UnitOfWork +from firefly_weave.settings import Settings + +# Each section's statements share this budget; a stricter timeout already in force is kept. +SECTION_TIMEOUT_MS = 2000 +# Telemetry export is degraded while anything was dropped within this many metric export intervals. +TELEMETRY_INTERVALS_RECENT = 3 +# Query failures a section reports as unavailable: a cancelled or timed-out statement, a lock timeout, a +# serialization conflict, a deadlock and exhausted database resources. Every other error propagates. +_TRANSIENT = ("57014", "55P03", "40001", "40P01", "53") +# Priority of each attention kind; critical alerts (1) arrive with alert storage. +PRIORITY: dict[str, int] = { + "job": 2, + "approval": 3, + "worker": 4, + "incident": 5, + "runner": 6, + "compatibility": 7, + "run": 8, +} +_TITLE = 200 +_SCOPE = "{alias}tenant_id=:tenant AND {alias}project_id=:project AND {alias}environment_id=:environment" +_TARGETS = "(:all_targets OR {column}=ANY(cast(:target_ids AS uuid[])))" +_SAME_SCOPE = "{a}.tenant_id={b}.tenant_id AND {a}.project_id={b}.project_id AND {a}.environment_id={b}.environment_id" +NOW = "cast(:now AS timestamptz)" +# Plans the approve operation would accept now: unexpired, on a fresh observation of the target's current revision +# and the deployment's current revision, for an intent the enabled target allows, and neither approved nor applied. +_AWAITING_APPROVAL = ( + "SELECT p.id,(p.payload->>'expires_at')::timestamptz AS expires_at,p.payload->>'intent' AS intent," + "t.payload->>'name' AS target_name FROM deployment_plans p " + f"JOIN deployment_targets t ON {_SAME_SCOPE.format(a='t', b='p')} AND t.id=p.target_id " + f"JOIN deployment_deployments d ON {_SAME_SCOPE.format(a='d', b='p')} AND d.id=p.deployment_id " + f"JOIN deployment_observations o ON {_SAME_SCOPE.format(a='o', b='p')} " + "AND o.id=(p.payload->>'observation_id')::uuid " + f"WHERE {_SCOPE.format(alias='p.')} AND {_TARGETS.format(column='p.target_id')} " + f"AND (p.payload->>'expires_at')::timestamptz>{NOW} AND (o.payload->>'expires_at')::timestamptz>{NOW} " + "AND (t.payload->>'revision')::integer=(p.payload->>'target_revision')::integer " + "AND NOT coalesce((t.payload->>'disabled')::boolean,false) " + "AND t.payload->'capabilities' @> jsonb_build_array(p.payload->>'intent') " + "AND (d.payload->>'revision')::integer=(p.payload->>'deployment_revision')::integer " + "AND o.payload->>'digest'=p.payload->>'observation_digest' " + f"AND NOT EXISTS(SELECT 1 FROM deployment_approvals a WHERE {_SAME_SCOPE.format(a='a', b='p')} AND a.id=p.id) " + f"AND NOT EXISTS(SELECT 1 FROM deployment_jobs j WHERE {_SAME_SCOPE.format(a='j', b='p')} " + "AND j.payload->>'plan_id'=p.id::text)" +) +_LIVE_RUNNER = ( + f"SELECT 1 FROM deployment_runners r WHERE {_SAME_SCOPE.format(a='r', b='t')} AND r.target_id=t.id " + f"AND NOT coalesce((r.payload->>'revoked')::boolean,false) AND (r.payload->>'expires_at')::timestamptz>{NOW}" +) +_LIVE_LEASES = ( + "SELECT owner,count(*) AS leases FROM task_leases WHERE " + + _SCOPE.format(alias="") + + f" AND status='active' AND expires_at>{NOW} AND deadline>{NOW} GROUP BY owner" +) +_RECENT = f"w.last_seen_at>{NOW}-interval '60 seconds'" +_CURRENT = f"NOT w.revoked AND coalesce(w.last_seen_at,f.registered_at)>{NOW}-interval '1 hour'" +_WORKER_FACTS = f"LEFT JOIN worker_facts f ON {_SAME_SCOPE.format(a='f', b='w')} AND f.worker_id=w.id" +# Active test runs, exactly as the run usage counter counts runs: by status, so a finished run without a recorded end +# is not active. The partial index run_facts_active_test has this predicate. +_ACTIVE_TEST = "test AND (status IS NULL OR status NOT IN ('succeeded','failed','cancelled','timed_out'))" +_READY_COUNTS = ( + "SELECT count(*) AS ready,extract(epoch FROM (cast(:now AS timestamptz)-min(ready_since))) AS oldest " + f"FROM task_facts WHERE {_SCOPE.format(alias='')} AND status='ready' AND ready_since<=cast(:now AS timestamptz)" +) +_RUN_CLASSIFICATION = ( + "r.classification_state,r.classification_policy,r.pinned_ir_version,r.pinned_features," + f"EXISTS(SELECT 1 FROM run_policy_blocks b WHERE {_SAME_SCOPE.format(a='b', b='r')} AND b.run_id=r.run_id) " + "AS policy_blocked" +) + + +class NativeExecutors(Protocol): + def healthy(self) -> bool: ... + + +def telemetry_host(endpoint: str | None) -> str | None: + """Only the host name of a collector endpoint: never its credentials, port, path or query.""" + return urlsplit(endpoint).hostname if endpoint else None + + +def _transient(error: DBAPIError) -> bool: + state = getattr(error.orig, "sqlstate", None) + return isinstance(state, str) and state.startswith(_TRANSIENT) + + +class SectionTimedOut(Exception): + """A section used up its database time before its next statement.""" + + +class Budget: + """One section's database time, shared by its statements: each runs under what is left of it.""" + + def __init__(self, session: AsyncSession, limit_ms: int, monotonic: Callable[[], float]) -> None: + self.session, self.monotonic = session, monotonic + self.deadline = monotonic() + limit_ms / 1000 + + async def limit(self) -> None: + """Bound the next statement by what is left; nothing left means the section ran out.""" + remaining = int((self.deadline - self.monotonic()) * 1000) + if remaining <= 0: + raise SectionTimedOut() + await self.session.execute( + text("SELECT set_config('statement_timeout',:value,true)"), {"value": f"{remaining}ms"} + ) + + async def execute(self, statement: Executable, values: dict[str, Any] | None = None) -> Result[Any]: + await self.limit() + return await self.session.execute(statement, values) + + +async def read_section[T]( + tx: Transaction, read: Callable[[Budget], Awaitable[T]], *, monotonic: Callable[[], float] = time.monotonic +) -> T | UnavailableSection: + """Run one section's statements in a savepoint, sharing one section budget. + + The budget is two seconds, or a stricter timeout already in force. A section that runs out of it, or meets a + transient query failure, rolls the savepoint back, which restores the earlier local timeout, and is unavailable. + A lost connection, cancellation, an authorization failure and any other error propagate, so a broken transaction + never answers later sections. + """ + session = tx.session + row = ( + await session.execute( + text( + "SELECT current_setting('statement_timeout') AS prior,setting::bigint AS prior_ms FROM pg_settings " + "WHERE name='statement_timeout'" + ) + ) + ).one() + bound = min(row.prior_ms, SECTION_TIMEOUT_MS) if row.prior_ms else SECTION_TIMEOUT_MS + try: + async with session.begin_nested(): + # Every statement of the section is bounded by the whole budget, and lowered to what is left of it. + await session.execute(text("SELECT set_config('statement_timeout',:value,true)"), {"value": f"{bound}ms"}) + value = await read(Budget(session, bound, monotonic)) + await session.execute(text("SELECT set_config('statement_timeout',:value,true)"), {"value": row.prior}) + return value + except SectionTimedOut: + return UnavailableSection(state="unavailable") + except DBAPIError as error: + if error.connection_invalidated or not _transient(error): + raise + return UnavailableSection(state="unavailable") + + +def _ready(row: Any) -> tuple[int, float | None]: + return int(row["ready"]), _seconds(row["oldest"]) + + +async def read_ready_counts(tx: Transaction, observed_at: datetime) -> tuple[int, float | None]: + """Ready tasks that are due at `observed_at`, and how long the oldest of them has waited. + + A retry whose next attempt is still in the future is stored as ready from that instant, so it joins the count + exactly when it becomes due; nothing is written to make it so. Claims stay the authority on what runs. + """ + result = await tx.session.execute(text(_READY_COUNTS), {**_bound(tx), "now": observed_at}) + return _ready(result.mappings().one()) + + +def _bound(tx: Transaction) -> dict[str, Any]: + return { + "tenant": tx.scope.tenant_id, + "project": tx.scope.project_id, + "environment": tx.scope.environment_id, + } + + +def _seconds(value: Decimal | float | None) -> float | None: + return None if value is None else max(0.0, float(value)) + + +def _title(value: str) -> str: + return value if len(value) <= _TITLE else value[: _TITLE - 1] + "…" + + +def _plural(count: int, noun: str) -> str: + return f"{count} {noun}" + ("" if count == 1 else "s") + + +def native_state(settings: Settings, dispatcher: NativeExecutors | None) -> HealthState: + """Native executors as readiness reports them: off when none are configured, down while not open.""" + if not settings.native_executors: + return "disabled" + if dispatcher is None: + return "down" + return "ok" if dispatcher.healthy() else "degraded" + + +def telemetry_state(options: TelemetryOptions, drops: TelemetryDrops) -> HealthState: + """Off unless export is on; degraded while this process dropped telemetry within the last three metric intervals. + + Configuration alone proves nothing: export is healthy only while nothing is being dropped. + """ + if not options.enabled: + return "disabled" + recent = TELEMETRY_INTERVALS_RECENT * options.metrics_interval_seconds + return "degraded" if drops.dropped_within(recent) else "ok" + + +def health_components( + latency: float | UnavailableSection, + snapshots: Iterable[LoopSnapshot], + *, + native: HealthState, + telemetry: HealthState, +) -> list[HealthComponent]: + """The database, each registered loop, native executors and telemetry export; states and counts, never messages.""" + database = ( + HealthComponent(component="database", state="ok", latency_ms=latency) + if isinstance(latency, float) + else HealthComponent(component="database", state="degraded") + ) + loops = [ + HealthComponent( + component=cast(HealthComponentName, snapshot.component), + state=snapshot.state, + last_started_at=snapshot.last_started_at, + last_ok_at=snapshot.last_ok_at, + consecutive_failures=snapshot.consecutive_failures, + recent_failures=snapshot.recent_failures, + complete_until=snapshot.complete_until, + ) + for snapshot in snapshots + ] + return [ + database, + *loops, + HealthComponent(component="native_executors", state=native), + HealthComponent(component="telemetry", state=telemetry), + ] + + +@dataclass(frozen=True) +class Candidate: + """An attention item before ranking: `at` orders items of one kind, and is the expiry for approvals.""" + + kind: AttentionKind + id: UUID | Literal["compatibility"] + title: str + at: datetime | None = None + expires_at: datetime | None = None + + +def attention_order(candidates: Iterable[Candidate]) -> list[AttentionItem]: + """At most 20 items: by priority, then soonest time (unknown last), then ID, deterministically.""" + + def key(item: Candidate) -> tuple[int, bool, datetime, int]: + identifier = item.id.int if isinstance(item.id, UUID) else 0 + return PRIORITY[item.kind], item.at is None, item.at or datetime.min.replace(tzinfo=UTC), identifier + + return [ + AttentionItem(kind=item.kind, id=item.id, title=_title(item.title), expires_at=item.expires_at) + for item in sorted(candidates, key=key)[:ATTENTION_LIMIT] + ] + + +@dataclass(frozen=True) +class _Reach: + """What the caller may read beyond `status.read`, from both the token and the principal stored now. + + Target sets are None for every target and empty for none. + """ + + incidents: bool + runs: bool + targets: tuple[UUID, ...] | None + approvals: tuple[UUID, ...] | None + + +def _intersect(left: tuple[UUID, ...] | None, right: tuple[UUID, ...] | None) -> tuple[UUID, ...] | None: + if left is None: + return right + if right is None: + return left + return tuple(sorted(set(left) & set(right))) + + +class _Reader: + """One section's statements, each under what is left of the section's budget.""" + + def __init__(self, tx: Transaction, observed_at: datetime, budget: Budget) -> None: + self.tx, self.budget = tx, budget + self.bound = {**_bound(tx), "now": observed_at} + + async def rows(self, sql: str, **values: Any) -> list[dict[str, Any]]: + result = await self.budget.execute(text(sql), {**self.bound, **values}) + return [dict(row) for row in result.mappings()] + + async def row(self, sql: str, **values: Any) -> dict[str, Any]: + return (await self.rows(sql, **values))[0] + + +def _targets(targets: tuple[UUID, ...] | None) -> dict[str, Any]: + return {"all_targets": targets is None, "target_ids": list(targets or ())} + + +class OperationsSummaryService: + """Lifespan-owned: one instance ID, one health registry and the exporters' own drop counter per process.""" + + def __init__( + self, + uow: UnitOfWork, + authorization: AuthorizationService, + settings: Settings, + health: LoopHealth, + compatibility: CompatibilityService, + *, + drops: TelemetryDrops, + instance: UUID, + started_monotonic: float, + native: Callable[[], NativeExecutors | None] = lambda: None, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self.uow, self.authorization, self.settings = uow, authorization, settings + self.health, self.compatibility = health, compatibility + self.drops, self.instance, self.started_monotonic = drops, instance, started_monotonic + self.native, self.monotonic = native, monotonic + + async def read(self, actor: Principal, scope: Scope, *, context: AuditContext) -> OperationsSummary: + if scope.environment_id is None: + raise AccessDenied() + self.authorization.require(actor, scope, "status.read", context=context) + async with self.uow.open(scope, mutation=False) as tx: + current = await load_principal(tx.session, actor.id) + self.authorization.require(current, scope, "status.read", context=context) + reach = self._reach(actor, current, scope, context) + observed_at = await tx.session.scalar(text("SELECT clock_timestamp()")) + assert isinstance(observed_at, datetime) + + async def section[T](read: Callable[[_Reader], Awaitable[T]]) -> T | UnavailableSection: + return await read_section( + tx, lambda budget: read(_Reader(tx, observed_at, budget)), monotonic=self.monotonic + ) + + latency = await read_section(tx, self._latency, monotonic=self.monotonic) + load = await section(self._load) + workers = await section(self._workers) + targets: TargetsSection | ForbiddenSection | UnavailableSection = ( + ForbiddenSection(state="forbidden") + if reach.targets == () + else await section(lambda reader: self._targets(reader, reach.targets)) + ) + incidents: IncidentsSection | ForbiddenSection | UnavailableSection = ( + await section(self._incidents) if reach.incidents else ForbiddenSection(state="forbidden") + ) + attention = await section(lambda reader: self._attention(reader, reach)) + return OperationsSummary( + observed_at=observed_at, + api=self._api(scope), + health=self._health(latency), + load=load, + workers=workers, + targets=targets, + incidents=incidents, + attention=attention, + ) + + async def telemetry(self, actor: Principal, scope: Scope, *, context: AuditContext) -> OperationsTelemetry: + """Whether telemetry is exported and to which collector hosts; never headers, credentials or full endpoints.""" + if scope.project_id is None: + raise AccessDenied() + self.authorization.require(actor, scope, "status.read", context=context) + async with self.uow.open(scope, mutation=False) as tx: + current = await load_principal(tx.session, actor.id) + self.authorization.require(current, scope, "status.read", context=context) + options = self.settings.telemetry + return OperationsTelemetry( + enabled=options.enabled, + traces=TelemetryEndpoint( + configured=options.traces_endpoint is not None, host=telemetry_host(options.traces_endpoint) + ), + metrics=TelemetryMetricEndpoint( + configured=options.metrics_endpoint is not None, + host=telemetry_host(options.metrics_endpoint), + interval_seconds=options.metrics_interval_seconds if options.metrics_endpoint else None, + ), + dropped=self.drops.count, + ) + + def _reach(self, actor: Principal, current: Principal, scope: Scope, context: AuditContext) -> _Reach: + """Which sections to fill. A capability the caller lacks is not an attempt to use it, so it records nothing; + reading a section the caller may read is recorded as allowed, as target-restricted reads are. + """ + + def permitted(capability: str) -> bool: + principals = (actor, current) + if not all(self.authorization.allows(principal, scope, capability) for principal in principals): + return False + for principal in principals: + self.authorization.require(principal, scope, capability, context=context) + return True + + readable = target_permissions(self.authorization, actor, current, scope, "deployment.read", context) + approvable = target_permissions(self.authorization, actor, current, scope, "deployment.approve", context) + return _Reach( + incidents=permitted("incident.read"), + runs=permitted("run.read"), + targets=readable, + approvals=_intersect(readable, approvable), + ) + + def _api(self, scope: Scope) -> ApiSection: + report = self.compatibility.report + # Only this project's findings are counted, as the compatibility report shows them. + findings = sum( + 1 + for finding in report.findings + if finding.scope is None + or (finding.scope.tenant_id == scope.tenant_id and finding.scope.project_id == scope.project_id) + ) + return ApiSection( + version=__version__, + build=self.settings.native_image_digest, + instance=self.instance, + uptime_seconds=max(0.0, self.monotonic() - self.started_monotonic), + compatibility=CompatibilitySummary( + mode="ready" if self.compatibility.ready else "restricted", findings=findings + ), + ) + + def _health(self, latency: float | UnavailableSection) -> list[HealthComponent]: + return health_components( + latency, + self.health.snapshots(), + native=native_state(self.settings, self.native()), + telemetry=telemetry_state(self.settings.telemetry, self.drops), + ) + + async def _latency(self, budget: Budget) -> float: + await budget.limit() + started = self.monotonic() + await budget.session.execute(text("SELECT 1")) + return round(max(0.0, self.monotonic() - started) * 1000, 3) + + async def _load(self, reader: _Reader) -> LoadSection: + usage = { + row["metric"]: int(row["value"]) + for row in await reader.rows( + "SELECT metric,value FROM operation_usage WHERE " + + _SCOPE.format(alias="") + + " AND metric IN ('runs_active','waits_active','outbox_pending','provider_pending','sources_enabled')" + ) + } + tests = await reader.row( + f"SELECT count(*) AS active FROM run_facts WHERE {_SCOPE.format(alias='')} AND {_ACTIVE_TEST}" + ) + ready, oldest_ready = _ready(await reader.row(_READY_COUNTS)) + leased = await reader.row( + "SELECT count(DISTINCT l.task_id) AS leased FROM task_leases l JOIN task_intents i " + f"ON {_SAME_SCOPE.format(a='i', b='l')} AND i.id=l.task_id WHERE {_SCOPE.format(alias='l.')} " + f"AND l.status='active' AND l.expires_at>{NOW} AND l.deadline>{NOW} AND i.status='leased'" + ) + overdue = await reader.row( + f"SELECT count(*) AS overdue,extract(epoch FROM ({NOW}-min(deadline))) AS oldest FROM run_deadlines " + f"WHERE {_SCOPE.format(alias='')} AND NOT consumed AND deadline<={NOW}" + ) + outbox = await reader.row( + f"SELECT extract(epoch FROM ({NOW}-min(next_at))) AS oldest FROM event_deliveries " + f"WHERE {_SCOPE.format(alias='')} AND status IN ('pending','retry') AND next_at<={NOW}" + ) + return LoadSection( + runs_active=max(0, usage.get("runs_active", 0) - int(tests["active"])), + tasks_ready=ready, + tasks_leased=int(leased["leased"]), + oldest_ready_seconds=oldest_ready, + waits_active=usage.get("waits_active", 0), + waits_overdue=int(overdue["overdue"]), + oldest_overdue_seconds=_seconds(overdue["oldest"]), + outbox_pending=usage.get("outbox_pending", 0), + outbox_oldest_seconds=_seconds(outbox["oldest"]), + provider_pending=usage.get("provider_pending", 0), + sources_enabled=usage.get("sources_enabled", 0), + ) + + async def _workers(self, reader: _Reader) -> WorkersSection: + """One grouped statement over the environment's workers, counted as the Workers page lists current ones.""" + row = await reader.row( + f"WITH live AS ({_LIVE_LEASES}) SELECT " + f"count(*) FILTER (WHERE {_CURRENT} AND NOT w.draining AND {_RECENT}) AS online," + f"count(*) FILTER (WHERE {_CURRENT} AND NOT w.draining AND NOT coalesce({_RECENT},false)) AS offline," + f"count(*) FILTER (WHERE {_CURRENT} AND w.draining) AS draining," + f"coalesce(sum((w.payload->>'capacity')::integer) FILTER (WHERE {_CURRENT} AND NOT w.draining " + f"AND {_RECENT}),0) AS capacity,coalesce(sum(a.leases),0) AS active_leases " + f"FROM worker_instances w {_WORKER_FACTS} LEFT JOIN live a ON a.owner=w.id " + f"WHERE {_SCOPE.format(alias='w.')}" + ) + return WorkersSection(**{key: int(value) for key, value in row.items()}) + + async def _targets(self, reader: _Reader, targets: tuple[UUID, ...] | None) -> TargetsSection: + """Each readable target counted once, however many runners, jobs or plans it has.""" + row = await reader.row( + "SELECT count(*) AS total," + f"count(*) FILTER (WHERE EXISTS({_LIVE_RUNNER})) AS runner_online," + "count(*) FILTER (WHERE EXISTS(SELECT 1 FROM deployment_jobs j WHERE " + f"{_SAME_SCOPE.format(a='j', b='t')} AND j.target_id=t.id " + "AND j.state IN ('queued','claimed','running','verifying'))) AS busy," + "count(*) FILTER (WHERE EXISTS(SELECT 1 FROM deployment_jobs j WHERE " + f"{_SAME_SCOPE.format(a='j', b='t')} AND j.target_id=t.id " + "AND j.state='reconciliation_required')) AS reconciliation_required " + f"FROM deployment_targets t WHERE {_SCOPE.format(alias='t.')} AND {_TARGETS.format(column='t.id')}", + **_targets(targets), + ) + plans = await reader.row(f"SELECT count(*) AS awaiting FROM ({_AWAITING_APPROVAL}) p", **_targets(targets)) + return TargetsSection( + **{key: int(value) for key, value in row.items()}, awaiting_approval=int(plans["awaiting"]) + ) + + async def _incidents(self, reader: _Reader) -> IncidentsSection: + """Incidents of test runs are left out; an incident without facts predates test runs and counts. + + Only known opening times count as opened in the last 24 hours: one the upgrade estimated is never counted. + """ + active = await reader.row( + "SELECT count(*) AS active FROM incidents i LEFT JOIN incident_facts f ON " + f"{_SAME_SCOPE.format(a='f', b='i')} AND f.incident_id=i.id " + f"WHERE {_SCOPE.format(alias='i.')} AND i.status='active' AND NOT coalesce(f.test,false)" + ) + opened = await reader.row( + f"SELECT count(*) AS opened FROM incident_facts WHERE {_SCOPE.format(alias='')} AND NOT test " + f"AND NOT opened_at_estimated AND opened_at>{NOW}-interval '24 hours'" + ) + return IncidentsSection(active=int(active["active"]), opened_24h=int(opened["opened"])) + + async def _attention(self, reader: _Reader, reach: _Reach) -> list[AttentionItem]: + """Up to 20 items per kind the caller may read, ranked together; alerts join once they are stored.""" + candidates: list[Candidate] = [] + limit = ATTENTION_LIMIT + if reach.targets != (): + for row in await reader.rows( + "SELECT j.id,t.payload->>'name' AS target_name," + "coalesce((j.payload->>'reconciliation_started_at')::timestamptz,j.created_at) AS at " + f"FROM deployment_jobs j JOIN deployment_targets t ON {_SAME_SCOPE.format(a='t', b='j')} " + f"AND t.id=j.target_id WHERE {_SCOPE.format(alias='j.')} AND {_TARGETS.format(column='j.target_id')} " + "AND j.state='reconciliation_required' ORDER BY at,j.id LIMIT :limit", + limit=limit, + **_targets(reach.targets), + ): + title = f"Deployment job on {row['target_name']} needs reconciliation" + candidates.append(Candidate("job", row["id"], title, row["at"])) + if reach.approvals != (): + for row in await reader.rows( + f"SELECT p.* FROM ({_AWAITING_APPROVAL}) p ORDER BY p.expires_at,p.id LIMIT :limit", + limit=limit, + **_targets(reach.approvals), + ): + intent = str(row["intent"]).replace("_", " ") + title = f"Approve the plan to {intent} on {row['target_name']}" + candidates.append(Candidate("approval", row["id"], title, row["expires_at"], row["expires_at"])) + for row in await reader.rows( + f"WITH live AS ({_LIVE_LEASES}) SELECT w.id,a.leases,coalesce(w.last_seen_at,f.registered_at) AS at " + f"FROM worker_instances w JOIN live a ON a.owner=w.id {_WORKER_FACTS} " + f"WHERE {_SCOPE.format(alias='w.')} AND NOT w.revoked AND NOT coalesce({_RECENT},false) " + "ORDER BY at ASC NULLS LAST,w.id LIMIT :limit", + limit=limit, + ): + leases = int(row["leases"]) + title = f"Offline worker still holds {_plural(leases, 'active lease')}" + candidates.append(Candidate("worker", row["id"], title, row["at"])) + if reach.incidents: + for row in await reader.rows( + f"SELECT i.id,f.opened_at,f.workflow_name,{_RUN_CLASSIFICATION} FROM incidents i " + f"LEFT JOIN incident_facts f ON {_SAME_SCOPE.format(a='f', b='i')} AND f.incident_id=i.id " + f"LEFT JOIN run_facts r ON {_SAME_SCOPE.format(a='r', b='i')} AND r.run_id=i.run_id " + f"WHERE {_SCOPE.format(alias='i.')} AND i.status='active' AND NOT coalesce(f.test,false) " + "ORDER BY f.opened_at ASC NULLS LAST,i.id LIMIT :limit", + limit=limit, + ): + named = row["workflow_name"] is not None and available(row) + title = f"Active incident in {row['workflow_name']}" if named else "Active incident" + candidates.append(Candidate("incident", row["id"], title, row["opened_at"])) + if reach.targets != (): + for row in await reader.rows( + "SELECT * FROM (SELECT DISTINCT ON (r.target_id) r.id,t.payload->>'name' AS target_name," + "(r.payload->>'expires_at')::timestamptz AS at FROM deployment_runners r " + f"JOIN deployment_targets t ON {_SAME_SCOPE.format(a='t', b='r')} AND t.id=r.target_id " + f"WHERE {_SCOPE.format(alias='r.')} AND {_TARGETS.format(column='r.target_id')} " + "AND NOT coalesce((t.payload->>'disabled')::boolean,false) " + "AND NOT coalesce((r.payload->>'revoked')::boolean,false) " + f"AND NOT EXISTS({_LIVE_RUNNER}) " + "ORDER BY r.target_id,(r.payload->>'expires_at')::timestamptz DESC,r.id DESC) offline " + "ORDER BY at,id LIMIT :limit", + limit=limit, + **_targets(reach.targets), + ): + title = f"No runner is online for {row['target_name']}" + candidates.append(Candidate("runner", row["id"], title, row["at"])) + if not self.compatibility.ready: + candidates.append(Candidate("compatibility", COMPATIBILITY_ITEM, "Execution compatibility is restricted")) + if reach.runs: + for row in await reader.rows( + f"SELECT r.run_id,r.ended_at,r.workflow_name,{_RUN_CLASSIFICATION} FROM run_facts r " + f"WHERE {_SCOPE.format(alias='r.')} AND r.status='failed' AND NOT r.test " + f"AND r.ended_at>{NOW}-interval '1 hour' ORDER BY r.ended_at,r.run_id LIMIT :limit", + limit=limit, + ): + title = f"A run of {row['workflow_name']} failed" if available(row) else "A run failed" + candidates.append(Candidate("run", row["run_id"], title, row["ended_at"])) + return attention_order(candidates) diff --git a/src/firefly_weave/operations/telemetry.py b/src/firefly_weave/operations/telemetry.py index 5f192d54..8134d431 100644 --- a/src/firefly_weave/operations/telemetry.py +++ b/src/firefly_weave/operations/telemetry.py @@ -60,7 +60,7 @@ "other", } ) -STATUSES = frozenset({"ok", "rejected", "failed", "cancelled", "blocked", "pending", "unknown", "other"}) +STATUSES = frozenset({"ok", "rejected", "failed", "cancelled", "blocked", "pending", "unknown", "unavailable", "other"}) ERRORS = frozenset( { "none", diff --git a/src/firefly_weave/operations/transport.py b/src/firefly_weave/operations/transport.py index bef3d4e0..3c3c202f 100644 --- a/src/firefly_weave/operations/transport.py +++ b/src/firefly_weave/operations/transport.py @@ -231,12 +231,22 @@ async def metadata_send(message: Message) -> None: None, ) status = 500 + unavailable = False + problem = bytearray() started = time.monotonic() async def observed_send(message: Message) -> None: - nonlocal status + nonlocal status, unavailable if message["type"] == "http.response.start": status = message["status"] + elif message["type"] == "http.response.body" and status == 501: + problem.extend(message.get("body", b"")[: max(0, 8193 - len(problem))]) + if not message.get("more_body") and len(problem) <= 8192: + try: + decoded = json.loads(problem) + unavailable = isinstance(decoded, dict) and decoded.get("code") == "WV-UNAVAILABLE" + except (ValueError, RecursionError): + pass await send(message) try: @@ -246,7 +256,13 @@ async def observed_send(message: Message) -> None: telemetry.record( "request", operation=operation, - status="ok" if status < 400 else "rejected" if status < 500 else "failed", + status="unavailable" + if unavailable + else "ok" + if status < 400 + else "rejected" + if status < 500 + else "failed", duration=time.monotonic() - started, ) diff --git a/src/firefly_weave/persistence/migrations.py b/src/firefly_weave/persistence/migrations.py index 604f1b91..abb0f181 100644 --- a/src/firefly_weave/persistence/migrations.py +++ b/src/firefly_weave/persistence/migrations.py @@ -29,7 +29,7 @@ from firefly_weave.settings import Settings, operations_from_env -SCHEMA_VERSION = "0031_operations_key_collation" +SCHEMA_VERSION = "0032_operations_facts" async def check_schema(connection: AsyncConnection) -> None: diff --git a/src/firefly_weave/providers/dispatcher.py b/src/firefly_weave/providers/dispatcher.py index 6421ba6d..524f5350 100644 --- a/src/firefly_weave/providers/dispatcher.py +++ b/src/firefly_weave/providers/dispatcher.py @@ -33,6 +33,7 @@ from firefly_weave.contracts.providers import ProviderReceipt from firefly_weave.contracts.runtime import StartRunRequest from firefly_weave.definitions.models import CatalogError, capacity_rejected +from firefly_weave.operations.facts import RunStartFacts from firefly_weave.providers.repository import ProviderRepository from firefly_weave.providers.service import ProviderIngressService from firefly_weave.runtime.repository import SCOPE @@ -71,6 +72,7 @@ async def dispatch_one(self, scope: Scope, receipt_id: UUID) -> ProviderReceipt: key, context=AuditContext(), tx=tx, + start_facts=RunStartFacts(origin="provider"), ) changes.update(state="dispatched", run_id=run.id, reason=None) else: diff --git a/src/firefly_weave/providers/loop.py b/src/firefly_weave/providers/loop.py index 0a8e8f2c..4b9c4599 100644 --- a/src/firefly_weave/providers/loop.py +++ b/src/firefly_weave/providers/loop.py @@ -26,6 +26,7 @@ from firefly_weave.access.scheduler import next_scope from firefly_weave.email.source import EmailSourceService from firefly_weave.operations.execution import ReservedSlots +from firefly_weave.operations.loop_health import TRAVERSAL_SECONDS, LoopHealth, track from firefly_weave.persistence.migrations import check_schema from firefly_weave.persistence.uow import UnitOfWork from firefly_weave.providers.dispatcher import ProviderDispatcher @@ -43,9 +44,12 @@ def __init__( uow: UnitOfWork, dispatcher: ProviderDispatcher, email_sources: EmailSourceService | None = None, + *, + health: LoopHealth | None = None, ) -> None: self.settings, self.uow, self.dispatcher = settings, uow, dispatcher self.email_sources = email_sources + self.health = health if health is not None else LoopHealth() self.engine: AsyncEngine | None = None self.task: asyncio.Task[None] | None = None @@ -74,11 +78,13 @@ async def open(self) -> None: # A request can open this loop. An empty context keeps the request's execution # lease and identity, which end with its response, out of the long-lived task. self.task = asyncio.create_task(self.poll(), name="weave-provider-inbox", context=Context()) + track(self.health, self.task, "provider_loop") except BaseException: await self.close() raise - async def cycle(self) -> None: + async def cycle(self) -> bool: + """One bounded provider and email turn; a failure raises, so a returned cycle completed.""" assert self.engine is not None async with PROVIDER_EXECUTION.execution(): async with asyncio.timeout(5): @@ -87,20 +93,27 @@ async def cycle(self) -> None: await connection.execute(text("SET LOCAL lock_timeout='1000ms'")) tenant = await connection.scalar(text("SELECT weave_provider_tenants(1)")) if tenant is None: - return + return True authority = await next_scope(self.uow, tenant, provider=True) if authority is not None: await self.dispatcher.scan(authority, 10) if self.email_sources is not None: await self.email_sources.scan(authority, 1) + return True async def poll(self) -> None: while True: + self.health.started("provider_loop") try: - await self.cycle() + healthy = await self.cycle() except Exception: logging.getLogger(__name__).error("Provider traversal failed; retained intents remain retryable") - await asyncio.sleep(1) + healthy = False + if healthy: + self.health.succeeded("provider_loop") + else: + self.health.failed("provider_loop") + await asyncio.sleep(TRAVERSAL_SECONDS) async def close(self) -> None: if self.task is not None: @@ -108,6 +121,7 @@ async def close(self) -> None: with suppress(asyncio.CancelledError): await self.task self.task = None + self.health.running("provider_loop", False) if self.engine is not None: await self.engine.dispose() self.engine = None diff --git a/src/firefly_weave/runtime/admission.py b/src/firefly_weave/runtime/admission.py index 2981cedb..0234f571 100644 --- a/src/firefly_weave/runtime/admission.py +++ b/src/firefly_weave/runtime/admission.py @@ -30,22 +30,23 @@ type Admission = Literal["available", "unavailable", "unsupported"] -def admission(row: dict[str, Any]) -> Admission: +def admission(row: dict[str, Any], *, imported: Any = None) -> Admission: """How this platform may treat a retained run. ``unsupported``: the run pins an IR version or language feature this platform does not run, for example after a rollback. It is transient: scanners skip the run without recording anything, and an upgrade runs it again. ``unavailable``: legacy or malformed evidence; scanners record a permanent policy block. + ``imported`` is the row's artifact already imported by the caller, which is then not imported again. """ - return _admit(row)[0] + return _admit(row, imported)[0] -def _admit(row: dict[str, Any]) -> tuple[Admission, tuple[str, ...]]: +def _admit(row: dict[str, Any], imported: Any = None) -> tuple[Admission, tuple[str, ...]]: """``admission``, with the features this platform does not list for an ``unsupported`` run.""" if row["state"].get("unavailable"): return "unavailable", () try: - artifact = import_artifact(row["artifact"]) + artifact = imported if imported is not None else import_artifact(row["artifact"]) if row["state"].get("admission_policy") == POLICY: return "available", () return ("unavailable" if has_markers(artifact.executable) else "available"), () diff --git a/src/firefly_weave/runtime/recovery.py b/src/firefly_weave/runtime/recovery.py index 00699bce..6d49313e 100644 --- a/src/firefly_weave/runtime/recovery.py +++ b/src/firefly_weave/runtime/recovery.py @@ -33,6 +33,7 @@ from firefly_weave.contracts.operations import TaskTiming from firefly_weave.contracts.runtime import RecoveryReport from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.facts import task_fact_update from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.deadlines import DeadlineService from firefly_weave.runtime.kernel import transition_async as transition @@ -139,13 +140,15 @@ async def _recover(self, tx: Transaction, kind: str) -> RecoveryReport: tx.session.info["weave_runtime_candidate"] = row["id"] result = await transition(view.state, event, import_artifact(row["artifact"])) await repository.persist(view.model_copy(update={"state": result.state}), event, result, event_hash(event)) - await repository.task_status(identifier, "incident") + await repository.task_status(identifier, "incident", at=event.timestamp) incidents += 1 for identifier in await self.tasks.recovery_candidates(tx, limit) if kind == "retry" else []: locked = await self.runtime.lock_task(tx, identifier, skip_locked=True) if locked is None: continue row, task = locked + if task["status"] == "handled": + continue if await self.runtime.observe_unavailable(tx, row): continue attempt = await self.tasks.recovery_attempt(tx, identifier) @@ -231,7 +234,10 @@ async def _recover(self, tx: Transaction, kind: str) -> RecoveryReport: if result.state.status in TERMINAL: continue await repository.execute( - f"UPDATE task_intents SET status=:status,next_attempt_at=:next WHERE {SCOPE} AND id=:id", + task_fact_update( + f"UPDATE task_intents SET status=:status,next_attempt_at=:next WHERE {SCOPE} AND id=:id" + ), + at=now, id=identifier, status="ready" if allowed else "incident", next=next_attempt if allowed else None, diff --git a/src/firefly_weave/runtime/repository.py b/src/firefly_weave/runtime/repository.py index a6657d38..78404fae 100644 --- a/src/firefly_weave/runtime/repository.py +++ b/src/firefly_weave/runtime/repository.py @@ -28,6 +28,7 @@ from firefly_weave.contracts.operations import TaskTiming from firefly_weave.contracts.runtime import RunView, StartRunRequest, UnavailableRunAcknowledgment from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.facts import FactRepository, RunStartFacts, author_instance, task_fact_update from firefly_weave.operations.outbox import OutboxService from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.models import ( @@ -139,7 +140,18 @@ async def require_work(self, identifier: UUID, state: dict[str, Any]) -> None: self.tx.session.info["weave_runtime_candidate"] = identifier await execute_pure(partial(logical_size, state, STATE_BYTES)) - async def insert(self, view: RunView, artifact: dict[str, Any], request: StartRunRequest, principal: UUID) -> None: + async def insert( + self, + view: RunView, + artifact: dict[str, Any], + request: StartRunRequest, + principal: UUID, + *, + start_facts: RunStartFacts, + at: datetime, + imported: Any = None, + ir: Any = None, + ) -> None: await self.execute( "INSERT INTO runs VALUES(:id,:tenant,:project,:environment,:activation_id,:principal," "cast(:artifact AS jsonb),cast(:activation AS jsonb),cast(:request AS jsonb),cast(:state AS jsonb))", @@ -152,6 +164,8 @@ async def insert(self, view: RunView, artifact: dict[str, Any], request: StartRu state=view.state.model_dump_json(), ) + await FactRepository(self.tx).insert_run(view, artifact, request, start_facts, at, imported=imported, ir=ir) + async def event(self, run_id: UUID, event_id: UUID) -> dict[str, Any] | None: row = ( ( @@ -210,7 +224,7 @@ async def persist( state=result.state.model_dump_json(), run=view.id, ) - await self.project_incidents(view) + await self.project_incidents(view, at=event.timestamp) for change in result.steps: await self.execute( "INSERT INTO step_instances " @@ -235,13 +249,19 @@ async def persist( task_version=pin.task_version, connector_target=pin.model_dump(mode="json"), ) + identifier = uuid4() + values = FactRepository(self.tx).task_values( + identifier, view.id, command, f"{payload['task_type']}@{payload['task_version']}", event.timestamp + ) await self.execute( - "INSERT INTO task_intents(id,tenant_id,project_id,environment_id,run_id,node_id,payloa" - "d,worker_release_id,operation_key,status) " + "WITH inserted AS (INSERT INTO task_intents(id,tenant_id,project_id,environment_id,run_id,node_id," + "payload,worker_release_id,operation_key,status) " "VALUES(:id,:tenant,:project,:environment,:run,:node," - "cast(:payload AS jsonb),:release,:operation,'ready')", - id=uuid4(), - run=view.id, + "cast(:payload AS jsonb),:release,:operation,'ready') RETURNING id) " + "INSERT INTO task_facts(tenant_id,project_id,environment_id,task_id,run_id,node_id,instance_key," + "task_type,status,created_at,ready_since) SELECT :tenant,:project,:environment,id,:run," + ":fact_node,:instance,:task_type,'ready',:at,:at FROM inserted", + **values, node=command.node_id, payload=json.dumps(payload), release=release, @@ -264,7 +284,10 @@ async def persist( await self.record_evidence(view, event, result, task_timing) if event.type == "task_completed": await self.execute( - f"UPDATE task_intents SET status='completed' WHERE {SCOPE} AND run_id=:run AND node_id=:node", + task_fact_update( + f"UPDATE task_intents SET status='completed' WHERE {SCOPE} AND run_id=:run AND node_id=:node" + ), + at=event.timestamp, run=view.id, node=event.data["node_id"], ) @@ -296,12 +319,16 @@ async def persist( ), ) await self.execute( - f"UPDATE task_intents SET status='cancelled' WHERE {SCOPE} AND run_id=:run " - "AND status IN ('ready','leased','retry_pending','incident','failed')", + task_fact_update( + f"UPDATE task_intents SET status='cancelled' WHERE {SCOPE} AND run_id=:run " + "AND status IN ('ready','leased','retry_pending','incident','failed')" + ), + at=event.timestamp, run=view.id, ) await self.execute(f"UPDATE run_deadlines SET consumed=true WHERE {SCOPE} AND run_id=:run", run=view.id) + await FactRepository(self.tx).project_run(view, event, result) await self.outbox.append( self.tx, IntegrationEvent( @@ -414,9 +441,12 @@ async def locked_task( ) return (dict(run), dict(task)) if task else None - async def task_status(self, identifier: UUID, status: str) -> None: + async def task_status(self, identifier: UUID, status: str, *, at: datetime | None = None) -> None: await self.execute( - f"UPDATE task_intents SET status=:status WHERE {SCOPE} AND id=:id", id=identifier, status=status + task_fact_update(f"UPDATE task_intents SET status=:status WHERE {SCOPE} AND id=:id"), + id=identifier, + status=status, + at=at if at is not None else await self.now(), ) async def task_deadline(self, run_id: UUID, task_deadline: datetime) -> datetime: @@ -487,19 +517,29 @@ async def expired_ready(self, limit: int) -> list[UUID]: ) return [row["id"] for row in rows] - async def project_incidents(self, view: RunView | TerminalControl) -> None: + async def project_incidents(self, view: RunView | TerminalControl, *, at: datetime | None = None) -> None: """Projection only: never changes or repairs authoritative accepted state.""" + at = at if at is not None else await self.now() active = {key: value.model_dump() for key, value in view.state.incidents.items()} if view.state.status == "suspended" and not active and view.state.incident: active["@legacy"] = {"node_id": None, "generation": None, "code": view.state.incident} for key, value in active.items(): + identity = author_instance(value["node_id"]) if value["node_id"] is not None else None await self.execute( - "INSERT INTO incidents(id,tenant_id,project_id,environment_id,run_id,incident_key," + "WITH changed AS (INSERT INTO incidents(id,tenant_id,project_id,environment_id,run_id,incident_key," "node_id,generation,origin_code,code,status,revision) VALUES(" "md5(cast(cast(:run AS uuid) AS text) || ':' || :key)::uuid,:tenant,:project,:environment,:run,:key," ":node,:generation,:code,:code,'active',1) ON CONFLICT(run_id,incident_key) DO UPDATE " "SET code=excluded.code,revision=incidents.revision+1 " - "WHERE incidents.status='active' AND incidents.code<>excluded.code", + "WHERE incidents.status='active' AND incidents.code<>excluded.code RETURNING id) " + "INSERT INTO incident_facts(tenant_id,project_id,environment_id,incident_id,node_id,instance_key," + "opened_at,updated_at,workflow_name,workflow_version,test) SELECT :tenant,:project,:environment," + "c.id,:fact_node,:instance,:at,:at,f.workflow_name,f.workflow_version,f.test FROM changed c " + f"JOIN run_facts f ON f.run_id=:run AND f.{SCOPE.replace(' AND ', ' AND f.')} " + "ON CONFLICT(incident_id) DO UPDATE SET updated_at=excluded.updated_at", + fact_node=identity.node_id if identity else None, + instance=identity.instance_key if identity else "", + at=at, run=view.id, key=key, node=value["node_id"], @@ -507,8 +547,10 @@ async def project_incidents(self, view: RunView | TerminalControl) -> None: code=value["code"], ) await self.execute( - f"UPDATE incidents SET status='closed',revision=revision+1 WHERE {SCOPE} AND run_id=:run " - "AND status='active' AND NOT (incident_key=ANY(cast(:keys AS text[])))", + f"WITH closed AS (UPDATE incidents SET status='closed',revision=revision+1 WHERE {SCOPE} AND run_id=:run " + "AND status='active' AND NOT (incident_key=ANY(cast(:keys AS text[]))) RETURNING id) " + f"UPDATE incident_facts SET updated_at=:at WHERE {SCOPE} AND incident_id IN (SELECT id FROM closed)", + at=at, run=view.id, keys=list(active), ) diff --git a/src/firefly_weave/runtime/scheduler.py b/src/firefly_weave/runtime/scheduler.py index 26b8eeb7..bd83088a 100644 --- a/src/firefly_weave/runtime/scheduler.py +++ b/src/firefly_weave/runtime/scheduler.py @@ -28,6 +28,7 @@ from firefly_weave.access.scheduler import next_scope, tenant_page from firefly_weave.definitions.models import CatalogError from firefly_weave.operations.execution import ReservedSlots +from firefly_weave.operations.loop_health import LoopHealth, track from firefly_weave.persistence.migrations import check_schema from firefly_weave.persistence.uow import UnitOfWork from firefly_weave.runtime.recovery import RecoveryService @@ -55,10 +56,12 @@ def __init__( schedules: Scheduler, *, operational: Callable[[], bool] = lambda: True, + health: LoopHealth | None = None, ) -> None: self.settings, self.uow, self.recovery = settings, uow, recovery self.schedules = schedules self.operational = operational + self.health = health if health is not None else LoopHealth() self.engine: AsyncEngine | None = None self.task: asyncio.Task[None] | None = None @@ -93,12 +96,37 @@ async def open(self) -> None: # A request can open this loop. An empty context keeps the request's execution # lease and identity, which end with its response, out of the long-lived task. self.task = asyncio.create_task(self.poll(), name="weave-recovery", context=Context()) + # Schedules fire inside the recovery cycle, so they run exactly while this task runs. + track(self.health, self.task, "recovery_loop", "schedules") except BaseException: await self.close() raise - async def cycle(self) -> None: + async def cycle(self) -> bool: + """One bounded traversal; True only when every tenant's recovery scan completed. + + Schedules are reported separately, and only while the platform is operational: a cycle counts as a + schedules success only when every visited tenant's schedule scan ran and completed. + """ + operational = self.operational() + if operational: + self.health.started("schedules") + try: + recovered, scheduled = await self._traverse() + except Exception: + if operational: + self.health.failed("schedules") + raise + if operational: + if scheduled: + self.health.succeeded("schedules") + else: + self.health.failed("schedules") + return recovered + + async def _traverse(self) -> tuple[bool, bool]: assert self.engine is not None + recovered = scheduled = True async with RECOVERY_EXECUTION.execution(): budget = min(self.settings.database_timeout_seconds, 5.0) async with asyncio.timeout(budget): @@ -120,8 +148,11 @@ async def cycle(self) -> None: logging.getLogger(__name__).error( "Tenant recovery scan failed (%s); traversal will continue", failure_cause(error) ) + # The tenant's schedules did not run in this cycle either. + recovered = scheduled = False continue if not self.operational(): + scheduled = False continue try: async with asyncio.timeout(budget): @@ -130,15 +161,24 @@ async def cycle(self) -> None: logging.getLogger(__name__).error( "Tenant schedule scan failed (%s); traversal will continue", failure_cause(error) ) + scheduled = False + return recovered, scheduled async def poll(self) -> None: while True: + self.health.started("recovery_loop") try: - await self.cycle() + recovered = await self.cycle() except Exception as error: logging.getLogger(__name__).error( "Recovery catalog scan failed (%s); retrying on next poll", failure_cause(error) ) + recovered = False + # A failure a cycle caught and logged is still a failed cycle, never a success. + if recovered: + self.health.succeeded("recovery_loop") + else: + self.health.failed("recovery_loop") await asyncio.sleep(self.settings.scheduler_poll_seconds) async def close(self) -> None: @@ -147,6 +187,8 @@ async def close(self) -> None: with suppress(asyncio.CancelledError): await self.task self.task = None + for name in ("recovery_loop", "schedules"): + self.health.running(name, False) if self.engine is not None: await self.engine.dispose() self.engine = None diff --git a/src/firefly_weave/runtime/service.py b/src/firefly_weave/runtime/service.py index dbbf14aa..de4e0a37 100644 --- a/src/firefly_weave/runtime/service.py +++ b/src/firefly_weave/runtime/service.py @@ -28,6 +28,7 @@ from firefly_weave.access.models import Principal from firefly_weave.access.repository import load_principal from firefly_weave.access.service import audit +from firefly_weave.api.transport import encode_cursor, encode_cursor_v2 from firefly_weave.compiler.api import import_artifact from firefly_weave.compiler.canonical import canonical_digest from firefly_weave.compiler.expressions import ExpressionFailure @@ -36,6 +37,7 @@ from firefly_weave.contracts.human_tasks import ManualControlRequest from firefly_weave.contracts.instance_keys import node_of from firefly_weave.contracts.operations import CancelRunRequest +from firefly_weave.contracts.run_views import RunListQuery from firefly_weave.contracts.runtime import ( CapacityRunAcknowledgment, RunListFilters, @@ -45,6 +47,9 @@ ) from firefly_weave.definitions.models import CatalogError from firefly_weave.definitions.service import DefinitionService +from firefly_weave.operations.fact_positions import RunPosition +from firefly_weave.operations.fact_reads import FactReadRepository, unavailable +from firefly_weave.operations.facts import RunStartFacts from firefly_weave.persistence.idempotency import Idempotency from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.kernel import KernelError, action_schemas, validate, validate_action, workflow @@ -166,8 +171,12 @@ async def start( context: AuditContext, tx: Transaction | None = None, not_after: datetime | None = None, + start_facts: RunStartFacts | None = None, ) -> RunView: self.require(actor, scope, "run.start", context) + start_facts = start_facts if start_facts is not None else RunStartFacts() + if not isinstance(start_facts, RunStartFacts): + raise ValueError("Trusted start facts required") async with self.definitions.transaction(scope, tx) as enlisted: replay = Idempotency( enlisted, @@ -204,7 +213,11 @@ async def start( if not_after is not None and now >= not_after: raise CatalogError(409, "WV-SCHEDULE-STALE", "Occurrence grace expired during admission") event = RuntimeEvent( - id=uuid4(), type="started", timestamp=now, sequence=1, data={"admission_policy": "classified-v1"} + id=uuid4(), + type="started", + timestamp=now, + sequence=1, + data={"admission_policy": "classified-v1", "origin": start_facts.origin}, ) result = await transition(RunState(input=request.input), event, artifact) self.require(await load_principal(enlisted.session, actor.id), scope, "run.start", context) @@ -219,7 +232,9 @@ async def start( correlation_key=request.correlation_key, business_key=request.business_key, ) - await repository.insert(view, envelope, request, actor.id) + await repository.insert( + view, envelope, request, actor.id, start_facts=start_facts, at=now, imported=artifact, ir=ir + ) from firefly_weave.files.authority import admit_files await admit_files(enlisted, view.id, request.input, actor, context=context) @@ -306,8 +321,10 @@ async def observe_unavailable(self, tx: Transaction, row: dict[str, Any]) -> boo await RuntimeRepository(tx, self.definitions.outbox).observe_policy_block(row) return decision != "available" - async def set_task_status(self, tx: Transaction, identifier: UUID, status: str) -> None: - await RuntimeRepository(tx, self.definitions.outbox).task_status(identifier, status) + async def set_task_status( + self, tx: Transaction, identifier: UUID, status: str, *, at: datetime | None = None + ) -> None: + await RuntimeRepository(tx, self.definitions.outbox).task_status(identifier, status, at=at) async def finish_verified_task( self, @@ -388,9 +405,9 @@ async def finish_verified_task( ), ) if rejection_code: - await repository.task_status(verified.task["id"], "incident") + await repository.task_status(verified.task["id"], "incident", at=event.timestamp) elif failed: - await repository.task_status(verified.task["id"], "failed") + await repository.task_status(verified.task["id"], "failed", at=event.timestamp) else: from firefly_weave.runtime.waits import settle @@ -562,7 +579,15 @@ async def retry_run( raise CatalogError(409, "WV-RUNTIME-STATE", "Only a terminal run can create a linked retry") # Retry replay is owned by the outer receipt; the nested start gets a fresh identity. start_key = str(uuid4()) - view = await self.start(current, scope, request, start_key, context=context, tx=tx) + view = await self.start( + current, + scope, + request, + start_key, + context=context, + tx=tx, + start_facts=RunStartFacts(origin="retry", retried_from_run_id=run_id), + ) await repository.execute( "INSERT INTO run_retry_links VALUES(:tenant,:project,:environment,:run,:parent)", run=view.id, @@ -590,18 +615,44 @@ async def list( limit: int = 50, cursor: UUID | None = None, filters: RunListFilters | None = None, + query: RunListQuery | None = None, + position: RunPosition | UUID | None = None, context: AuditContext, ) -> dict[str, Any]: from firefly_weave.access.repository import load_principal - from firefly_weave.persistence.paging import page_ids + from firefly_weave.persistence.paging import PAGE_BYTES + + # The keyword arguments are the original contract: ID order and raw UUID positions in both directions. + legacy_adapter = query is None + if query is None: + values = (filters or RunListFilters()).model_dump(mode="json") + values["status"] = [values["status"]] if values["status"] is not None else [] + query = RunListQuery.model_validate_json(json.dumps({**values, "limit": limit, "order": "id"})) + position = cursor + elif filters is not None or cursor is not None: + raise ValueError("Use either a query or legacy list arguments") + if query.order != "id" and isinstance(position, UUID): + raise ValueError("Chronological order requires a time position") + if query.order == "id" and isinstance(position, RunPosition): + raise ValueError("ID order requires an ID position") self.require(actor, scope, "run.read", context) async with self.definitions.transaction(scope, None, mutation=False) as tx: actor = await load_principal(tx.session, actor.id) self.require(actor, scope, "run.read", context) - ids = await page_ids(tx, "runs", limit, cursor, run_filters=(filters or RunListFilters()).model_dump()) + repository = FactReadRepository(tx) + rows = await repository.legacy_rows(query, position) + limit = query.limit + # Measure the whole visible page before any full run is fetched; the lookahead row is not charged. + if await repository.page_bytes([row["run_id"] for row in rows[:limit]]) > PAGE_BYTES: + raise CatalogError(429, "WV-PAGE-LIMIT", "Selected page exceeds the logical byte limit") items = [] - for identifier in ids[:limit]: + for row in rows[:limit]: + identifier = row["run_id"] + # Only a policy block withholds a full-run item here; the authoritative read classifies the rest. + if row["policy_blocked"]: + items.append(unavailable(identifier).model_dump(mode="json")) + continue try: items.append( (await self.read(actor, scope, identifier, context=context, tx=tx)).model_dump(mode="json") @@ -617,4 +668,18 @@ async def list( "omissions": [{"path": "", "reason": "classification_unavailable"}], } ) - return {"items": items, "next_cursor": str(ids[limit - 1]) if len(ids) > limit else None} + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + if query.order == "id": + next_cursor = ( + str(last["run_id"]) + if legacy_adapter + else encode_cursor(scope, query.cursor_collection(), last["run_id"]) + ) + else: + at = last["updated_at"] if query.order == "updated_desc" else last["started_at"] + next_cursor = encode_cursor_v2( + scope, query.cursor_collection(), at.isoformat() if at else None, str(last["run_id"]) + ) + return {"items": items, "next_cursor": next_cursor} diff --git a/src/firefly_weave/runtime/terminal_capacity.py b/src/firefly_weave/runtime/terminal_capacity.py index c39e73ad..d81a225e 100644 --- a/src/firefly_weave/runtime/terminal_capacity.py +++ b/src/firefly_weave/runtime/terminal_capacity.py @@ -29,6 +29,7 @@ from firefly_weave.definitions.models import CatalogError from firefly_weave.operations.execution import execute_pure from firefly_weave.operations.exports import lock_digest +from firefly_weave.operations.facts import FactRepository, task_fact_update from firefly_weave.operations.redaction import Omission from firefly_weave.runtime.admission import terminal_signals from firefly_weave.runtime.models import RuntimeEvent @@ -164,7 +165,10 @@ async def persist_terminal( break await revoke_attempts(repository.tx, row["id"], tasks, control_failure=event.type == "cancelled") await repository.execute( - f"UPDATE task_intents SET status='cancelled' WHERE {SCOPE} AND id=ANY(cast(:ids AS uuid[]))", + task_fact_update( + f"UPDATE task_intents SET status='cancelled' WHERE {SCOPE} AND id=ANY(cast(:ids AS uuid[]))" + ), + at=event.timestamp, ids=[task["id"] for task in tasks], ) await repository.execute( @@ -172,7 +176,10 @@ async def persist_terminal( run=row["id"], ) await repository.execute( - f"UPDATE incidents SET status='closed',revision=revision+1 WHERE {SCOPE} AND run_id=:run AND status='active'", + f"WITH closed AS (UPDATE incidents SET status='closed',revision=revision+1 WHERE {SCOPE} AND run_id=:run " + "AND status='active' RETURNING id) UPDATE incident_facts SET updated_at=:at " + f"WHERE {SCOPE} AND incident_id IN (SELECT id FROM closed)", + at=event.timestamp, run=row["id"], ) if row.get("artifact"): @@ -212,6 +219,9 @@ async def persist_terminal( sequence=event.sequence, evidence=evidence.model_dump_json(), ) + await FactRepository(repository.tx).project_terminal( + row["id"], cast(Literal["cancelled", "timed_out"], event.type), event.timestamp, event.sequence + ) await repository.outbox.append( repository.tx, IntegrationEvent( diff --git a/src/firefly_weave/sdk/client.py b/src/firefly_weave/sdk/client.py index a4849259..1cce6af2 100644 --- a/src/firefly_weave/sdk/client.py +++ b/src/firefly_weave/sdk/client.py @@ -19,7 +19,7 @@ from __future__ import annotations import inspect -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Any, Literal, Protocol, cast from urllib.parse import urlsplit from uuid import UUID @@ -56,6 +56,7 @@ TaskGroup, TaskGroupRequest, ) +from firefly_weave.contracts.incident_queries import IncidentOrder, IncidentQuery from firefly_weave.contracts.integration_events import DeliveryAttempt, DeliveryView, Subscription, SubscriptionRequest from firefly_weave.contracts.language import LanguageManifest from firefly_weave.contracts.lumi import ( @@ -102,6 +103,7 @@ catalog_lock, ) from firefly_weave.contracts.run_lifecycle import RunLifecycle, RunLifecycleRequest, RunPurgeRequest +from firefly_weave.contracts.run_views import RunListOrder, RunListQuery from firefly_weave.contracts.runtime import ( CapacityRunAcknowledgment, RunView, @@ -115,6 +117,13 @@ from firefly_weave.contracts.teams import TeamsReactivateRequest, TeamsReference, TeamsRevokeRequest from firefly_weave.contracts.values import JsonObject from firefly_weave.contracts.whatsapp import WhatsAppDeliveryState, WhatsAppStatusFact +from firefly_weave.contracts.worker_queries import ( + ReleaseUsage, + ReleaseUsageQuery, + RetireStaleRequest, + RetireStaleResult, + WorkerListQuery, +) from firefly_weave.contracts.workers import ( ClaimRequest, CompleteRequest, @@ -134,6 +143,7 @@ ) from firefly_weave.operations.debug.models import DebugCommand, DebugCreate, DebugSession from firefly_weave.sdk.errors import ContractError, PreconditionFailed, TransportError, WeaveError +from firefly_weave.sdk.worker_registration import negotiate_worker_registration from firefly_weave.triggers.models import Trigger, TriggerRequest if TYPE_CHECKING: @@ -221,7 +231,7 @@ async def invoke( body: BaseModel | None = None, revision: int | None = None, idempotency_key: str | None = None, - query: dict[str, str | int] | None = None, + query: Mapping[str, str | int | list[str]] | None = None, adapter: str | None = None, ) -> Any: """Typed registry boundary also used by the thin CLI; no arbitrary URL input.""" @@ -540,16 +550,22 @@ async def run_incidents( *, limit: int = 50, cursor: str | None = None, + order: IncidentOrder = "opened_desc", + query: IncidentQuery | None = None, ) -> Page[IncidentView | UnavailableResource]: + """One run's incidents with either a typed ``query`` or the keyword arguments; mixing them raises.""" return cast( Page[IncidentView | UnavailableResource], await self.invoke( "incidents.run_list", identifier=identifier, - query=self._page(limit, cursor), + query=self._incident_query(limit, cursor, order, query), ), ) + async def read_incident(self, identifier: UUID) -> IncidentView | UnavailableResource: + return cast(IncidentView | UnavailableResource, await self.invoke("incidents.read", identifier=identifier)) + async def resolve_incident(self, identifier: UUID, request: IncidentResolution, *, revision: int) -> IncidentView: return cast( IncidentView, await self.invoke("incidents.resolve", identifier=identifier, body=request, revision=revision) @@ -559,7 +575,13 @@ async def register_release(self, request: ReleaseRequest) -> WorkerRelease: return cast(WorkerRelease, await self.invoke("releases.create", body=request)) async def register_worker(self, request: InstanceRequest) -> WorkerInstance: - return cast(WorkerInstance, await self.invoke("workers.create", body=request)) + """Register a worker; a server that predates `identity` gets one retry without it.""" + + async def send(include_identity: bool) -> WorkerInstance: + value = request if include_identity else request.model_copy(update={"identity": None}) + return cast(WorkerInstance, await self.invoke("workers.create", body=value)) + + return await negotiate_worker_registration(send, has_identity=request.identity is not None) async def read_release(self, identifier: UUID) -> WorkerRelease: return cast(WorkerRelease, await self.invoke("releases.read", identifier=identifier)) @@ -715,29 +737,77 @@ async def list_runs( correlation_key: str | None = None, status: str | None = None, include_archived: bool = False, + order: RunListOrder = "started_desc", + query: RunListQuery | None = None, ) -> Page[RunView | UnavailableResource]: + """List runs with either a typed ``query`` or the keyword filters; mixing them raises ``ValueError``.""" + values: dict[str, str | int | list[str]] + if query is not None: + keywords = (limit, cursor, business_key, correlation_key, status, include_archived, order) + if keywords != (50, None, None, None, None, False, "started_desc"): + raise ValueError("Use either a query or keyword list arguments") + values = { + key: str(value).lower() if isinstance(value, bool) else value + for key, value in query.model_dump(mode="json", exclude_none=True).items() + if not isinstance(value, list) or value + } + else: + values = { + **self._page(limit, cursor), + "order": order, + "include_archived": str(include_archived).lower(), + **{ + key: value + for key, value in { + "business_key": business_key, + "correlation_key": correlation_key, + "status": status, + }.items() + if value is not None + }, + } return cast( Page[RunView | UnavailableResource], await self.invoke( "runs.list", - query={ - **self._page(limit, cursor), - **{ - key: value - for key, value in { - "business_key": business_key, - "correlation_key": correlation_key, - "status": status, - }.items() - if value is not None - }, - "include_archived": str(include_archived).lower(), - }, + query=values, ), ) - async def list_workers(self, *, limit: int = 50, cursor: str | None = None) -> Page[WorkerStatus]: - return cast(Page[WorkerStatus], await self.invoke("workers.list", query=self._page(limit, cursor))) + async def list_workers( + self, *, limit: int = 50, cursor: str | None = None, query: WorkerListQuery | None = None + ) -> Page[WorkerStatus]: + """Workers with either a typed ``query`` or the page keywords; mixing them raises ``ValueError``. + + Without a query the server lists current workers, most recently seen first. + """ + if query is not None and (limit, cursor) != (50, None): + raise ValueError("Use either a query or keyword list arguments") + values = self._page(limit, cursor) if query is None else self._query(query) + return cast(Page[WorkerStatus], await self.invoke("workers.list", query=values)) + + async def retire_stale_workers(self, request: RetireStaleRequest, *, idempotency_key: str) -> RetireStaleResult: + """Revoke (or, with ``dry_run``, preview) up to 100 workers offline longer than the threshold.""" + return cast( + RetireStaleResult, + await self.invoke("workers.retire_stale", body=request, idempotency_key=idempotency_key), + ) + + async def release_usage(self, identifier: UUID, *, query: ReleaseUsageQuery | None = None) -> ReleaseUsage: + """Worker counts, pinning activations and the readable deployments that run a release.""" + return cast( + ReleaseUsage, + await self.invoke("releases.usage", identifier=identifier, query=self._query(query or ReleaseUsageQuery())), + ) + + @staticmethod + def _query(query: BaseModel) -> dict[str, str | int | list[str]]: + # Booleans travel as true or false; unset filters and empty lists are left out. + return { + key: str(value).lower() if isinstance(value, bool) else value + for key, value in query.model_dump(mode="json", exclude_none=True).items() + if not isinstance(value, list) or value + } async def list_releases(self, *, limit: int = 50, cursor: str | None = None) -> Page[WorkerRelease]: return cast(Page[WorkerRelease], await self.invoke("releases.list", query=self._page(limit, cursor))) @@ -746,13 +816,34 @@ async def list_triggers(self, *, limit: int = 50, cursor: str | None = None) -> return cast(Page[Trigger], await self.invoke("triggers.list", query=self._page(limit, cursor))) async def list_incidents( - self, *, limit: int = 50, cursor: str | None = None + self, + *, + limit: int = 50, + cursor: str | None = None, + order: IncidentOrder = "opened_desc", + query: IncidentQuery | None = None, ) -> Page[IncidentView | UnavailableResource]: + """Incidents with either a typed ``query`` or the keyword arguments; mixing them raises ``ValueError``.""" return cast( Page[IncidentView | UnavailableResource], - await self.invoke("incidents.list", query=self._page(limit, cursor)), + await self.invoke("incidents.list", query=self._incident_query(limit, cursor, order, query)), ) + @classmethod + def _incident_query( + cls, limit: int, cursor: str | None, order: IncidentOrder, query: IncidentQuery | None + ) -> dict[str, str | int | list[str]]: + if query is None: + return {**cls._page(limit, cursor), "order": order} + if (limit, cursor, order) != (50, None, "opened_desc"): + raise ValueError("Use either a query or keyword list arguments") + # Statuses travel as repeated parameters; empty filters are left out. + return { + key: str(value).lower() if isinstance(value, bool) else value + for key, value in query.model_dump(mode="json", exclude_none=True).items() + if not isinstance(value, list) or value + } + async def list_schedules(self, *, limit: int = 50, cursor: str | None = None) -> Page[ScheduleView]: return cast(Page[ScheduleView], await self.invoke("schedules.list", query=self._page(limit, cursor))) diff --git a/src/firefly_weave/sdk/platform_ai_setup.py b/src/firefly_weave/sdk/platform_ai_setup.py index 09e8fba3..622503c7 100644 --- a/src/firefly_weave/sdk/platform_ai_setup.py +++ b/src/firefly_weave/sdk/platform_ai_setup.py @@ -288,7 +288,8 @@ async def ensure_connection(client: WeaveClient, request: ConnectionRequest) -> async def presence(client: WeaveClient, release_id: str) -> dict[str, Any]: - page = await client.invoke("workers.list", query={"limit": 100}) + # Every worker of the release, so one not seen for hours reads as stale rather than not seen yet. + page = await client.invoke("workers.list", query={"release_id": release_id, "state": "all", "limit": 100}) found = [item for item in page.items if str(item.release_id) == release_id and not item.revoked] best = next((item for item in found if item.presence == "recent"), found[0] if found else None) if best is None: diff --git a/src/firefly_weave/sdk/worker_identity.py b/src/firefly_weave/sdk/worker_identity.py new file mode 100644 index 00000000..1dc169b6 --- /dev/null +++ b/src/firefly_weave/sdk/worker_identity.py @@ -0,0 +1,127 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""The identity a worker reports when it registers. + +It reads exactly five settings and never lists, logs or echoes the environment. Optional +values that are not valid (an unusual hostname, platform or Kubernetes name) are left out +instead of stopping the worker; labels the operator set explicitly fail clearly. +""" + +from __future__ import annotations + +import os +import platform +import socket +from collections.abc import Mapping +from importlib import metadata +from typing import Literal + +from pydantic import TypeAdapter, ValidationError + +from firefly_weave.contracts.workers import WorkerHost, WorkerIdentity, WorkerVersion + +REPORT_HOST = "WEAVE_WORKER_REPORT_HOST" +LABELS = "WEAVE_WORKER_LABELS" +NAMESPACE = "WEAVE_POD_NAMESPACE" +NODE = "WEAVE_NODE_NAME" +KUBERNETES = "KUBERNETES_SERVICE_HOST" + +_HOST: TypeAdapter[str] = TypeAdapter(WorkerHost) +_VERSION: TypeAdapter[str] = TypeAdapter(WorkerVersion) + + +def _bounded(adapter: TypeAdapter[str], value: object) -> str | None: + if not isinstance(value, str): + return None + try: + return adapter.validate_python(value, strict=True) + except ValidationError: + return None + + +def _report_host(raw: str | None) -> bool: + if raw is None: + return True + value = raw.strip().lower() + if value in {"true", "1", "yes"}: + return True + if value in {"false", "0", "no"}: + return False + raise ValueError(f"{REPORT_HOST} must be true or false") + + +def _labels(raw: str | None) -> dict[str, str]: + labels: dict[str, str] = {} + for item in (raw or "").split(","): + if not item.strip(): + continue + key, separator, value = item.partition("=") + key, value = key.strip(), value.strip() + if not separator or not key or key in labels: + raise ValueError(f"{LABELS} needs distinct key=value pairs separated by commas") + labels[key] = value + return labels + + +def _sdk_version() -> str | None: + try: + return metadata.version("firefly-weave") + except metadata.PackageNotFoundError: + return None + + +def worker_identity( + *, + runtime: Literal["embedded", "container", "process"] | None = None, + environ: Mapping[str, str] | None = None, +) -> WorkerIdentity: + """Describe this worker for its registration. + + `runtime` is what the caller knows about how it runs; without it a worker in a + Kubernetes pod reports `container` and any other worker `process`. The host is the + machine's hostname (the pod name in Kubernetes) unless `WEAVE_WORKER_REPORT_HOST=false`. + """ + settings = os.environ if environ is None else environ + kubernetes = bool(settings.get(KUBERNETES)) + chosen: Literal["embedded", "container", "process"] = runtime or ("container" if kubernetes else "process") + hint: Literal["kubernetes", "docker", "none"] = ( + "kubernetes" if kubernetes else "docker" if chosen == "container" else "none" + ) + host = _bounded(_HOST, socket.gethostname()) if _report_host(settings.get(REPORT_HOST)) else None + labels = _labels(settings.get(LABELS)) + try: + return WorkerIdentity( + host=host, + runtime=chosen, + sdk_version=_bounded(_VERSION, _sdk_version()), + language_version=_bounded(_VERSION, platform.python_version()), + os=_bounded(_VERSION, platform.system()), + arch=_bounded(_VERSION, platform.machine()), + platform_hint=hint, + namespace=_bounded(_HOST, settings.get(NAMESPACE)), + node=_bounded(_HOST, settings.get(NODE)), + labels=labels, + ) + except ValidationError: + pass + # Raised outside the handler so no chained validation error repeats the rejected input, + # which may be a value the operator meant to keep private. + raise ValueError( + f"{LABELS} accepts up to 8 labels. Keys have up to 64 letters, digits, '.', '_', '/' or '-' " + "and do not name a token, password, secret or authorization. Values have up to 256 " + "characters without control characters, and the whole identity stays within 4 KiB." + ) diff --git a/src/firefly_weave/sdk/worker_registration.py b/src/firefly_weave/sdk/worker_registration.py new file mode 100644 index 00000000..028f8867 --- /dev/null +++ b/src/firefly_weave/sdk/worker_registration.py @@ -0,0 +1,74 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Worker registration that stays compatible with servers that predate worker identity. + +An older server refuses the unknown `identity` field with 422 `WV-VALIDATION`. Registration +then retries exactly once without it, on the same client, authentication and transport. Any +other failure, including a second refusal, a transport error or cancellation, is raised as is. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Mapping +from typing import TYPE_CHECKING + +from firefly_weave.contracts.values import JsonValue +from firefly_weave.sdk.errors import WeaveError + +if TYPE_CHECKING: + import httpx + + +def identity_rejected(error: WeaveError | httpx.HTTPStatusError) -> bool: + """Whether a registration failed the way a server without worker identity refuses it.""" + if isinstance(error, WeaveError): + return error.status == 422 and error.code == "WV-VALIDATION" + if error.response.status_code != 422: + return False + try: + body = error.response.json() + except ValueError: + return False + return isinstance(body, dict) and body.get("code") == "WV-VALIDATION" + + +async def negotiate_worker_registration[T](send: Callable[[bool], Awaitable[T]], *, has_identity: bool) -> T: + """Call `send(True)` when there is an identity, and `send(False)` once if it is refused.""" + import httpx + + try: + return await send(has_identity) + except (WeaveError, httpx.HTTPStatusError) as error: + if not has_identity or not identity_rejected(error): + raise + return await send(False) + + +async def register_worker_http( + client: httpx.AsyncClient, environment_url: str, body: Mapping[str, JsonValue] +) -> httpx.Response: + """Register through a worker's existing HTTP client; `body` itself is never changed.""" + + async def send(include_identity: bool) -> httpx.Response: + payload = dict(body) + if not include_identity: + payload.pop("identity", None) + response = await client.post(environment_url.rstrip("/") + "/workers", json=payload) + response.raise_for_status() + return response + + return await negotiate_worker_registration(send, has_identity=body.get("identity") is not None) diff --git a/src/firefly_weave/studio/service.py b/src/firefly_weave/studio/service.py index 3f860866..6f9bb787 100644 --- a/src/firefly_weave/studio/service.py +++ b/src/firefly_weave/studio/service.py @@ -101,6 +101,9 @@ "deployment_runners", "environments", "incidents", + # The environment summary and telemetry settings re-check each section's capability server-side. + "operations_summary", + "operations_telemetry", # Draft save/retire and Simulate are authorized server-side like every other family. "drafts", "debug", @@ -108,7 +111,7 @@ ) # Single read operations from families that otherwise stay operator-only: releases.create # registers image identities and remains a CLI/operator step. -STUDIO_OPERATIONS = frozenset({"releases.list", "releases.read"}) +STUDIO_OPERATIONS = frozenset({"releases.list", "releases.read", "releases.usage"}) CATALOG_COLLECTIONS = frozenset({"drafts", "workflows", "actions", "connectors", "decision-tables"}) diff --git a/src/firefly_weave/triggers/kafka.py b/src/firefly_weave/triggers/kafka.py index b9be6bd8..536aede9 100644 --- a/src/firefly_weave/triggers/kafka.py +++ b/src/firefly_weave/triggers/kafka.py @@ -50,6 +50,7 @@ from firefly_weave.contracts.runtime import StartRunRequest from firefly_weave.contracts.values import JsonObject, JsonValue from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.facts import RunStartFacts from firefly_weave.operations.redaction import project from firefly_weave.persistence.idempotency import lock from firefly_weave.persistence.uow import Transaction @@ -416,6 +417,7 @@ async def consume(self, record: BrokerRecord, *, authority: ConsumerAuthority) - f"kafka:{route.id}:{event}", context=AuditContext(), tx=tx, + start_facts=RunStartFacts(origin="broker"), ) run_id = run.id else: diff --git a/src/firefly_weave/triggers/kafka_loop.py b/src/firefly_weave/triggers/kafka_loop.py index 4e65cfc1..36a61c67 100644 --- a/src/firefly_weave/triggers/kafka_loop.py +++ b/src/firefly_weave/triggers/kafka_loop.py @@ -28,6 +28,7 @@ from firefly_weave.connectors.broker import BrokerConnectionConfig from firefly_weave.connectors.kafka_transport import BrokerClients, BrokerOwner from firefly_weave.operations.execution import ReservedSlots +from firefly_weave.operations.loop_health import TRAVERSAL_SECONDS, LoopHealth, track from firefly_weave.persistence.migrations import check_schema from firefly_weave.persistence.uow import UnitOfWork from firefly_weave.settings import Settings @@ -39,11 +40,22 @@ class KafkaLoop: - def __init__(self, settings: Settings, uow: UnitOfWork, triggers: KafkaTrigger, clients: BrokerClients) -> None: + def __init__( + self, + settings: Settings, + uow: UnitOfWork, + triggers: KafkaTrigger, + clients: BrokerClients, + *, + health: LoopHealth | None = None, + ) -> None: self.settings, self.uow, self.triggers, self.clients = settings, uow, triggers, clients + self.health = health if health is not None else LoopHealth() self.engine: AsyncEngine | None = None self.task: asyncio.Task[None] | None = None self.turns: set[asyncio.Task[None]] = set() + # A running consumer task does not prove a connected source: a failed turn fails the next cycle. + self.turn_failed = False async def open(self) -> None: if not self.settings.broker.enabled or not self.settings.kafka_consumer_enabled: @@ -73,16 +85,19 @@ async def open(self) -> None: # A request can open this loop. An empty context keeps the request's execution # lease and identity, which end with its response, out of the long-lived task. self.task = asyncio.create_task(self.poll(), name="weave-broker-traversal", context=Context()) + track(self.health, self.task, "kafka_loop") except BaseException: await self.close() raise - async def cycle(self) -> None: + async def cycle(self) -> bool: + """One bounded traversal turn; False when a consumer turn failed since the previous cycle.""" + healthy, self.turn_failed = not self.turn_failed, False if ( len(self.turns) >= self.settings.broker.max_clients - 1 or len(self.clients.owners) >= self.settings.broker.max_clients - 1 ): - return + return healthy assert self.engine is not None async with asyncio.timeout(5): async with self.engine.begin() as connection: @@ -90,15 +105,16 @@ async def cycle(self) -> None: await connection.execute(text("SET LOCAL lock_timeout='1000ms'")) tenant = await connection.scalar(text("SELECT weave_broker_tenants(1)")) if tenant is None: - return + return healthy authority = await next_scope(self.uow, tenant, broker=True) if authority is None: - return + return healthy source = await self.triggers.scheduled_claim(authority) if source is not None: task = asyncio.create_task(self.turn(source), name="weave-kafka-consumer") self.turns.add(task) task.add_done_callback(self.turns.discard) + return healthy async def turn(self, source: KafkaSource) -> None: owner: BrokerOwner | None = None @@ -149,6 +165,7 @@ async def monitor() -> None: except asyncio.CancelledError: raise except Exception: + self.turn_failed = True logging.getLogger(__name__).error( "Broker consumer turn failed; source authority and replay remain required" ) @@ -163,11 +180,17 @@ async def monitor() -> None: async def poll(self) -> None: while True: + self.health.started("kafka_loop") try: - await self.cycle() + healthy = await self.cycle() except Exception: logging.getLogger(__name__).error("Broker traversal failed; retrying bounded turn") - await asyncio.sleep(1) + healthy = False + if healthy: + self.health.succeeded("kafka_loop") + else: + self.health.failed("kafka_loop") + await asyncio.sleep(TRAVERSAL_SECONDS) async def close(self) -> None: if self.task: @@ -175,6 +198,7 @@ async def close(self) -> None: with suppress(asyncio.CancelledError): await self.task self.task = None + self.health.running("kafka_loop", False) turns = tuple(self.turns) for task in turns: task.cancel() diff --git a/src/firefly_weave/triggers/scheduler.py b/src/firefly_weave/triggers/scheduler.py index 78c5de3f..1ab3ccfd 100644 --- a/src/firefly_weave/triggers/scheduler.py +++ b/src/firefly_weave/triggers/scheduler.py @@ -31,6 +31,7 @@ from firefly_weave.contracts.access import Scope from firefly_weave.contracts.runtime import StartRunRequest from firefly_weave.definitions.models import CatalogError, capacity_rejected +from firefly_weave.operations.facts import RunStartFacts from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.repository import SCOPE, RuntimeRepository from firefly_weave.runtime.service import RuntimeService @@ -127,6 +128,7 @@ async def _fire(self, tx: Transaction, row: dict[str, Any]) -> int: occurrence_key(row["id"], row["revision"], instant), context=AuditContext(), tx=tx, + start_facts=RunStartFacts(origin="schedule"), not_after=instant + timedelta(seconds=60), ) await repository.execute( diff --git a/src/firefly_weave/triggers/service.py b/src/firefly_weave/triggers/service.py index d147c49c..aed2b4c4 100644 --- a/src/firefly_weave/triggers/service.py +++ b/src/firefly_weave/triggers/service.py @@ -37,6 +37,7 @@ from firefly_weave.contracts.access import Scope from firefly_weave.contracts.runtime import StartRunRequest from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.facts import RunStartFacts from firefly_weave.runtime.repository import SCOPE, RuntimeRepository from firefly_weave.runtime.service import RuntimeService from firefly_weave.runtime.signals import SignalService @@ -209,6 +210,7 @@ async def receive(self, trigger_id: UUID, raw_body: bytes, headers: dict[str, st f"webhook:{trigger.id}:{event}", context=context, tx=tx, + start_facts=RunStartFacts(origin="webhook"), ) run_id = run.id else: diff --git a/src/firefly_weave/workers/leases.py b/src/firefly_weave/workers/leases.py index 75591631..cbbd2cbb 100644 --- a/src/firefly_weave/workers/leases.py +++ b/src/firefly_weave/workers/leases.py @@ -51,6 +51,7 @@ UnavailableCompletionReceipt, ) from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.facts import FactRepository from firefly_weave.operations.redaction import Omission from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.repository import RuntimeRepository @@ -174,7 +175,8 @@ async def claim(self, tx: Transaction, worker_id: UUID, limit: int) -> list[Task capability=capability, policy=json.dumps({"side_effect": action["sideEffect"], "retry": action["retry"]}), ) - await self.runtime.set_task_status(enlisted, identifier, "leased") + await self.runtime.set_task_status(enlisted, identifier, "leased", at=now) + await FactRepository(enlisted).step_claimed(identifier, generation, worker_id, now) lease = TaskLease( proof=LeaseProof(task_id=identifier, generation=generation, owner=worker_id, token=token), input=payload["input"], @@ -577,6 +579,9 @@ async def recovery_candidates(self, tx: Transaction, limit: int) -> list[UUID]: rows = await WorkerRepository(tx).rows( f"SELECT task_id FROM task_leases WHERE {SCOPE} AND " "((status='active' AND expires_at<=clock_timestamp()) OR status='failed') " + "AND EXISTS (SELECT 1 FROM task_intents t WHERE t.id=task_leases.task_id " + "AND t.tenant_id=:tenant AND t.project_id=:project AND t.environment_id=:environment " + "AND t.status<>'handled') " "AND task_id NOT IN (SELECT t.id FROM task_intents t JOIN runtime_capacity_blocks b ON " "b.run_id=t.run_id WHERE b.active) " f"AND (SELECT {runnable('r')} FROM runs r WHERE r.id=" diff --git a/src/firefly_weave/workers/models.py b/src/firefly_weave/workers/models.py index 8a047778..9f3e9fe0 100644 --- a/src/firefly_weave/workers/models.py +++ b/src/firefly_weave/workers/models.py @@ -17,12 +17,15 @@ """Lease identities and shared safe failures.""" import hashlib +import json import secrets from dataclasses import dataclass from datetime import datetime, timedelta from typing import Any, Literal -from firefly_weave.contracts.workers import WorkerInstance, WorkerStatus +from pydantic import ValidationError + +from firefly_weave.contracts.workers import WorkerIdentity, WorkerInstance, WorkerPlacement, WorkerStatus from firefly_weave.definitions.models import CatalogError from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.kernel import checked_deadline @@ -44,6 +47,42 @@ def unavailable() -> CatalogError: return CatalogError(409, "WV-LEASE", "Lease or worker authority unavailable") +def stored_identity(value: object) -> WorkerIdentity | None: + """A worker's recorded identity; empty (never reported) or unreadable reads as unknown.""" + if not value: + return None + try: + return WorkerIdentity.model_validate_json(json.dumps(value)) + except ValidationError: + return None + + +def stored_placement(value: object) -> WorkerPlacement | None: + """A placement the fleet query derived; one that does not validate reads as unknown.""" + if not value: + return None + try: + return WorkerPlacement.model_validate_json(json.dumps(value)) + except ValidationError: + return None + + +def fleet_status(row: dict[str, Any], observed_at: datetime) -> WorkerStatus: + """A worker read from one fleet query row, without any per-worker query.""" + return observed_worker( + WorkerInstance.model_validate_json(json.dumps({**row["payload"], "revoked": row["revoked"]})), + last_seen_at=row["last_seen_at"], + draining=row["draining"], + revision=row["control_revision"], + active_leases=row["active_leases"], + observed_at=observed_at, + identity=stored_identity(row["identity"]), + registered_at=row["registered_at"], + placement=stored_placement(row["placement"]), + drained_at=row["drained_at"], + ) + + def observed_worker( instance: WorkerInstance, *, @@ -52,6 +91,10 @@ def observed_worker( revision: int, active_leases: int, observed_at: datetime, + identity: WorkerIdentity | None = None, + registered_at: datetime | None = None, + placement: WorkerPlacement | None = None, + drained_at: datetime | None = None, ) -> WorkerStatus: expires = last_seen_at + timedelta(seconds=60) if last_seen_at else None presence: Literal["unknown", "recent", "stale"] = ( @@ -62,6 +105,10 @@ def observed_worker( available = 0 return WorkerStatus( **instance.model_dump(), + identity=identity, + registered_at=registered_at, + placement=placement, + drained_at=drained_at, revision=revision, draining=draining, presence=presence, diff --git a/src/firefly_weave/workers/repository.py b/src/firefly_weave/workers/repository.py index 959d08c9..004fe50b 100644 --- a/src/firefly_weave/workers/repository.py +++ b/src/firefly_weave/workers/repository.py @@ -23,12 +23,72 @@ from sqlalchemy import text -from firefly_weave.contracts.workers import CompletionReceipt, WorkerInstance, WorkerRelease, WorkerStatus +from firefly_weave.contracts.worker_queries import RETIREMENT_BATCH, ReleaseUsagePosition, WorkerListQuery +from firefly_weave.contracts.workers import ( + CompletionReceipt, + WorkerIdentity, + WorkerInstance, + WorkerRelease, + WorkerStatus, +) from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.fact_positions import RunPosition +from firefly_weave.persistence.paging import PAGE_BYTES from firefly_weave.persistence.uow import Transaction -from firefly_weave.workers.models import observed_worker, unavailable +from firefly_weave.workers.models import observed_worker, stored_identity, unavailable SCOPE = "tenant_id=:tenant AND project_id=:project AND environment_id=:environment" +NOW = "cast(:now AS timestamptz)" +_W_SCOPE = "w.tenant_id=:tenant AND w.project_id=:project AND w.environment_id=:environment" +_FACTS = ( + "LEFT JOIN worker_facts f ON f.worker_id=w.id AND f.tenant_id=w.tenant_id AND f.project_id=w.project_id " + "AND f.environment_id=w.environment_id" +) +# A lease that still holds its worker: active, and neither expired nor past its deadline. +_ACTIVE_LEASE = ( + "SELECT 1 FROM task_leases l WHERE l.tenant_id=w.tenant_id AND l.project_id=w.project_id " + f"AND l.environment_id=w.environment_id AND l.owner=w.id AND l.status='active' AND l.expires_at>{NOW} " + f"AND l.deadline>{NOW}" +) +# A worker's last contact; one never seen counts from its recorded registration, and one without either has none. +_CONTACT = "coalesce(w.last_seen_at,f.registered_at)" +_RETIRABLE = ( + f"FROM worker_instances w {_FACTS} WHERE {_W_SCOPE} AND NOT w.revoked " + f"AND {_CONTACT}>'name' AS target_name " + "FROM deployment_observations o JOIN deployment_targets t ON t.id=o.target_id AND t.tenant_id=o.tenant_id " + "AND t.project_id=o.project_id AND t.environment_id=o.environment_id " + "WHERE o.tenant_id=:tenant AND o.project_id=:project AND o.environment_id=:environment " + "AND (:all_targets OR o.target_id=ANY(cast(:target_ids AS uuid[]))) " + "AND (o.payload->>'complete')::boolean AND (o.payload->>'target_revision')::integer=t.revision " + "ORDER BY o.target_id,(o.payload->>'observed_at')::timestamptz DESC,o.id DESC), " + "placements AS (SELECT DISTINCT member->>'host' AS host,o.target_id,o.target_name,d.id AS deployment_id," + "component->>'name' AS component FROM latest_complete o " + "CROSS JOIN LATERAL jsonb_array_elements(o.payload->'resources') resource " + "CROSS JOIN LATERAL jsonb_array_elements(coalesce(resource->'members','[]'::jsonb)) member " + "JOIN deployment_deployments d ON d.target_id=o.target_id AND d.tenant_id=:tenant AND d.project_id=:project " + "AND d.environment_id=:environment CROSS JOIN LATERAL jsonb_array_elements(d.payload->'components') component " + f"WHERE (o.payload->>'expires_at')::timestamptz>{NOW} AND resource->>'name'=component->>'name' " + "AND resource->>'kind'='worker' AND component->>'kind'='worker'), " + "unique_placement AS (SELECT host,(jsonb_agg(jsonb_build_object('target_id',target_id,'target_name'," + "target_name,'deployment_id',deployment_id,'component',component))->0) AS placement FROM placements " + "GROUP BY host HAVING count(*)=1)" +) +# Filters bind as typed nulls, so every filter combination is the same statement. +_FLEET_FILTERS = ( + f"(:all_workers OR (NOT revoked AND coalesce(last_seen_at,registered_at)>{NOW}-interval '1 hour')) " + "AND (cast(:presence AS text) IS NULL OR presence=cast(:presence AS text)) " + "AND (cast(:draining AS boolean) IS NULL OR draining=cast(:draining AS boolean)) " + "AND (cast(:release AS uuid) IS NULL OR release_id=cast(:release AS uuid)) " + "AND (cast(:task AS text) IS NULL OR payload->'task_types' @> jsonb_build_array(cast(:task AS text))) " + "AND (cast(:host_prefix AS text) IS NULL OR starts_with(identity->>'host',cast(:host_prefix AS text))) " + "AND (cast(:target AS uuid) IS NULL OR (placement->>'target_id')::uuid=cast(:target AS uuid))" +) class WorkerRepository: @@ -76,9 +136,226 @@ async def observe(self, identifier: UUID) -> None: id=identifier, ) + async def record_registration(self, identifier: UUID, identity: WorkerIdentity | None) -> None: + """Record a new instance's facts in its registration transaction, never in its payload.""" + await self.execute( + "INSERT INTO worker_facts(tenant_id,project_id,environment_id,worker_id,registered_at,identity) " + "VALUES(:tenant,:project,:environment,:id,clock_timestamp(),cast(:identity AS jsonb))", + id=identifier, + identity=json.dumps({} if identity is None else identity.model_dump(mode="json")), + ) + + async def record_drain(self, identifier: UUID, *, draining: bool, was_draining: bool) -> None: + """Record when a worker starts draining; a repeated drain keeps that time and resuming clears it.""" + if draining and not was_draining: + await self.execute( + f"UPDATE worker_facts SET drained_at=clock_timestamp() WHERE {SCOPE} AND worker_id=:id", id=identifier + ) + elif not draining: + await self.execute( + f"UPDATE worker_facts SET drained_at=NULL WHERE {SCOPE} AND worker_id=:id AND drained_at IS NOT NULL", + id=identifier, + ) + + async def fleet( + self, + query: WorkerListQuery, + position: RunPosition | None, + *, + observed_at: datetime, + authorized_targets: tuple[UUID, ...] | None, + worker_id: UUID | None = None, + ) -> list[dict[str, Any]]: + """One page of workers and its lookahead row, with grouped active leases and permitted placement. + + One statement serves every filter, order and position. `authorized_targets` is None for every target and + empty for none. A page above the logical byte limit is refused before its payloads are returned. + """ + column = "last_seen_at" if query.order == "last_seen_desc" else "registered_at" + keyset = "" + bounds: dict[str, Any] = {} + if position is not None: + bounds["position_id"] = position.id + if position.at is None: + keyset = f" AND {column} IS NULL AND id{NOW} AND deadline>{NOW}" + + (" AND owner=:worker" if one else "") + + " GROUP BY owner), " + + _PLACEMENT_CTES + + ", fleet AS (SELECT w.id,w.release_id,w.payload,w.revoked,w.draining,w.control_revision," + "w.last_seen_at,f.registered_at,f.identity,f.drained_at,coalesce(a.active_leases,0) AS active_leases," + f"CASE WHEN w.last_seen_at IS NULL THEN 'unknown' WHEN w.last_seen_at>{NOW}-interval '60 seconds' " + "THEN 'recent' ELSE 'stale' END AS presence,p.placement " + f"FROM worker_instances w {_FACTS} LEFT JOIN active a ON a.owner=w.id " + "LEFT JOIN unique_placement p ON p.host=f.identity->>'host' AND f.identity->>'runtime'<>'embedded' " + f"WHERE {_W_SCOPE}" + (" AND w.id=:worker" if one else "") + "), " + f"page AS (SELECT fleet.*,row_number() OVER (ORDER BY {order}) AS page_row FROM fleet " + f"WHERE {_FLEET_FILTERS}{keyset} ORDER BY {order} LIMIT :page_size), " + "measured AS (SELECT page.*,sum(CASE WHEN page_row<=:limit THEN octet_length(payload::text)" + "+octet_length(coalesce(identity,'{}'::jsonb)::text) ELSE 0 END) OVER () AS page_bytes FROM page) " + "SELECT id,release_id,revoked,draining,control_revision,last_seen_at,registered_at,drained_at," + "active_leases,presence,placement,CASE WHEN page_bytes<=:budget THEN payload END AS payload," + "CASE WHEN page_bytes<=:budget THEN identity END AS identity,page_bytes>:budget AS oversized " + "FROM measured ORDER BY page_row" + ) + rows = await self.rows( + sql, + now=observed_at, + worker=worker_id, + all_targets=authorized_targets is None, + target_ids=list(authorized_targets or ()), + all_workers=query.state == "all", + presence=query.presence, + draining=query.draining, + release=query.release_id, + task=query.task_type, + host_prefix=query.host_prefix, + target=query.target_id, + page_size=query.limit + 1, + limit=query.limit, + budget=PAGE_BYTES, + **bounds, + ) + if any(row["oversized"] for row in rows): + raise CatalogError(429, "WV-PAGE-LIMIT", "Selected page exceeds the logical byte limit") + return rows + + async def retirable(self, cutoff: datetime, now: datetime) -> int: + """Every worker a retirement at this cutoff would revoke, across all batches.""" + rows = await self.rows(f"SELECT count(*) AS count {_RETIRABLE}", cutoff=cutoff, now=now) + return int(rows[0]["count"]) + + async def retirement_batch(self, cutoff: datetime, now: datetime, *, lock: bool) -> list[UUID]: + """The oldest retirable workers, oldest last contact first; locking skips rows another command holds.""" + rows = await self.rows( + f"SELECT w.id {_RETIRABLE} ORDER BY {_CONTACT} ASC,w.id ASC LIMIT :batch" + + (" FOR UPDATE OF w SKIP LOCKED" if lock else ""), + cutoff=cutoff, + now=now, + batch=RETIREMENT_BATCH, + ) + return [row["id"] for row in rows] + + async def retire(self, identifiers: list[UUID], cutoff: datetime, now: datetime) -> list[UUID]: + """Revoke the locked workers that still qualify; the conditions are checked again on the locked rows.""" + if not identifiers: + return [] + rows = await self.rows( + "UPDATE worker_instances w SET revoked=true,control_revision=w.control_revision+1 " + f"WHERE {_W_SCOPE} AND w.id=ANY(cast(:ids AS uuid[])) AND NOT w.revoked " + "AND coalesce(w.last_seen_at,(SELECT f.registered_at FROM worker_facts f WHERE f.worker_id=w.id " + "AND f.tenant_id=w.tenant_id AND f.project_id=w.project_id AND f.environment_id=w.environment_id))" + f" str: + rows = await self.rows(f"SELECT image_digest FROM worker_releases WHERE {SCOPE} AND id=:id", id=identifier) + if not rows: + raise CatalogError(404, "WV-NOT-FOUND", "Worker resource not found") + return str(rows[0]["image_digest"]) + + async def release_workers(self, identifier: UUID, now: datetime) -> dict[str, int]: + rows = await self.rows( + "SELECT count(*) FILTER (WHERE revoked) AS revoked," + "count(*) FILTER (WHERE NOT revoked AND draining) AS draining," + f"count(*) FILTER (WHERE NOT revoked AND NOT draining AND last_seen_at>{NOW}-interval '60 seconds') " + "AS online,count(*) FILTER (WHERE NOT revoked AND NOT draining AND (last_seen_at IS NULL " + f"OR last_seen_at<={NOW}-interval '60 seconds')) AS offline " + f"FROM worker_instances WHERE {SCOPE} AND release_id=:release", + release=identifier, + now=now, + ) + return {key: int(value) for key, value in rows[0].items()} + + async def release_activations(self, identifier: UUID) -> int: + """Activation revisions pinning the release that can still start runs. + + A revision counts unless it, the latest revision of the same activation, or its workflow version is + retired, as the start check decides. + """ + pinned = ( + "EXISTS(SELECT 1 FROM jsonb_each_text(coalesce(a.payload->'request'->'{key}','{{}}'::jsonb)) pin " + "WHERE pin.value=:release)" + ) + rows = await self.rows( + "SELECT count(*) AS count FROM activation_revisions a WHERE a.tenant_id=:tenant AND a.project_id=:project " + f"AND a.environment_id=:environment AND ({pinned.format(key='worker_release_ids')} " + f"OR {pinned.format(key='connector_release_ids')}) " + "AND NOT coalesce((a.payload->>'retired')::boolean,false) " + "AND NOT coalesce((SELECT (l.payload->>'retired')::boolean FROM activation_revisions l " + "WHERE l.tenant_id=a.tenant_id AND l.project_id=a.project_id AND l.environment_id=a.environment_id " + "AND l.name=a.name ORDER BY l.revision DESC LIMIT 1),false) " + "AND NOT EXISTS(SELECT 1 FROM definition_retirements r WHERE r.tenant_id=a.tenant_id " + "AND r.project_id=a.project_id AND r.version_id=a.version_id)", + release=str(identifier), + ) + return int(rows[0]["count"]) + + async def release_deployments( + self, + identifier: UUID, + digest: str, + *, + authorized_targets: tuple[UUID, ...] | None, + position: ReleaseUsagePosition | None, + limit: int, + ) -> list[dict[str, Any]]: + """Deployment components that run exactly this admitted release, in deployment and component order.""" + keyset = "" + bounds: dict[str, Any] = {} + if position is not None: + keyset = ( + " AND (d.id,(component->>'name') COLLATE \"C\")>" + '(cast(:deployment AS uuid),cast(:component AS text) COLLATE "C")' + ) + bounds = dict(deployment=position.deployment_id, component=position.component) + return await self.rows( + "SELECT d.target_id,t.payload->>'name' AS target_name,d.id AS deployment_id," + "d.payload->>'name' AS deployment_name,component->>'name' AS component," + "(component->>'replicas')::integer AS replicas FROM deployment_deployments d " + "JOIN deployment_targets t ON t.id=d.target_id AND t.tenant_id=d.tenant_id AND t.project_id=d.project_id " + "AND t.environment_id=d.environment_id " + "CROSS JOIN LATERAL jsonb_array_elements(d.payload->'components') component " + "WHERE d.tenant_id=:tenant AND d.project_id=:project AND d.environment_id=:environment " + "AND (:all_targets OR d.target_id=ANY(cast(:target_ids AS uuid[]))) " + "AND component->>'worker_release_id'=:release AND split_part(component->>'image','@',2)=:digest" + + keyset + + " ORDER BY d.id,(component->>'name') COLLATE \"C\" LIMIT :limit", + release=str(identifier), + digest=digest, + all_targets=authorized_targets is None, + target_ids=list(authorized_targets or ()), + limit=limit + 1, + **bounds, + ) + async def status(self, identifier: UUID, observed_at: datetime | None = None) -> WorkerStatus: now = observed_at or await self.now() - rows = await self.rows(f"SELECT * FROM worker_instances WHERE {SCOPE} AND id=:id", id=identifier) + rows = await self.rows( + "SELECT w.payload,w.revoked,w.draining,w.control_revision,w.last_seen_at," + "f.identity,f.registered_at,f.drained_at FROM worker_instances w LEFT JOIN worker_facts f " + "ON f.worker_id=w.id AND f.tenant_id=w.tenant_id AND f.project_id=w.project_id " + "AND f.environment_id=w.environment_id WHERE w.tenant_id=:tenant AND w.project_id=:project " + "AND w.environment_id=:environment AND w.id=:id", + id=identifier, + ) if not rows: raise CatalogError(404, "WV-NOT-FOUND", "Worker resource not found") row = rows[0] @@ -95,6 +372,9 @@ async def status(self, identifier: UUID, observed_at: datetime | None = None) -> revision=row["control_revision"], active_leases=active[0]["count"], observed_at=now, + identity=stored_identity(row["identity"]), + registered_at=row["registered_at"], + drained_at=row["drained_at"], ) async def lease(self, task: UUID, generation: int) -> dict[str, Any]: diff --git a/src/firefly_weave/workers/service.py b/src/firefly_weave/workers/service.py index 1ba36139..9daeb8a8 100644 --- a/src/firefly_weave/workers/service.py +++ b/src/firefly_weave/workers/service.py @@ -17,7 +17,7 @@ """Immutable release admission and scoped worker identities.""" import json -from datetime import datetime +from datetime import datetime, timedelta from typing import Any from uuid import UUID, uuid4 @@ -28,12 +28,23 @@ from firefly_weave.access.models import Principal from firefly_weave.access.repository import load_principal from firefly_weave.access.service import audit +from firefly_weave.api.transport import decode_cursor_v2, encode_cursor_v2 from firefly_weave.compiler.api import CompiledArtifact from firefly_weave.compiler.catalog import CatalogSnapshot from firefly_weave.compiler.expressions import measure_value from firefly_weave.compiler.ir import executable_node_kinds from firefly_weave.contracts.access import Scope from firefly_weave.contracts.catalog import ActivationRequest +from firefly_weave.contracts.worker_queries import ( + ReleaseDeploymentUsage, + ReleaseUsage, + ReleaseUsageQuery, + ReleaseWorkerCounts, + RetireStaleRequest, + RetireStaleResult, + WorkerListQuery, + release_usage_position, +) from firefly_weave.contracts.workers import ( ConnectorExecutionPin, CredentialGrantRequest, @@ -48,8 +59,11 @@ from firefly_weave.definitions.ports import ConnectionBindingPort, WorkerAdmissionPort from firefly_weave.definitions.repository import DefinitionRepository from firefly_weave.definitions.service import DefinitionService +from firefly_weave.operations.fact_positions import RunPosition +from firefly_weave.operations.permissions import target_permissions from firefly_weave.persistence.idempotency import Idempotency, lock from firefly_weave.persistence.uow import Transaction +from firefly_weave.workers.models import fleet_status from firefly_weave.workers.repository import SCOPE, WorkerRepository @@ -73,6 +87,28 @@ async def require( for current in (actor, await load_principal(tx.session, actor.id)): self.definitions.authorization.require(current, scope, capability, resource=resource, context=context) + def precheck( + self, actor: Principal, scope: Scope, capability: str, context: AuditContext, resource: str | None = None + ) -> None: + """`capability` on the token principal, before any unit of work opens.""" + if scope.environment_id is None: + raise AccessDenied() + self.definitions.authorization.require(actor, scope, capability, resource=resource, context=context) + + async def current( + self, + actor: Principal, + scope: Scope, + capability: str, + context: AuditContext, + tx: Transaction, + resource: str | None = None, + ) -> Principal: + """`capability` on the principal stored now, inside the caller's unit of work; `precheck` runs first.""" + current = await load_principal(tx.session, actor.id) + self.definitions.authorization.require(current, scope, capability, resource=resource, context=context) + return current + async def capabilities(self, tx: Transaction, base: CatalogSnapshot) -> CatalogSnapshot: rows = await WorkerRepository(tx).rows( "SELECT payload FROM worker_releases WHERE tenant_id=:tenant AND project_id=:project" @@ -160,7 +196,8 @@ async def register_instance( raise AccessDenied() for capability in request.task_types: await self.require(actor, scope, "worker.register", context, tx, capability) - instance = WorkerInstance(id=uuid4(), principal_id=actor.id, **request.model_dump()) + # The stored instance and the registration answer keep their bytes; identity is a fact. + instance = WorkerInstance(id=uuid4(), principal_id=actor.id, **request.model_dump(exclude={"identity"})) await repository.execute( "INSERT INTO worker_instances " "(id,tenant_id,project_id,environment_id,release_id,principal_id,payload,revoked) " @@ -171,6 +208,7 @@ async def register_instance( principal=actor.id, payload=instance.model_dump_json(), ) + await repository.record_registration(instance.id, request.identity) await audit( tx.session, actor, @@ -228,7 +266,7 @@ async def control( return WorkerStatus.model_validate_json(json.dumps(prior)) repository = WorkerRepository(enlisted) rows = await repository.rows( - f"SELECT revoked,control_revision FROM worker_instances WHERE {SCOPE} AND id=:id FOR UPDATE", + f"SELECT revoked,draining,control_revision FROM worker_instances WHERE {SCOPE} AND id=:id FOR UPDATE", id=identifier, ) if not rows: @@ -243,6 +281,7 @@ async def control( id=identifier, draining=draining, ) + await repository.record_drain(identifier, draining=draining, was_draining=rows[0]["draining"]) result = await repository.status(identifier) await audit( enlisted.session, @@ -399,6 +438,113 @@ async def grant_connection( context=context, ) + async def retire_stale( + self, + actor: Principal, + scope: Scope, + request: RetireStaleRequest, + idempotency_key: str, + *, + context: AuditContext, + tx: Transaction | None = None, + ) -> RetireStaleResult: + """Revoke at most one batch of workers without an active lease whose last contact is older than the threshold. + + A dry run selects the same oldest-first batch without revoking it. Both save their answer for replay under + the same Idempotency-Key; a worker that was never seen counts from its registration, and one whose + registration time is unknown is never retired. + """ + self.precheck(actor, scope, "release.retire", context) + async with self.definitions.transaction(scope, tx) as enlisted: + await self.current(actor, scope, "release.retire", context, enlisted) + replay = Idempotency( + enlisted, + actor.id, + f"workers.retire_stale:{scope.environment_id}", + idempotency_key, + request.model_dump(mode="json"), + ) + if prior := await replay.replay(): + return RetireStaleResult.model_validate_json(json.dumps(prior)) + repository = WorkerRepository(enlisted) + now = await repository.now() + cutoff = now - timedelta(seconds=request.older_than_seconds) + matched = await repository.retirable(cutoff, now) + if request.dry_run: + result = RetireStaleResult( + matched=matched, + revoked=0, + worker_ids=await repository.retirement_batch(cutoff, now, lock=False), + ) + else: + locked = await repository.retirement_batch(cutoff, now, lock=True) + revoked = await repository.retire(locked, cutoff, now) + result = RetireStaleResult(matched=matched, revoked=len(revoked), worker_ids=revoked) + await audit( + enlisted.session, + actor, + "worker.retire_stale", + str(scope.environment_id), + scope=scope, + capability="release.retire", + context=context, + details={ + "dry_run": request.dry_run, + "older_than_seconds": request.older_than_seconds, + **result.model_dump(mode="json"), + }, + ) + await replay.save(result.model_dump(mode="json")) + return result + + async def release_usage( + self, + actor: Principal, + scope: Scope, + release_id: UUID, + query: ReleaseUsageQuery, + *, + context: AuditContext, + ) -> ReleaseUsage: + """Worker counts, pinning activations and a page of the readable deployments that run an admitted release. + + The cursor is bound to the release and to the targets this reader may see, which only the read unit knows. + """ + self.precheck(actor, scope, "catalog.read", context) + async with self.definitions.transaction(scope, None, mutation=False) as tx: + current = await self.current(actor, scope, "catalog.read", context, tx) + repository = WorkerRepository(tx) + digest = await repository.release_digest(release_id) + targets = target_permissions( + self.definitions.authorization, actor, current, scope, "deployment.read", context + ) + collection = query.cursor_collection(release_id, targets) + position = release_usage_position(decode_cursor_v2(query.cursor, scope, collection)) + now = await repository.now() + workers = ReleaseWorkerCounts(**await repository.release_workers(release_id, now)) + activations = await repository.release_activations(release_id) + rows = ( + [] + if targets == () + else await repository.release_deployments( + release_id, digest, authorized_targets=targets, position=position, limit=query.limit + ) + ) + visible = rows[: query.limit] + cursor = None + if len(rows) > query.limit: + last = visible[-1] + cursor = encode_cursor_v2(scope, collection, last["component"], str(last["deployment_id"])) + return ReleaseUsage( + workers=workers, + deployments=[ + ReleaseDeploymentUsage.model_validate_json(json.dumps(row, default=str)) for row in visible + ], + activations=activations, + next_cursor=cursor, + deployments_hidden=targets == (), + ) + async def list( self, actor: Principal, @@ -407,25 +553,73 @@ async def list( releases: bool = False, limit: int = 50, cursor: UUID | None = None, + query: WorkerListQuery | None = None, + position: RunPosition | None = None, context: AuditContext, ) -> dict[str, Any]: + """Releases in ID order, or one page of workers from a single fleet query. + + Workers take a typed query and the decoded position of its cursor; the keyword `limit` selects the first page + of current workers. + """ + if not releases: + if query is None: + if cursor is not None: + raise ValueError("Continue a worker list with its query and position") + query = WorkerListQuery(limit=limit) + elif cursor is not None or limit != 50: + raise ValueError("Use either a query or legacy list arguments") + return await self._fleet(actor, scope, query, position, context=context) + if query is not None or position is not None: + raise ValueError("Release lists take no worker query") from firefly_weave.persistence.paging import page_ids - capability = "catalog.read" if releases else "status.read" async with self.definitions.transaction(scope, None, mutation=False) as tx: - await self.require(actor, scope, capability, context, tx) - ids = await page_ids(tx, "worker_releases" if releases else "worker_instances", limit, cursor) - observed_at = await WorkerRepository(tx).now() + await self.require(actor, scope, "catalog.read", context, tx) + ids = await page_ids(tx, "worker_releases", limit, cursor) items = [ - ( - await self.read( - actor, scope, identifier, releases=releases, context=context, tx=tx, observed_at=observed_at - ) - ).model_dump(mode="json", by_alias=True) + (await self.read(actor, scope, identifier, releases=True, context=context, tx=tx)).model_dump( + mode="json", by_alias=True + ) for identifier in ids[:limit] ] return {"items": items, "next_cursor": str(ids[limit - 1]) if len(ids) > limit else None} + async def _fleet( + self, + actor: Principal, + scope: Scope, + query: WorkerListQuery, + position: RunPosition | None, + *, + context: AuditContext, + ) -> dict[str, Any]: + authorization = self.definitions.authorization + target = None if query.target_id is None else str(query.target_id) + self.precheck(actor, scope, "status.read", context) + if target is not None: + authorization.require(actor, scope, "deployment.read", resource=target, context=context) + async with self.definitions.transaction(scope, None, mutation=False) as tx: + current = await self.current(actor, scope, "status.read", context, tx) + if target is not None: + authorization.require(current, scope, "deployment.read", resource=target, context=context) + targets = target_permissions(authorization, actor, current, scope, "deployment.read", context) + repository = WorkerRepository(tx) + observed_at = await repository.now() + rows = await repository.fleet(query, position, observed_at=observed_at, authorized_targets=targets) + visible = rows[: query.limit] + next_cursor = None + if len(rows) > query.limit: + last = visible[-1] + at = last["last_seen_at"] if query.order == "last_seen_desc" else last["registered_at"] + next_cursor = encode_cursor_v2( + scope, query.cursor_collection(), at.isoformat() if at else None, str(last["id"]) + ) + return { + "items": [fleet_status(row, observed_at).model_dump(mode="json") for row in visible], + "next_cursor": next_cursor, + } + async def read( self, actor: Principal, @@ -437,21 +631,43 @@ async def read( tx: Transaction | None = None, observed_at: datetime | None = None, ) -> WorkerStatus | WorkerRelease: + if not releases: + return await self._worker(actor, scope, identifier, context=context, tx=tx, observed_at=observed_at) async with self.definitions.transaction(scope, tx, mutation=False) as enlisted: - await self.require( - actor, - scope, - "catalog.read" if releases else "status.read", - context, - enlisted, - None if releases else str(identifier), - ) - table = "worker_releases" if releases else "worker_instances" + await self.require(actor, scope, "catalog.read", context, enlisted) rows = await WorkerRepository(enlisted).rows( - f"SELECT * FROM {table} WHERE {SCOPE} AND id=:id", id=identifier + f"SELECT payload FROM worker_releases WHERE {SCOPE} AND id=:id", id=identifier + ) + if not rows: + raise CatalogError(404, "WV-NOT-FOUND", "Worker resource not found") + return WorkerRelease.model_validate_json(json.dumps(rows[0]["payload"])) + + async def _worker( + self, + actor: Principal, + scope: Scope, + identifier: UUID, + *, + context: AuditContext, + tx: Transaction | None, + observed_at: datetime | None, + ) -> WorkerStatus: + """One worker through the fleet statement, so a read shows what a list shows, placement included.""" + self.precheck(actor, scope, "status.read", context, str(identifier)) + async with self.definitions.transaction(scope, tx, mutation=False) as enlisted: + current = await self.current(actor, scope, "status.read", context, enlisted, str(identifier)) + targets = target_permissions( + self.definitions.authorization, actor, current, scope, "deployment.read", context + ) + repository = WorkerRepository(enlisted) + now = observed_at or await repository.now() + rows = await repository.fleet( + WorkerListQuery(state="all", limit=1), + None, + observed_at=now, + authorized_targets=targets, + worker_id=identifier, ) if not rows: raise CatalogError(404, "WV-NOT-FOUND", "Worker resource not found") - if releases: - return WorkerRelease.model_validate_json(json.dumps(rows[0]["payload"])) - return await WorkerRepository(enlisted).status(identifier, observed_at) + return fleet_status(rows[0], now) diff --git a/studio/src/app/app.html b/studio/src/app/app.html index e4790607..913617f8 100644 --- a/studio/src/app/app.html +++ b/studio/src/app/app.html @@ -429,6 +429,18 @@

Your workflows, woven together.

} + } @else if (view === "overview") { +
+ @defer (on immediate) { + + } +
+ } @else if (view === "runs") { +
+ @defer (on immediate) { + + } +
} @else if (view === "incidents") {
@defer (on immediate) { diff --git a/studio/src/app/app.ts b/studio/src/app/app.ts index 56f13069..3fb6c96e 100644 --- a/studio/src/app/app.ts +++ b/studio/src/app/app.ts @@ -203,11 +203,14 @@ import { import { ClustersPage } from "./operate/clusters/clusters-page"; import { WorkersPage } from "./operate/workers/workers-page"; import { IncidentsPage } from "./operate/incidents/incidents-page"; +import { OverviewPage } from "./operate/overview/overview-page"; +import { RunsPage } from "./operate/runs/runs-page"; import { navEntryVisible, viewFromPath, viewPath, } from "./operate/operate-routes"; +import { scopeKeyOf } from "./operate/operate-list"; import { RecordsView } from "./operations/records-view"; import { SettingsPage } from "./settings/settings-page"; import { LumiPanel } from "./lumi/lumi-panel"; @@ -232,6 +235,7 @@ export type View = | "home" | "workflows" | "designer" + | "overview" | "runs" | "incidents" | "tasks" @@ -395,6 +399,8 @@ const sideEffects: Record = { ClustersPage, WorkersPage, IncidentsPage, + OverviewPage, + RunsPage, SettingsPage, LumiPanel, ], @@ -446,6 +452,27 @@ export class App implements CanvasHost { platformBusy = ""; // Fences platform checks and status reads against later platform changes. private platformGeneration = 0; + /** Checks of who is signed in that have not answered yet. */ + private identityChecks?: Set>; + /** + * True while Studio is still finding out who is signed in: until it + * answers, the person a command would be sent for is not known. + */ + get checkingIdentity() { + return !!this.identityChecks?.size; + } + /** Resolves once no check of who is signed in is still out. */ + async identitySettled() { + while (this.identityChecks?.size) + await Promise.allSettled([...this.identityChecks]); + } + private trackIdentity(check: Promise): Promise { + const checks = (this.identityChecks ??= new Set()); + checks.add(check); + const done = () => checks.delete(check); + check.then(done, done); + return check; + } private stepDataGeneration = 0; private stepDataPrincipal = ""; /** Samples follow account invalidation, while routine same-account checks retain them. */ @@ -459,8 +486,9 @@ export class App implements CanvasHost { emailSubmission: Record | null = null; runHistory: Record | null = null; runCanvas: StructuredCanvasAdapter | null = null; + /** The published version `runCanvas` draws, exported once per open run. */ + private runCanvasVersion = ""; runLifecycle: Record | null = null; - runIncludeArchived = false; debugSession: Record | null = null; pendingMutation: { path: string; @@ -669,6 +697,7 @@ export class App implements CanvasHost { { id: "connections", label: "Connections" }, { id: "tasks", label: "My tasks" }, { id: "email", label: "Email" }, + { id: "overview", label: "Overview" }, { id: "runs", label: "Runs" }, { id: "incidents", label: "Incidents" }, { id: "workers", label: "Workers" }, @@ -679,7 +708,7 @@ export class App implements CanvasHost { ["", ["home"]], ["Build", ["workflows", "connections"]], ["Work", ["tasks", "email"]], - ["Operate", ["runs", "incidents", "workers", "clusters"]], + ["Operate", ["overview", "runs", "incidents", "workers", "clusters"]], ]; private navCache: { key: string; @@ -791,8 +820,14 @@ export class App implements CanvasHost { records: Record[] = []; nextCursor: string | null = null; selectedRecord: Record | null = null; - /** The Runs list was refused (403): the page names the access it needs. */ - runsRefused = false; + /** + * The Runs page while it is open. A command taken there (Start run, Cancel, + * Retry, Archive, Restore, Delete data) reads its list again through it. + */ + runsList: { + refresh(): Promise; + started(run: Record, scope: string): void; + } | null = null; /** The run whose read was refused (403); it stays until another opens. */ runRefused = ""; loading = false; @@ -946,7 +981,6 @@ export class App implements CanvasHost { commandKey: string | null = null; unknownCommand = false; taskFilter = ""; - runFilters = { business_key: "", correlation_key: "", status: "" }; showStartRun = false; /** The activation the start-run dialog preselects. */ runStartActivation = ""; @@ -1629,7 +1663,6 @@ export class App implements CanvasHost { this.records = []; this.nextCursor = null; this.taskRunFilter = ""; - this.runsRefused = false; this.runRefused = ""; } this.view = view; @@ -6221,11 +6254,18 @@ export class App implements CanvasHost { } if ( this.view === "connect" || + this.view === "overview" || this.view === "clusters" || this.view === "workers" || this.view === "incidents" ) return; + if (this.view === "runs") { + // Operate › Runs reads its own list; run titles and Retry run still + // read the published versions. + void this.loadWorkflowVersions(); + return; + } if (this.view === "home") { await this.refreshHome(); return; @@ -6248,7 +6288,6 @@ export class App implements CanvasHost { const requestedView = this.view; const collections: Partial> = { workflows: [this.libraryCollection, false], - runs: ["runs", true], connections: ["connections", true], tasks: ["human-tasks", true], email: ["email/conversations", true], @@ -6258,7 +6297,6 @@ export class App implements CanvasHost { this.loading = false; return; } - if (this.view === "runs") void this.loadWorkflowVersions(); if (this.view === "workflows" && this.libraryCollection === "workflows") void this.loadActivations(sequence); try { @@ -6266,32 +6304,18 @@ export class App implements CanvasHost { c[0], c[1], append ? (this.nextCursor ?? undefined) : undefined, - this.view === "runs" - ? { - ...this.runFilters, - include_archived: this.runIncludeArchived ? "true" : "", - } - : this.view === "tasks" && this.taskFilter - ? { status: this.taskFilter } - : {}, + this.view === "tasks" && this.taskFilter + ? { status: this.taskFilter } + : {}, ); if (sequence !== this.listSequence || requestedView !== this.view) return; this.records = append ? [...this.records, ...result.items] : result.items; this.nextCursor = result.next_cursor; - if (this.view === "runs") { - this.rememberRuns(result.items); - this.runsRefused = false; - } if (this.view === "workflows" && this.libraryCollection === "workflows") for (const item of result.items) this.rememberVersion(item); this.error = ""; } catch (e) { - // A refused Runs list names the access it needs on the page, instead - // of a banner over an empty list that claims there are no runs. - if (requestedView === "runs" && describeError(e).status === 403) { - if (sequence === this.listSequence && requestedView === this.view) - this.runsRefused = true; - } else this.fail(e); + this.fail(e); } finally { this.cdr.markForCheck(); if (sequence === this.listSequence) this.loading = false; @@ -6433,6 +6457,39 @@ export class App implements CanvasHost { /** Tasks filtered to one run ("Open My tasks" from a run). */ taskRunFilter = ""; private runTaskSequence = 0; + private runFollow = 0; + /** + * The open run changed while the Runs page follows it: Studio shows its new + * view and reads its lifecycle and history again. The drawing of its + * version stays; it never changes during a run. + */ + async followRun(view: Record) { + const id = view["id"]; + if (this.view !== "runs" || this.selectedRecord?.["id"] !== id) return; + const follow = ++this.runFollow; + this.selectedRecord = view; + this.cdr.markForCheck(); + const base = `${this.api.environment}/runs/${encodeURIComponent(String(id))}`; + try { + const [lifecycle, history] = await Promise.all([ + this.api.request>(`${base}/lifecycle`), + this.api.request>(`${base}/history?limit=100`), + ]); + if ( + follow !== this.runFollow || + this.view !== "runs" || + this.selectedRecord?.["id"] !== id + ) + return; + this.runLifecycle = lifecycle; + this.runHistory = history; + void this.findRunTask(this.selectedRecord); + } catch { + // The run's next read, 2 s later, tries again. + } finally { + this.cdr.markForCheck(); + } + } private refreshRunTaskAfterIdentity() { if (this.view === "runs" && this.selectedRecord) void this.findRunTask(this.selectedRecord); @@ -6535,8 +6592,11 @@ export class App implements CanvasHost { const activation = detail["activation"] as | { request?: { version_id?: string } } | undefined; - if (activation?.request?.version_id) { - const versionId = activation.request.version_id; + const versionId = activation?.request?.version_id; + if ( + versionId && + !(this.runCanvas && this.runCanvasVersion === versionId) + ) { const exported = await this.api.request<{ document: Workflow }>( `${this.api.project}/workflows/${versionId}/export`, ); @@ -6544,6 +6604,7 @@ export class App implements CanvasHost { canvas.setSource(JSON.stringify(exported.document), "json"); canvas.readonly = true; this.runCanvas = canvas; + this.runCanvasVersion = versionId; const metadata = exported.document?.metadata; if (metadata?.name) this.workflowVersions.set(versionId, { @@ -6860,9 +6921,20 @@ export class App implements CanvasHost { this.cdr.markForCheck(); } } + /** + * After a command on the Runs page: the published versions and the list's + * first page again, so the list shows what the command changed. + */ + async refreshRuns() { + if (this.view !== "runs") return; + void this.loadWorkflowVersions(); + await this.runsList?.refresh(); + } /** Starts the run; failures stay in the dialog, success opens the new run. */ async confirmStartRun(body: StartRunRequest) { this.runStartError = null; + // The environment and person the run starts for: the list shows it only there. + const scope = scopeKeyOf(this); try { const run = await this.safeMutation>( `${this.api.environment}/runs`, @@ -6877,10 +6949,9 @@ export class App implements CanvasHost { run: () => void this.viewRun(run), }); if (this.view === "runs") { - await this.refresh(); - // The list may not include a run this new yet; show it first. - if (run["id"] && !this.records.some((r) => r["id"] === run["id"])) - this.records = [run, ...this.records]; + // The list may not include a run this new yet; it shows it first. + this.runsList?.started(run, scope); + await this.refreshRuns(); } } catch (e) { const plain = describeError(e); @@ -6922,54 +6993,7 @@ export class App implements CanvasHost { async viewRun(run: Record) { if (this.view !== "runs") await this.navigate("runs"); if (this.view !== "runs" || !run["id"]) return; - const listed = this.records.find((r) => r["id"] === run["id"]); - if (!listed) this.records = [run, ...this.records]; - await this.open(listed ?? run); - // A run that could not be read leaves no placeholder row behind. - if (!listed && this.runRefused === run["id"]) - this.records = this.records.filter((r) => r !== run); - } - applyRunFilters() { - if (this.runFilterTimer) clearTimeout(this.runFilterTimer); - this.runFilterTimer = null; - this.nextCursor = null; - this.records = []; - this.closeRecord(); - void this.refresh(); - } - private runFilterTimer: ReturnType | null = null; - /** - * Runs filters apply as they change: the typed keys after a 400 ms pause, - * the status and "Include archived runs" at once. - */ - setRunFilter( - key: "business_key" | "correlation_key" | "status", - value: string, - ) { - if (this.runFilters[key] === value) return; - this.runFilters = { ...this.runFilters, [key]: value }; - if (key === "status") return this.applyRunFilters(); - if (this.runFilterTimer) clearTimeout(this.runFilterTimer); - this.runFilterTimer = setTimeout(() => this.applyRunFilters(), 400); - } - toggleArchivedRuns() { - this.runIncludeArchived = !this.runIncludeArchived; - this.applyRunFilters(); - } - /** How many Runs filters are set ("Filters (2)" on narrow windows). */ - get runFilterCount() { - return ( - [ - this.runFilters.business_key, - this.runFilters.correlation_key, - this.runFilters.status, - ].filter(Boolean).length + (this.runIncludeArchived ? 1 : 0) - ); - } - clearRunFilters() { - this.runIncludeArchived = false; - this.runFilters = { business_key: "", correlation_key: "", status: "" }; - this.applyRunFilters(); + await this.open(run); } applyTaskFilter(event: Event) { this.taskFilter = this.value(event); @@ -7577,7 +7601,10 @@ export class App implements CanvasHost { pretty(value: unknown) { return JSON.stringify(value, null, 2); } - async loadIdentity() { + loadIdentity() { + return this.trackIdentity(this.readIdentity()); + } + private async readIdentity() { if (!this.profile) return; const sampleScope = this.stepDataScope; try { @@ -7671,7 +7698,6 @@ export class App implements CanvasHost { this.showActivation = false; this.exportFallback = null; this.records = []; - this.runsRefused = false; this.runRefused = ""; this.closeRecord(); this.emailDetail = null; @@ -7692,7 +7718,10 @@ export class App implements CanvasHost { * Reads the host's platform connection and checks the active platform * silently. Failures become a non-blocking notice; local work never waits. */ - async verifyPlatform(startup = false) { + verifyPlatform(startup = false) { + return this.trackIdentity(this.readPlatform(startup)); + } + private async readPlatform(startup: boolean) { const generation = ++this.platformGeneration; const status = await this.connection.status().catch(() => null); if (generation !== this.platformGeneration || !this.paired) return; @@ -7717,7 +7746,10 @@ export class App implements CanvasHost { } await this.checkPlatform(generation); } - private async checkPlatform(generation = ++this.platformGeneration) { + private checkPlatform(generation = ++this.platformGeneration) { + return this.trackIdentity(this.testPlatform(generation)); + } + private async testPlatform(generation: number) { const sampleScope = this.stepDataScope; const name = this.profile?.name ?? ""; try { @@ -8976,7 +9008,7 @@ export class App implements CanvasHost { ? "Run archived." : "Run restored.", ); - await this.refresh(); + await this.refreshRuns(); } catch (e) { this.fail(e); } finally { diff --git a/studio/src/app/operate/incidents/incident-drawer.ts b/studio/src/app/operate/incidents/incident-drawer.ts index 7b43278d..ec8ede31 100644 --- a/studio/src/app/operate/incidents/incident-drawer.ts +++ b/studio/src/app/operate/incidents/incident-drawer.ts @@ -15,19 +15,23 @@ limitations under the License. Author: Firefly Software Foundation SPDX-License-Identifier: Apache-2.0 */ -// One incident: what happened, the run's last engine events (types only), -// how it was resolved, and Resolve incident while it is active. +// One incident: what happened (its step and full step instance, the run and +// its status, when it opened, marked when that time is an estimate), the +// run's last engine events (types only, never their data), how it was +// resolved, and Resolve incident while it is active. A time the platform +// doesn't know reads Unknown. import { Component, input, output } from "@angular/core"; import type { App } from "../../app"; import type { PlainError } from "../../errors"; import { shortId } from "../../format"; import { Icon } from "../../icon"; -import { toneAttribute } from "../../status-labels"; +import { statusLabel, statusTone, toneAttribute } from "../../status-labels"; import { OperateState } from "../operate-state"; import { timeAbsolute, timeIso, timeRelative } from "../operate-time"; import { incidentStatusLabel, incidentTone, + openedText, resolutionLabel, type EventLine, type IncidentView, @@ -61,6 +65,9 @@ export interface RecentEvents {

{{ label(i) }} + @if (i.test) { + Test run + }

@if (i.external_effects_may_continue) {

@@ -79,17 +86,62 @@ export interface RecentEvents { {{ i.origin_code }} } +

Opened
+
+ @if (i.opened_at && relative(i.opened_at)) { + + @if (i.opened_at_estimated) { + From the run's last recorded event: the incident's own opening + wasn't kept. + } + } @else { + Unknown + } +
+ @if (i.updated_at && relative(i.updated_at)) { +
Last changed
+
+ +
+ } +
Workflow
+
+ {{ + i.workflow ? i.workflow.name + " " + i.workflow.version : "Unknown" + }} +
Step
{{ i.node_id || "Not tied to a step" }}
+ @if (i.instance_key && i.instance_key !== i.node_id) { +
Step instance
+
+ {{ i.instance_key }} +
+ } @if (i.generation !== null) {
Attempt
{{ i.generation }}
}
Run
-
+
+ @if (i.run_status) { + {{ + runLabel(i.run_status) + }} + }
@if (i.resolution; as resolution) { @@ -200,8 +252,18 @@ export class IncidentDrawer { decision(kind: Parameters[0]) { return resolutionLabel(kind); } - /** A resolution time that isn't a date shows nothing rather than "Unknown". */ + /** A time that isn't a date shows nothing; the template says Unknown. */ relative(value: string) { return timeRelative(value, ""); } + /** When it opened, with an estimate marked as one. */ + opened(incident: IncidentView) { + return openedText(incident, (value) => this.relative(value)); + } + runLabel(status: string) { + return statusLabel("run", status); + } + runTone(status: string) { + return toneAttribute(statusTone("run", status)); + } } diff --git a/studio/src/app/operate/incidents/incidents-model.ts b/studio/src/app/operate/incidents/incidents-model.ts index dcd7edfb..6cba9522 100644 --- a/studio/src/app/operate/incidents/incidents-model.ts +++ b/studio/src/app/operate/incidents/incidents-model.ts @@ -48,6 +48,19 @@ export interface IncidentView { evidence_reference?: string | null; } | null; external_effects_may_continue: boolean; + /** The full step instance; left out for an incident not tied to one. */ + instance_key?: string; + /** When it opened; left out while unknown. */ + opened_at?: string; + /** + * The opening time was inferred from the run's last recorded event; left + * out (false) when it is exact. Never true without `opened_at`. + */ + opened_at_estimated?: boolean; + updated_at?: string; + workflow?: { name: string; version: string }; + run_status?: string; + test?: boolean; } export interface UnavailableIncident { id: string; @@ -150,54 +163,147 @@ export function checkResolution( }; } -/** Inbox filters; they live in the address. Active is the default. */ +/** + * Inbox filters, which the platform applies; they live in the address. + * Active incidents, newest opened first, without test runs by default. + */ export interface IncidentFilters { status: IncidentStatus | "all"; code: string; + workflow: string; + nodeId: string; + /** RFC 3339 instants, or "" for no bound. */ + openedAfter: string; + openedBefore: string; + includeTest: boolean; + order: IncidentOrder; } +export type IncidentOrder = "opened_desc" | "opened_asc" | "id"; export const defaultIncidentFilters: IncidentFilters = { status: "active", code: "", + workflow: "", + nodeId: "", + openedAfter: "", + openedBefore: "", + includeTest: false, + order: "opened_desc", }; -export function filterIncidents( - incidents: readonly IncidentRecord[], +/** The `incidents.list` query for the filters, from the start or after `cursor`. */ +export function incidentQuery( filters: IncidentFilters, -): IncidentRecord[] { - return incidents.filter((incident) => { - if (isUnavailableIncident(incident)) - return filters.status === "all" && !filters.code; - if (filters.status !== "all" && incident.status !== filters.status) - return false; - return !filters.code || incident.code === filters.code; + cursor: string | null, +): string { + const query = new URLSearchParams({ + limit: "50", + order: filters.order, + include_test: String(filters.includeTest), }); + if (filters.status !== "all") query.append("status", filters.status); + for (const [key, value] of [ + ["code", filters.code], + ["workflow", filters.workflow], + ["node_id", filters.nodeId], + ["opened_after", filters.openedAfter], + ["opened_before", filters.openedBefore], + ]) + if (value) query.set(key, value); + if (cursor) query.set("cursor", cursor); + return query.toString(); +} +/** How many filters differ from the defaults ("Clear filters" shows from one). */ +export function incidentFilterCount(filters: IncidentFilters): number { + return [ + filters.status !== "active", + filters.code, + filters.workflow, + filters.nodeId, + filters.openedAfter, + filters.openedBefore, + filters.includeTest, + ].filter(Boolean).length; } const statusValues: readonly string[] = ["all", "active", "resolved", "closed"]; +const orderValues: readonly string[] = ["opened_desc", "opened_asc", "id"]; +const instant = (value: string | null) => + value && Number.isFinite(Date.parse(value)) + ? new Date(value).toISOString() + : ""; export function incidentFiltersFromQuery(search: string): IncidentFilters { const query = new URLSearchParams(search); const status = query.get("status") ?? "active"; + const order = query.get("order") ?? ""; return { status: statusValues.includes(status) ? (status as IncidentFilters["status"]) : "active", code: query.get("code") ?? "", + workflow: query.get("workflow") ?? "", + nodeId: query.get("node_id") ?? "", + openedAfter: instant(query.get("opened_after")), + openedBefore: instant(query.get("opened_before")), + includeTest: query.get("include_test") === "true", + order: orderValues.includes(order) + ? (order as IncidentOrder) + : "opened_desc", }; } -export function incidentFiltersToQuery(filters: IncidentFilters): string { +/** The address query for the filters (and an open incident), "" for the defaults. */ +export function incidentFiltersToQuery( + filters: IncidentFilters, + incident = "", +): string { const query = new URLSearchParams(); if (filters.status !== "active") query.set("status", filters.status); if (filters.code) query.set("code", filters.code); + if (filters.workflow) query.set("workflow", filters.workflow); + if (filters.nodeId) query.set("node_id", filters.nodeId); + if (filters.openedAfter) query.set("opened_after", filters.openedAfter); + if (filters.openedBefore) query.set("opened_before", filters.openedBefore); + if (filters.includeTest) query.set("include_test", "true"); + if (filters.order !== "opened_desc") query.set("order", filters.order); + if (incident) query.set("incident", incident); const text = query.toString(); return text ? `?${text}` : ""; } -/** The codes the loaded incidents carry, sorted. */ -export function incidentCodes(incidents: readonly IncidentRecord[]) { - return [ - ...new Set( - incidents - .filter((item) => !isUnavailableIncident(item)) - .map((item) => (item as IncidentView).code), - ), - ].sort(); +/** The incident an address opens ("?incident=ID"), or "". */ +export function incidentFromQuery(search: string): string { + const id = new URLSearchParams(search).get("incident") ?? ""; + return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test( + id, + ) + ? id.toLowerCase() + : ""; +} +/** The codes and workflows the loaded incidents carry, sorted, for suggestions. */ +export function incidentSuggestions(incidents: readonly IncidentRecord[]) { + const codes = new Set(); + const workflows = new Set(); + const steps = new Set(); + for (const item of incidents) { + if (isUnavailableIncident(item)) continue; + codes.add(item.code); + if (item.workflow?.name) workflows.add(item.workflow.name); + if (item.node_id) steps.add(item.node_id); + } + return { + codes: [...codes].sort(), + workflows: [...workflows].sort(), + steps: [...steps].sort(), + }; +} +/** + * When an incident opened, for the drawer: "Unknown" without a time, and the + * estimate marked as one. + */ +export function openedText( + incident: Pick, + format: (value: string) => string, +): string { + if (!incident.opened_at) return "Unknown"; + const text = format(incident.opened_at); + if (!text) return "Unknown"; + return (incident.opened_at_estimated ?? false) ? `${text} (estimated)` : text; } /** The last engine events of a run: their types and sequence numbers only. */ diff --git a/studio/src/app/operate/incidents/incidents-page.ts b/studio/src/app/operate/incidents/incidents-page.ts index ef63e92a..484f3dd6 100644 --- a/studio/src/app/operate/incidents/incidents-page.ts +++ b/studio/src/app/operate/incidents/incidents-page.ts @@ -15,10 +15,13 @@ limitations under the License. Author: Firefly Software Foundation SPDX-License-Identifier: Apache-2.0 */ -// Operate › Incidents: active incidents first, filters kept in the address, -// a drawer with what happened, and the Resolve incident dialog. The server -// checks incident.resolve, the revision and the receipt on every decision. -// Reloads every 15 s while visible, except while a decision is being made. +// Operate › Incidents: active incidents, newest opened first, filtered by the +// platform; the address keeps the filters and the open incident +// ("?incident=ID"), which opens from incidents.read even when no loaded page +// lists it. A drawer says what happened, and the Resolve incident dialog +// decides on the revision the person looked at. The server checks +// incident.resolve, the revision and the receipt on every decision. Reloads +// every 15 s while visible, except while a decision is being made. import { ChangeDetectionStrategy, ChangeDetectorRef, @@ -38,11 +41,11 @@ import { Icon } from "../../icon"; import { ModalSheet, sheetWhen } from "../../modal-sheet"; import { toneAttribute } from "../../status-labels"; import { MAX_PAGES } from "../operate-limits"; +import type { Page } from "../../operations/deployment-contracts"; import { ListReads, environmentOf, listFailure, - readPages, scopeKeyOf, } from "../operate-list"; import { OperateState } from "../operate-state"; @@ -50,13 +53,16 @@ import { viewFromPath, viewPath } from "../operate-routes"; import { Poller, browserEnvironment } from "../operate-store"; import { RefreshStatus } from "../refresh-status"; import { IncidentDrawer, type RecentEvents } from "./incident-drawer"; +import { timeAbsolute, timeIso, timeRelative } from "../operate-time"; import { defaultIncidentFilters, - filterIncidents, - incidentCodes, + incidentFilterCount, incidentFiltersFromQuery, incidentFiltersToQuery, + incidentFromQuery, + incidentQuery, incidentStatusLabel, + incidentSuggestions, incidentTone, isUnavailableIncident, lastEvents, @@ -67,6 +73,8 @@ import { } from "./incidents-model"; import { ResolveIncidentDialog } from "./resolve-incident-dialog"; +/** The longest opening range the platform accepts: 400 days. */ +const MAX_RANGE_MS = 400 * 86_400_000; /** A run's events are read 100 at a time, at most 1,000. */ const EVENT_PAGES = 10; const noEvents: RecentEvents = { @@ -134,7 +142,23 @@ const noEvents: RecentEvents = { } @else if (!loaded) { } @else { - } - } @else { -
-
-
+
+

{{ outcome.title }}

+ @if (!bulkRunning) { + + } +
+
    + @for (result of outcome.results; track result.id) { +
  • + {{ short(result.id) }}{{ result.text }} +
    • + } +
+
+ } + @if (!workers.length) { + @if (filterCount(filters)) { + -
-
- WorkerStatusReleaseLoadLast contact + + + } @else { + + } + } @else { +
+
+
+
+
+ @if (canSelect()) { + + } + WorkerStatusWhereReleaseVersionLoadLast contactRegistered +
-
-
- @for (worker of visible(); track worker.id) { - @if (unavailable(worker)) { -
- Unavailable workerStudio can't show this worker. -
- } @else { - @let w = status(worker); - @let load = loadOf(w); -
- {{ typesText(w) }} - @for (badge of badges(w); track badge.label) { - {{ badge.label }} + @for (worker of workers; track worker.id) { + @if (unavailable(worker)) { +
+ @if (canSelect()) { + + } + Unavailable workerStudio can't show this worker. +
+ } @else { + @let w = status(worker); + @let load = loadOf(w); +
+ @if (canSelect()) { + + } +
+ role="cell" + data-wide + class="cell-text" + [attr.title]="where(w)" + >{{ where(w) }}{{ + short(w.release_id) + }}{{ typesText(w) }}{{ version(w) || "Unknown" }} + @if (w.last_seen_at) { + + } @else { + Never + } + @if (w.registered_at) { + + } @else { + Unknown + } + +
+ } } - } +
+ @if (canLoadMore) { + + } @else if (nextCursor) { +

+ Showing the first 500 workers. Narrow the filters to see + others. +

+ }
+ @if (selected) { + + }
- @if (selected) { -
} - @if (v === "runs") { - - } @if (v === "workflows") {
}
- @if (v !== "runs") { - - } + @if (v === "tasks") {